From 8c56e4516e55c7ed9c14779f0e77e00f055d9a81 Mon Sep 17 00:00:00 2001
From: vozerov-gridgain <vozerov@gridgain.com>
Date: Fri, 2 Sep 2016 18:05:16 +0300
Subject: [PATCH 001/446] IGNITE-3827: Removed double marshalling of keys in
 DataStreamerImpl.addData(Map) method.

---
 .../datastreamer/DataStreamerImpl.java        | 23 ++++++++-----------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index e565cbaa43676..a3bae249d13ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -87,7 +87,6 @@
 import org.apache.ignite.internal.util.lang.GridPeerDeployAware;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.A;
@@ -513,23 +512,19 @@ else if (autoFlushFreq == 0)
 
             activeFuts.add(resFut);
 
-            Collection<KeyCacheObject> keys = null;
+            Collection<KeyCacheObject> keys =
+                new GridConcurrentHashSet<>(entries.size(), U.capacity(entries.size()), 1);
 
-            if (entries.size() > 1) {
-                keys = new GridConcurrentHashSet<>(entries.size(), U.capacity(entries.size()), 1);
+            Collection<DataStreamerEntry> entries0 = new ArrayList<>(entries.size());
 
-                for (Map.Entry<K, V> entry : entries)
-                    keys.add(cacheObjProc.toCacheKeyObject(cacheObjCtx, null, entry.getKey(), true));
-            }
+            for (Map.Entry<K, V> entry : entries) {
+                KeyCacheObject key = cacheObjProc.toCacheKeyObject(cacheObjCtx, null, entry.getKey(), true);
+                CacheObject val = cacheObjProc.toCacheObject(cacheObjCtx, entry.getValue(), true);
 
-            Collection<? extends DataStreamerEntry> entries0 = F.viewReadOnly(entries, new C1<Entry<K, V>, DataStreamerEntry>() {
-                @Override public DataStreamerEntry apply(Entry<K, V> e) {
-                    KeyCacheObject key = cacheObjProc.toCacheKeyObject(cacheObjCtx, null, e.getKey(), true);
-                    CacheObject val = cacheObjProc.toCacheObject(cacheObjCtx, e.getValue(), true);
+                keys.add(key);
 
-                    return new DataStreamerEntry(key, val);
-                }
-            });
+                entries0.add(new DataStreamerEntry(key, val));
+            }
 
             load0(entries0, resFut, keys, 0);
 

From 4dc624fc9a8852f77f1fe7db4dc06a474b34c2eb Mon Sep 17 00:00:00 2001
From: vozerov-gridgain <vozerov@gridgain.com>
Date: Fri, 2 Sep 2016 18:23:09 +0300
Subject: [PATCH 002/446] IGNITE-3829: Optimized affinity key field name
 handling.

---
 .../CacheObjectBinaryProcessorImpl.java       | 35 +++++++++++++++----
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 03378743c2ee3..ecd27f7a19af7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.binary.BinaryBasicNameMapper;
+import org.apache.ignite.binary.BinaryField;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryObjectBuilder;
 import org.apache.ignite.binary.BinaryObjectException;
@@ -89,6 +90,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T1;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -159,6 +161,9 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
     /** Metadata updates collected before metadata cache is initialized. */
     private final Map<Integer, BinaryMetadata> metaBuf = new ConcurrentHashMap<>();
 
+    /** Cached affinity key field names. */
+    private final ConcurrentHashMap<Integer, T1<BinaryField>> affKeyFields = new ConcurrentHashMap<>();
+
     /**
      * @param ctx Kernal context.
      */
@@ -684,22 +689,38 @@ public GridBinaryMarshaller marshaller() {
      * @return Affinity key.
      */
     public Object affinityKey(BinaryObject po) {
+        // Fast path for already cached field.
+        if (po instanceof BinaryObjectEx) {
+            int typeId = ((BinaryObjectEx)po).typeId();
+
+            T1<BinaryField> fieldHolder = affKeyFields.get(typeId);
+
+            if (fieldHolder != null) {
+                BinaryField field = fieldHolder.get();
+
+                return field != null ? field.value(po) : po;
+            }
+        }
+
+        // Slow path if affinity field is not cached yet.
         try {
             BinaryType meta = po instanceof BinaryObjectEx ? ((BinaryObjectEx)po).rawType() : po.type();
 
             if (meta != null) {
-                String affKeyFieldName = meta.affinityKeyFieldName();
+                String name = meta.affinityKeyFieldName();
+
+                affKeyFields.putIfAbsent(meta.typeId(), new T1<>(meta.field(name)));
 
-                if (affKeyFieldName != null)
-                    return po.field(affKeyFieldName);
+                if (name != null)
+                    return po.field(name);
             }
             else if (po instanceof BinaryObjectEx) {
-                int id = ((BinaryObjectEx)po).typeId();
+                int typeId = ((BinaryObjectEx)po).typeId();
 
-                String affKeyFieldName = binaryCtx.affinityKeyFieldName(id);
+                String name = binaryCtx.affinityKeyFieldName(typeId);
 
-                if (affKeyFieldName != null)
-                    return po.field(affKeyFieldName);
+                if (name != null)
+                    return po.field(name);
             }
         }
         catch (BinaryObjectException e) {

From 7b0d2326ccc62afd5d162b056398f96d8d7c9100 Mon Sep 17 00:00:00 2001
From: vozerov-gridgain <vozerov@gridgain.com>
Date: Mon, 5 Sep 2016 10:19:48 +0300
Subject: [PATCH 003/446] IGNITE-3829: Additional fix.

---
 .../cache/binary/CacheObjectBinaryProcessorImpl.java  | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index ecd27f7a19af7..82e67aca1518a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -709,10 +709,15 @@ public Object affinityKey(BinaryObject po) {
             if (meta != null) {
                 String name = meta.affinityKeyFieldName();
 
-                affKeyFields.putIfAbsent(meta.typeId(), new T1<>(meta.field(name)));
+                if (name != null) {
+                    BinaryField field = meta.field(name);
 
-                if (name != null)
-                    return po.field(name);
+                    affKeyFields.putIfAbsent(meta.typeId(), new T1<>(field));
+
+                    return field.value(po);
+                }
+                else
+                    affKeyFields.putIfAbsent(meta.typeId(), new T1<BinaryField>(null));
             }
             else if (po instanceof BinaryObjectEx) {
                 int typeId = ((BinaryObjectEx)po).typeId();

From 9c6824b4f33fbdead64299d9e0c34365d5d4a570 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Thu, 24 Nov 2016 16:27:05 +0300
Subject: [PATCH 004/446] IGNITE-3958 Fixed "Client node should not start rest
 processor".

---
 .../rest/RestProcessorMultiStartSelfTest.java | 48 ++++++++++++++++++-
 .../apache/ignite/IgniteSystemProperties.java |  6 +++
 .../apache/ignite/internal/IgniteKernal.java  | 12 ++++-
 .../processors/rest/GridRestProcessor.java    | 15 ++++++
 4 files changed, 79 insertions(+), 2 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/RestProcessorMultiStartSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/RestProcessorMultiStartSelfTest.java
index 7714dbfb4299b..5b880799c86bc 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/RestProcessorMultiStartSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/RestProcessorMultiStartSelfTest.java
@@ -17,8 +17,10 @@
 
 package org.apache.ignite.internal.processors.rest;
 
+import java.util.Map;
 import org.apache.ignite.configuration.ConnectorConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -28,15 +30,26 @@ public class RestProcessorMultiStartSelfTest extends GridCommonAbstractTest {
     /** */
     private static final int GRID_CNT = 3;
 
+    /** */
+    private static boolean client = false;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
         cfg.setConnectorConfiguration(new ConnectorConfiguration());
+        cfg.setClientMode(client);
 
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        client = false;
+    }
+
     /**
      * Test that multiple nodes can start with JETTY enabled.
      *
@@ -53,4 +66,37 @@ public void testMultiStart() throws Exception {
             stopAllGrids();
         }
     }
-}
\ No newline at end of file
+
+    /**
+     * Test that multiple nodes can start with JETTY enabled.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMultiStartWithClient() throws Exception {
+        try {
+            int clnIdx = GRID_CNT - 1;
+
+            for (int i = 0; i < clnIdx; i++) {
+                startGrid(i);
+
+                GridRestProcessor rest = grid(i).context().rest();
+
+                assertNotNull(rest);
+                assertFalse(((Map)GridTestUtils.getFieldValue(rest, "handlers")).isEmpty());
+            }
+
+            client = true;
+
+            startGrid(clnIdx);
+
+            GridRestProcessor rest = grid(GRID_CNT - 1).context().rest();
+
+            // Check that rest processor doesn't start.
+            assertNotNull(rest);
+            assertTrue(((Map)GridTestUtils.getFieldValue(rest, "handlers")).isEmpty());
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 043c95ade0fbc..de6cbed1fb3a5 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -109,6 +109,12 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_REST_MAX_TASK_RESULTS = "IGNITE_REST_MAX_TASK_RESULTS";
 
+    /**
+     * This property allows to override default behavior that rest processor
+     * doesn't start on client node. If set {@code true} than rest processor will be started on client node.
+     */
+    public static final String IGNITE_REST_START_ON_CLIENT = "IGNITE_REST_START_ON_CLIENT";
+
     /**
      * This property defines the maximum number of attempts to remap near get to the same
      * primary node. Remapping may be needed when topology is changed concurrently with
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 1963509d62908..ef9c651ad99a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -179,6 +179,7 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DAEMON;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_NO_ASCII;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_REST_START_ON_CLIENT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_STARVATION_CHECK_INTERVAL;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SUCCESS_FILE;
@@ -1632,7 +1633,16 @@ private String onOff(boolean b) {
     private boolean isRestEnabled() {
         assert cfg != null;
 
-        return cfg.getConnectorConfiguration() != null;
+        return cfg.getConnectorConfiguration() != null &&
+            // By default rest processor doesn't start on client nodes.
+            (!isClientNode() || (isClientNode() && IgniteSystemProperties.getBoolean(IGNITE_REST_START_ON_CLIENT)));
+    }
+
+    /**
+     * @return {@code True} if node client or daemon otherwise {@code false}.
+     */
+    private boolean isClientNode() {
+        return cfg.isClientMode() || cfg.isDaemon();
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
index 6d20547e8fa41..3f0478564d4df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
@@ -75,6 +75,7 @@
 import org.apache.ignite.thread.IgniteThread;
 import org.jsr166.LongAdder8;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_REST_START_ON_CLIENT;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_AUTH_FAILED;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_FAILED;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_SECURITY_CHECK_FAILED;
@@ -436,6 +437,13 @@ public GridRestProcessor(GridKernalContext ctx) {
     /** {@inheritDoc} */
     @Override public void start() throws IgniteCheckedException {
         if (isRestEnabled()) {
+            if (notStartOnClient()) {
+                U.quietAndInfo(log, "REST protocols do not start on client node. " +
+                    "To start the protocols on client node set '-DIGNITE_REST_START_ON_CLIENT=true' system property.");
+
+                return;
+            }
+
             // Register handlers.
             addHandler(new GridCacheCommandHandler(ctx));
             addHandler(new GridTaskCommandHandler(ctx));
@@ -471,6 +479,13 @@ public GridRestProcessor(GridKernalContext ctx) {
         }
     }
 
+    /**
+     * @return {@code True} if rest processor should not start on client node.
+     */
+    private boolean notStartOnClient() {
+        return ctx.clientNode() && !IgniteSystemProperties.getBoolean(IGNITE_REST_START_ON_CLIENT);
+    }
+
     /** {@inheritDoc} */
     @Override public void onKernalStart() throws IgniteCheckedException {
         if (isRestEnabled()) {

From 56998e704e9a67760c70481c10c56e72c0a866bb Mon Sep 17 00:00:00 2001
From: Konstantin Dudkov <kdudkov@ya.ru>
Date: Fri, 28 Oct 2016 16:27:34 +0300
Subject: [PATCH 005/446] ignite-4088 Added methods to create/destroy multiple
 caches. This closes #1174.

(cherry picked from commit f445e7b)
---
 .../main/java/org/apache/ignite/Ignite.java   |  77 ++++-
 .../apache/ignite/internal/IgniteKernal.java  |  81 +++++
 .../processors/cache/GridCacheProcessor.java  | 291 +++++++++++++-----
 .../IgniteDynamicCacheStartSelfTest.java      | 217 +++++++++++--
 .../processors/igfs/IgfsIgniteMock.java       |  19 ++
 .../testframework/junits/IgniteMock.java      |  15 +
 .../junits/multijvm/IgniteProcessProxy.java   |  15 +
 .../org/apache/ignite/IgniteSpringBean.java   |  21 ++
 8 files changed, 616 insertions(+), 120 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java
index bd21468ae73b1..0de08d56584a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignite.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java
@@ -20,6 +20,7 @@
 import java.util.Collection;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
+import javax.cache.CacheException;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cluster.ClusterGroup;
@@ -220,8 +221,24 @@ public interface Ignite extends AutoCloseable {
      *
      * @param cacheCfg Cache configuration to use.
      * @return Instance of started cache.
+     * @throws CacheException If a cache with the same name already exists or other error occurs.
      */
-    public <K, V> IgniteCache<K, V> createCache(CacheConfiguration<K, V> cacheCfg);
+    public <K, V> IgniteCache<K, V> createCache(CacheConfiguration<K, V> cacheCfg) throws CacheException;
+
+    /**
+     * Dynamically starts new caches with the given cache configurations.
+     * <p>
+     * If local node is an affinity node, this method will return the instance of started caches.
+     * Otherwise, it will create a client caches on local node.
+     * <p>
+     * If for one of configurations a cache with the same name already exists in the grid, an exception will be thrown regardless
+     * whether the given configuration matches the configuration of the existing cache or not.
+     *
+     * @param cacheCfgs Collection of cache configuration to use.
+     * @return Collection of instances of started caches.
+     * @throws CacheException If one of created caches exists or other error occurs.
+     */
+    public Collection<IgniteCache> createCaches(Collection<CacheConfiguration> cacheCfgs) throws CacheException;
 
     /**
      * Dynamically starts new cache using template configuration.
@@ -233,8 +250,9 @@ public interface Ignite extends AutoCloseable {
      *
      * @param cacheName Cache name.
      * @return Instance of started cache.
+     * @throws CacheException If a cache with the same name already exists or other error occurs.
      */
-    public <K, V> IgniteCache<K, V> createCache(String cacheName);
+    public <K, V> IgniteCache<K, V> createCache(String cacheName) throws CacheException;
 
     /**
      * Gets existing cache with the given name or creates new one with the given configuration.
@@ -245,23 +263,39 @@ public interface Ignite extends AutoCloseable {
      *
      * @param cacheCfg Cache configuration to use.
      * @return Existing or newly created cache.
+     * @throws CacheException If error occurs.
      */
-    public <K, V> IgniteCache<K, V> getOrCreateCache(CacheConfiguration<K, V> cacheCfg);
+    public <K, V> IgniteCache<K, V> getOrCreateCache(CacheConfiguration<K, V> cacheCfg) throws CacheException;
 
     /**
      * Gets existing cache with the given name or creates new one using template configuration.
      *
      * @param cacheName Cache name.
      * @return Existing or newly created cache.
+     * @throws CacheException If error occurs.
      */
-    public <K, V> IgniteCache<K, V> getOrCreateCache(String cacheName);
+    public <K, V> IgniteCache<K, V> getOrCreateCache(String cacheName) throws CacheException;
+
+    /**
+     * Gets existing caches with the given name or created one with the given configuration.
+     * <p>
+     * If a cache with the same name already exist, this method will not check that the given
+     * configuration matches the configuration of existing cache and will return an instance
+     * of the existing cache.
+     *
+     * @param cacheCfgs Collection of cache configuration to use.
+     * @return Collection of existing or newly created caches.
+     * @throws CacheException If error occurs.
+     */
+    public Collection<IgniteCache> getOrCreateCaches(Collection<CacheConfiguration> cacheCfgs) throws CacheException;
 
     /**
      * Adds cache configuration template.
      *
      * @param cacheCfg Cache configuration template.
+     * @throws CacheException If error occurs.
      */
-    public <K, V> void addCacheConfiguration(CacheConfiguration<K, V> cacheCfg);
+    public <K, V> void addCacheConfiguration(CacheConfiguration<K, V> cacheCfg) throws CacheException;
 
     /**
      * Dynamically starts new cache with the given cache configuration.
@@ -275,10 +309,11 @@ public interface Ignite extends AutoCloseable {
      * @param cacheCfg Cache configuration to use.
      * @param nearCfg Near cache configuration to use on local node in case it is not an
      *      affinity node.
+     * @throws CacheException If a cache with the same name already exists or other error occurs.
      * @return Instance of started cache.
      */
     public <K, V> IgniteCache<K, V> createCache(CacheConfiguration<K, V> cacheCfg,
-        NearCacheConfiguration<K, V> nearCfg);
+        NearCacheConfiguration<K, V> nearCfg) throws CacheException;
 
     /**
      * Gets existing cache with the given cache configuration or creates one if it does not exist.
@@ -293,9 +328,10 @@ public <K, V> IgniteCache<K, V> createCache(CacheConfiguration<K, V> cacheCfg,
      * @param cacheCfg Cache configuration.
      * @param nearCfg Near cache configuration for client.
      * @return {@code IgniteCache} instance.
+     * @throws CacheException If error occurs.
      */
     public <K, V> IgniteCache<K, V> getOrCreateCache(CacheConfiguration<K, V> cacheCfg,
-        NearCacheConfiguration<K, V> nearCfg);
+        NearCacheConfiguration<K, V> nearCfg) throws CacheException;
 
     /**
      * Starts a near cache on local node if cache was previously started with one of the
@@ -305,8 +341,10 @@ public <K, V> IgniteCache<K, V> getOrCreateCache(CacheConfiguration<K, V> cacheC
      * @param cacheName Cache name.
      * @param nearCfg Near cache configuration.
      * @return Cache instance.
+     * @throws CacheException If error occurs.
      */
-    public <K, V> IgniteCache<K, V> createNearCache(@Nullable String cacheName, NearCacheConfiguration<K, V> nearCfg);
+    public <K, V> IgniteCache<K, V> createNearCache(@Nullable String cacheName, NearCacheConfiguration<K, V> nearCfg)
+        throws CacheException;
 
     /**
      * Gets existing near cache with the given name or creates a new one.
@@ -314,15 +352,26 @@ public <K, V> IgniteCache<K, V> getOrCreateCache(CacheConfiguration<K, V> cacheC
      * @param cacheName Cache name.
      * @param nearCfg Near configuration.
      * @return {@code IgniteCache} instance.
+     * @throws CacheException If error occurs.
      */
-    public <K, V> IgniteCache<K, V> getOrCreateNearCache(@Nullable String cacheName, NearCacheConfiguration<K, V> nearCfg);
+    public <K, V> IgniteCache<K, V> getOrCreateNearCache(@Nullable String cacheName, NearCacheConfiguration<K, V> nearCfg)
+        throws CacheException;
 
     /**
      * Stops dynamically started cache.
      *
      * @param cacheName Cache name to stop.
+     * @throws CacheException If error occurs.
+     */
+    public void destroyCache(String cacheName) throws CacheException;
+
+    /**
+     * Stops dynamically started caches.
+     *
+     * @param cacheNames Collection of cache names to stop.
+     * @throws CacheException If error occurs.
      */
-    public void destroyCache(String cacheName);
+    public void destroyCaches(Collection<String> cacheNames) throws CacheException;
 
     /**
      * Gets an instance of {@link IgniteCache} API. {@code IgniteCache} is a fully-compatible
@@ -330,8 +379,9 @@ public <K, V> IgniteCache<K, V> getOrCreateCache(CacheConfiguration<K, V> cacheC
      *
      * @param name Cache name.
      * @return Instance of the cache for the specified name.
+     * @throws CacheException If error occurs.
      */
-    public <K, V> IgniteCache<K, V> cache(@Nullable String name);
+    public <K, V> IgniteCache<K, V> cache(@Nullable String name) throws CacheException;
 
     /**
      * Gets the collection of names of currently available caches.
@@ -357,8 +407,9 @@ public <K, V> IgniteCache<K, V> getOrCreateCache(CacheConfiguration<K, V> cacheC
      *
      * @param cacheName Cache name ({@code null} for default cache).
      * @return Data streamer.
+     * @throws IllegalStateException If node is stopping.
      */
-    public <K, V> IgniteDataStreamer<K, V> dataStreamer(@Nullable String cacheName);
+    public <K, V> IgniteDataStreamer<K, V> dataStreamer(@Nullable String cacheName) throws IllegalStateException;
 
     /**
      * Gets an instance of IGFS (Ignite In-Memory File System). If one is not
@@ -372,7 +423,7 @@ public <K, V> IgniteCache<K, V> getOrCreateCache(CacheConfiguration<K, V> cacheC
      * @return IGFS instance.
      * @throws IllegalArgumentException If IGFS with such name is not configured.
      */
-    public IgniteFileSystem fileSystem(String name);
+    public IgniteFileSystem fileSystem(String name) throws IllegalArgumentException;
 
     /**
      * Gets all instances of IGFS (Ignite In-Memory File System).
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index ef9c651ad99a2..c5365069b809b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2541,6 +2541,33 @@ public <K, V> IgniteInternalCache<K, V> getCache(@Nullable String name) {
         }
     }
 
+
+    /** {@inheritDoc} */
+    @Override public Collection<IgniteCache> createCaches(Collection<CacheConfiguration> cacheCfgs) {
+        A.notNull(cacheCfgs, "cacheCfgs");
+
+        guard();
+
+        try {
+            ctx.cache().dynamicStartCaches(cacheCfgs,
+                true,
+                true).get();
+
+            List<IgniteCache> createdCaches = new ArrayList<>(cacheCfgs.size());
+
+            for (CacheConfiguration cacheCfg : cacheCfgs)
+                createdCaches.add(ctx.cache().publicJCache(cacheCfg.getName()));
+
+            return createdCaches;
+        }
+        catch (IgniteCheckedException e) {
+            throw CU.convertToCacheException(e);
+        }
+        finally {
+            unguard();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> createCache(String cacheName) {
         guard();
@@ -2584,6 +2611,32 @@ public <K, V> IgniteInternalCache<K, V> getCache(@Nullable String name) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public Collection<IgniteCache> getOrCreateCaches(Collection<CacheConfiguration> cacheCfgs) {
+        A.notNull(cacheCfgs, "cacheCfgs");
+
+        guard();
+
+        try {
+            ctx.cache().dynamicStartCaches(cacheCfgs,
+                false,
+                true).get();
+
+            List<IgniteCache> createdCaches = new ArrayList<>(cacheCfgs.size());
+
+            for (CacheConfiguration cacheCfg : cacheCfgs)
+                createdCaches.add(ctx.cache().publicJCache(cacheCfg.getName()));
+
+            return createdCaches;
+        }
+        catch (IgniteCheckedException e) {
+            throw CU.convertToCacheException(e);
+        }
+        finally {
+            unguard();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> createCache(
         CacheConfiguration<K, V> cacheCfg,
@@ -2745,6 +2798,18 @@ private void checkNearCacheStarted(IgniteCacheProxy<?, ?> cache) throws IgniteCh
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void destroyCaches(Collection<String> cacheNames) {
+        IgniteInternalFuture stopFut = destroyCachesAsync(cacheNames, true);
+
+        try {
+            stopFut.get();
+        }
+        catch (IgniteCheckedException e) {
+            throw CU.convertToCacheException(e);
+        }
+    }
+
     /**
      * @param cacheName Cache name.
      * @param checkThreadTx If {@code true} checks that current thread does not have active transactions.
@@ -2761,6 +2826,22 @@ public IgniteInternalFuture<?> destroyCacheAsync(String cacheName, boolean check
         }
     }
 
+    /**
+     * @param cacheNames Collection of cache names.
+     * @param checkThreadTx If {@code true} checks that current thread does not have active transactions.
+     * @return Ignite future.
+     */
+    public IgniteInternalFuture<?> destroyCachesAsync(Collection<String> cacheNames, boolean checkThreadTx) {
+        guard();
+
+        try {
+            return ctx.cache().dynamicDestroyCaches(cacheNames, checkThreadTx);
+        }
+        finally {
+            unguard();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> getOrCreateCache(String cacheName) {
         guard();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 5e777fd8a39c2..0e0d76941ae49 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2284,99 +2284,92 @@ public IgniteInternalFuture<?> dynamicStartCache(
         if (checkThreadTx)
             checkEmptyTransactions();
 
-        DynamicCacheDescriptor desc = registeredCaches.get(maskNull(cacheName));
-
-        DynamicCacheChangeRequest req = new DynamicCacheChangeRequest(cacheName, ctx.localNodeId());
-
-        req.failIfExists(failIfExists);
-
-        if (ccfg != null) {
-            try {
-                cloneCheckSerializable(ccfg);
-            }
-            catch (IgniteCheckedException e) {
-                return new GridFinishedFuture<>(e);
-            }
-
-            if (desc != null) {
-                if (failIfExists) {
-                    return new GridFinishedFuture<>(new CacheExistsException("Failed to start cache " +
-                        "(a cache with the same name is already started): " + cacheName));
-                }
-                else {
-                    CacheConfiguration descCfg = desc.cacheConfiguration();
-
-                    // Check if we were asked to start a near cache.
-                    if (nearCfg != null) {
-                        if (CU.affinityNode(ctx.discovery().localNode(), descCfg.getNodeFilter())) {
-                            // If we are on a data node and near cache was enabled, return success, else - fail.
-                            if (descCfg.getNearConfiguration() != null)
-                                return new GridFinishedFuture<>();
-                            else
-                                return new GridFinishedFuture<>(new IgniteCheckedException("Failed to start near " +
-                                    "cache (local node is an affinity node for cache): " + cacheName));
-                        }
-                        else
-                            // If local node has near cache, return success.
-                            req.clientStartOnly(true);
-                    }
-                    else
-                        req.clientStartOnly(true);
+        try {
+            DynamicCacheChangeRequest req = prepareCacheChangeRequest(
+                ccfg,
+                cacheName,
+                nearCfg,
+                cacheType,
+                failIfExists,
+                failIfNotStarted);
 
-                    req.deploymentId(desc.deploymentId());
+            if (req != null)
+                return F.first(initiateCacheChanges(F.asList(req), failIfExists));
+            else
+                return new GridFinishedFuture<>();
+        }
+        catch (Exception e) {
+            return new GridFinishedFuture<>(e);
+        }
+    }
 
-                    req.startCacheConfiguration(descCfg);
-                }
-            }
-            else {
-                req.deploymentId(IgniteUuid.randomUuid());
+    /**
+     * Dynamically starts multiple caches.
+     *
+     * @param ccfgList Collection of cache configuration.
+     * @param failIfExists Fail if exists flag.
+     * @param checkThreadTx If {@code true} checks that current thread does not have active transactions.
+     * @return Future that will be completed when all caches are deployed.
+     */
+    public IgniteInternalFuture<?> dynamicStartCaches(
+        Collection<CacheConfiguration> ccfgList,
+        boolean failIfExists,
+        boolean checkThreadTx
+    ) {
+        return dynamicStartCaches(ccfgList, CacheType.USER, failIfExists, checkThreadTx);
+    }
 
-                try {
-                    CacheConfiguration cfg = new CacheConfiguration(ccfg);
+    /**
+     * Dynamically starts multiple caches.
+     *
+     * @param ccfgList Collection of cache configuration.
+     * @param cacheType Cache type.
+     * @param failIfExists Fail if exists flag.
+     * @param checkThreadTx If {@code true} checks that current thread does not have active transactions.
+     * @return Future that will be completed when all caches are deployed.
+     */
+    private IgniteInternalFuture<?> dynamicStartCaches(
+        Collection<CacheConfiguration> ccfgList,
+        CacheType cacheType,
+        boolean failIfExists,
+        boolean checkThreadTx
+    ) {
+        if (checkThreadTx)
+            checkEmptyTransactions();
 
-                    CacheObjectContext cacheObjCtx = ctx.cacheObjects().contextForCache(cfg);
+        List<DynamicCacheChangeRequest> reqList = new ArrayList<>(ccfgList.size());
 
-                    initialize(false, cfg, cacheObjCtx);
+        try {
+            for (CacheConfiguration ccfg : ccfgList) {
+                DynamicCacheChangeRequest req = prepareCacheChangeRequest(
+                    ccfg,
+                    ccfg.getName(),
+                    null,
+                    cacheType,
+                    failIfExists,
+                    true
+                );
 
-                    req.startCacheConfiguration(cfg);
-                }
-                catch (IgniteCheckedException e) {
-                    return new GridFinishedFuture(e);
-                }
+                if (req != null)
+                    reqList.add(req);
             }
         }
-        else {
-            req.clientStartOnly(true);
-
-            if (desc != null)
-                ccfg = desc.cacheConfiguration();
-
-            if (ccfg == null) {
-                if (failIfNotStarted)
-                    return new GridFinishedFuture<>(new CacheExistsException("Failed to start client cache " +
-                        "(a cache with the given name is not started): " + cacheName));
-                else
-                    return new GridFinishedFuture<>();
-            }
-
-            req.deploymentId(desc.deploymentId());
-            req.startCacheConfiguration(ccfg);
+        catch (Exception e) {
+            return new GridFinishedFuture<>(e);
         }
 
-        // Fail cache with swap enabled creation on grid without swap space SPI.
-        if (ccfg.isSwapEnabled())
-            for (ClusterNode n : ctx.discovery().allNodes())
-                if (!GridCacheUtils.clientNode(n) && !GridCacheUtils.isSwapEnabled(n))
-                    return new GridFinishedFuture<>(new IgniteCheckedException("Failed to start cache " +
-                        cacheName + " with swap enabled: Remote Node with ID " + n.id().toString().toUpperCase() +
-                        " has not swap SPI configured"));
+        if (!reqList.isEmpty()) {
+            GridCompoundFuture<?, ?> compoundFut = new GridCompoundFuture<>();
 
-        if (nearCfg != null)
-            req.nearCacheConfiguration(nearCfg);
+            for (DynamicCacheStartFuture fut : initiateCacheChanges(reqList, failIfExists))
+                compoundFut.add((IgniteInternalFuture)fut);
 
-        req.cacheType(cacheType);
+            compoundFut.markInitialized();
 
-        return F.first(initiateCacheChanges(F.asList(req), failIfExists));
+            return compoundFut;
+        }
+        else
+            return new GridFinishedFuture<>();
     }
 
     /**
@@ -2395,6 +2388,35 @@ public IgniteInternalFuture<?> dynamicDestroyCache(String cacheName, boolean che
         return F.first(initiateCacheChanges(F.asList(t), false));
     }
 
+    /**
+     * @param cacheNames Collection of cache names to destroy.
+     * @param checkThreadTx If {@code true} checks that current thread does not have active transactions.
+     * @return Future that will be completed when cache is destroyed.
+     */
+    public IgniteInternalFuture<?> dynamicDestroyCaches(Collection<String> cacheNames, boolean checkThreadTx) {
+        if (checkThreadTx)
+            checkEmptyTransactions();
+
+        List<DynamicCacheChangeRequest> reqs = new ArrayList<>(cacheNames.size());
+
+        for (String cacheName : cacheNames) {
+            DynamicCacheChangeRequest t = new DynamicCacheChangeRequest(cacheName, ctx.localNodeId());
+
+            t.stop(true);
+
+            reqs.add(t);
+        }
+
+        GridCompoundFuture<?, ?> compoundFut = new GridCompoundFuture<>();
+
+        for (DynamicCacheStartFuture fut : initiateCacheChanges(reqs, false))
+            compoundFut.add((IgniteInternalFuture)fut);
+
+        compoundFut.markInitialized();
+
+        return compoundFut;
+    }
+
     /**
      * @param cacheName Cache name to close.
      * @return Future that will be completed when cache is closed.
@@ -2416,6 +2438,7 @@ public IgniteInternalFuture<?> dynamicCloseCache(String cacheName) {
 
     /**
      * @param reqs Requests.
+     * @param failIfExists Fail if exists flag.
      * @return Collection of futures.
      */
     @SuppressWarnings("TypeMayBeWeakened")
@@ -3607,6 +3630,114 @@ private <T> T withBinaryContext(IgniteOutClosureX<T> c) throws IgniteCheckedExce
         }
     }
 
+    /**
+     * Prepares DynamicCacheChangeRequest for cache creation.
+     *
+     * @param ccfg Cache configuration
+     * @param cacheName Cache name
+     * @param nearCfg Near cache configuration
+     * @param cacheType Cache type
+     * @param failIfExists Fail if exists flag.
+     * @param failIfNotStarted If {@code true} fails if cache is not started.
+     * @return Request or {@code null} if cache already exists.
+     * @throws IgniteCheckedException if some of pre-checks failed
+     * @throws CacheExistsException if cache exists and failIfExists flag is {@code true}
+     */
+    private DynamicCacheChangeRequest prepareCacheChangeRequest(
+        @Nullable CacheConfiguration ccfg,
+        String cacheName,
+        @Nullable NearCacheConfiguration nearCfg,
+        CacheType cacheType,
+        boolean failIfExists,
+        boolean failIfNotStarted
+    ) throws IgniteCheckedException {
+        DynamicCacheDescriptor desc = registeredCaches.get(maskNull(cacheName));
+
+        DynamicCacheChangeRequest req = new DynamicCacheChangeRequest(cacheName, ctx.localNodeId());
+
+        req.failIfExists(failIfExists);
+
+        if (ccfg != null) {
+            cloneCheckSerializable(ccfg);
+
+            if (desc != null) {
+                if (failIfExists) {
+                    throw new CacheExistsException("Failed to start cache " +
+                        "(a cache with the same name is already started): " + cacheName);
+                }
+                else {
+                    CacheConfiguration descCfg = desc.cacheConfiguration();
+
+                    // Check if we were asked to start a near cache.
+                    if (nearCfg != null) {
+                        if (CU.affinityNode(ctx.discovery().localNode(), descCfg.getNodeFilter())) {
+                            // If we are on a data node and near cache was enabled, return success, else - fail.
+                            if (descCfg.getNearConfiguration() != null)
+                                return null;
+                            else
+                                throw new IgniteCheckedException("Failed to start near " +
+                                    "cache (local node is an affinity node for cache): " + cacheName);
+                        }
+                        else
+                            // If local node has near cache, return success.
+                            req.clientStartOnly(true);
+                    }
+                    else
+                        req.clientStartOnly(true);
+
+                    req.deploymentId(desc.deploymentId());
+
+                    req.startCacheConfiguration(descCfg);
+                }
+            }
+            else {
+                req.deploymentId(IgniteUuid.randomUuid());
+
+                CacheConfiguration cfg = new CacheConfiguration(ccfg);
+
+                CacheObjectContext cacheObjCtx = ctx.cacheObjects().contextForCache(cfg);
+
+                initialize(false, cfg, cacheObjCtx);
+
+                req.startCacheConfiguration(cfg);
+            }
+        }
+        else {
+            req.clientStartOnly(true);
+
+            if (desc != null)
+                ccfg = desc.cacheConfiguration();
+
+            if (ccfg == null) {
+                if (failIfNotStarted) {
+                    throw new CacheExistsException("Failed to start client cache " +
+                        "(a cache with the given name is not started): " + cacheName);
+                }
+                else
+                    return null;
+            }
+
+            req.deploymentId(desc.deploymentId());
+            req.startCacheConfiguration(ccfg);
+        }
+
+        // Fail cache with swap enabled creation on grid without swap space SPI.
+        if (ccfg.isSwapEnabled())
+            for (ClusterNode n : ctx.discovery().allNodes())
+                if (!GridCacheUtils.clientNode(n) && !GridCacheUtils.isSwapEnabled(n)) {
+                    throw new IgniteCheckedException("Failed to start cache " +
+                        cacheName + " with swap enabled: Remote Node with ID " + n.id().toString().toUpperCase() +
+                        " has not swap SPI configured");
+                }
+
+        if (nearCfg != null)
+            req.nearCacheConfiguration(nearCfg);
+
+        req.cacheType(cacheType);
+
+        return req;
+    }
+
     /**
      * @param obj Object to clone.
      * @return Object copy.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
index c9cd750c72f7c..48e06ee7c82a0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
@@ -17,7 +17,9 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.CountDownLatch;
@@ -181,7 +183,8 @@ public void testStartStopCacheMultithreadedSameNode() throws Exception {
                 info("Succeeded: " + System.identityHashCode(fut));
 
                 succeeded++;
-            } catch (IgniteCheckedException e) {
+            }
+            catch (IgniteCheckedException e) {
                 info(e.getMessage());
 
                 failed++;
@@ -246,7 +249,8 @@ public void testStartCacheMultithreadedDifferentNodes() throws Exception {
                 info("Succeeded: " + System.identityHashCode(fut));
 
                 succeeded++;
-            } catch (IgniteCheckedException e) {
+            }
+            catch (IgniteCheckedException e) {
                 info(e.getMessage());
 
                 failed++;
@@ -288,6 +292,20 @@ public void testStartStopCacheSimpleAtomic() throws Exception {
         checkStartStopCacheSimple(CacheAtomicityMode.ATOMIC);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartStopCachesSimpleTransactional() throws Exception {
+        checkStartStopCachesSimple(CacheAtomicityMode.TRANSACTIONAL);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartStopCachesSimpleAtomic() throws Exception {
+        checkStartStopCachesSimple(CacheAtomicityMode.ATOMIC);
+    }
+
     /**
      * @param mode Cache atomicity mode.
      * @throws Exception If failed.
@@ -325,10 +343,10 @@ private void checkStartStopCacheSimple(CacheAtomicityMode mode) throws Exception
         for (int g = 0; g < nodeCount(); g++)
             caches[g] = grid(g).cache(DYNAMIC_CACHE_NAME);
 
-        kernal.context().cache().dynamicDestroyCache(DYNAMIC_CACHE_NAME, true).get();
+        kernal.destroyCache(DYNAMIC_CACHE_NAME);
 
         for (int g = 0; g < nodeCount(); g++) {
-            final IgniteKernal kernal0 = (IgniteKernal) grid(g);
+            final IgniteKernal kernal0 = (IgniteKernal)grid(g);
 
             final int idx = g;
 
@@ -345,6 +363,87 @@ private void checkStartStopCacheSimple(CacheAtomicityMode mode) throws Exception
         }
     }
 
+    /**
+     * @param mode Cache atomicity mode.
+     * @throws Exception If failed.
+     */
+    private void checkStartStopCachesSimple(CacheAtomicityMode mode) throws Exception {
+        final IgniteEx kernal = grid(0);
+        final int cacheCnt = 3;
+
+        List<CacheConfiguration> ccfgList = new ArrayList<>();
+
+        for (int i = 0; i < cacheCnt; i++) {
+            CacheConfiguration ccfg = new CacheConfiguration();
+            ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+            ccfg.setAtomicityMode(mode);
+            ccfg.setName(DYNAMIC_CACHE_NAME + Integer.toString(i));
+
+            ccfgList.add(ccfg);
+        }
+
+        kernal.createCaches(ccfgList);
+
+        for (int g = 0; g < nodeCount(); g++) {
+            IgniteEx kernal0 = grid(g);
+
+            for (IgniteInternalFuture f : kernal0.context().cache().context().exchange().exchangeFutures())
+                f.get();
+
+            info("Getting cache for node: " + g);
+
+            for (int i = 0; i < cacheCnt; i++)
+                assertNotNull(grid(g).cache(DYNAMIC_CACHE_NAME + Integer.toString(i)));
+        }
+
+        for (int i = 0; i < cacheCnt; i++)
+            grid(0).cache(DYNAMIC_CACHE_NAME + Integer.toString(i)).put(Integer.toString(i), Integer.toString(i));
+
+        for (int g = 0; g < nodeCount(); g++) {
+            for (int i = 0; i < cacheCnt; i++) {
+                assertEquals(
+                    Integer.toString(i),
+                    grid(g).cache(DYNAMIC_CACHE_NAME + Integer.toString(i)).get(Integer.toString(i))
+                );
+            }
+        }
+
+        // Grab caches before stop.
+        final IgniteCache[] caches = new IgniteCache[nodeCount() * cacheCnt];
+
+        for (int g = 0; g < nodeCount(); g++) {
+            for (int i = 0; i < cacheCnt; i++)
+                caches[g * nodeCount() + i] = grid(g).cache(DYNAMIC_CACHE_NAME + Integer.toString(i));
+        }
+
+        List<String> namesToDestroy = new ArrayList<>();
+
+        for (int i = 0; i < cacheCnt; i++)
+            namesToDestroy.add(DYNAMIC_CACHE_NAME + Integer.toString(i));
+
+        kernal.destroyCaches(namesToDestroy);
+
+        for (int g = 0; g < nodeCount(); g++) {
+            final IgniteKernal kernal0 = (IgniteKernal)grid(g);
+
+            for (int i = 0; i < cacheCnt; i++) {
+                final int idx = g * nodeCount() + i;
+                final int expVal = i;
+
+                for (IgniteInternalFuture f : kernal0.context().cache().context().exchange().exchangeFutures())
+                    f.get();
+
+                assertNull(kernal0.cache(DYNAMIC_CACHE_NAME));
+
+                GridTestUtils.assertThrows(log, new Callable<Object>() {
+                    @Override public Object call() throws Exception {
+                        return caches[idx].get(Integer.toString(expVal));
+                    }
+                }, IllegalStateException.class, null);
+            }
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -378,13 +477,13 @@ public void testStartStopCacheAddNode() throws Exception {
             }
 
             // Undeploy cache.
-            kernal.context().cache().dynamicDestroyCache(DYNAMIC_CACHE_NAME, true).get();
+            kernal.destroyCache(DYNAMIC_CACHE_NAME);
 
             startGrid(nodeCount() + 1);
 
             // Check that cache is not deployed on new node after undeploy.
             for (int g = 0; g < nodeCount() + 2; g++) {
-                final IgniteKernal kernal0 = (IgniteKernal) grid(g);
+                final IgniteKernal kernal0 = (IgniteKernal)grid(g);
 
                 for (IgniteInternalFuture f : kernal0.context().cache().context().exchange().exchangeFutures())
                     f.get();
@@ -431,16 +530,16 @@ public void testDeployFilter() throws Exception {
             for (int g = 0; g < nodeCount(); g++) {
                 for (int i = 0; i < 100; i++) {
                     assertFalse(grid(g).affinity(DYNAMIC_CACHE_NAME).mapKeyToPrimaryAndBackups(i)
-                            .contains(grid(nodeCount()).cluster().localNode()));
+                        .contains(grid(nodeCount()).cluster().localNode()));
 
                     assertFalse(grid(g).affinity(DYNAMIC_CACHE_NAME).mapKeyToPrimaryAndBackups(i)
-                            .contains(grid(nodeCount() + 1).cluster().localNode()));
+                        .contains(grid(nodeCount() + 1).cluster().localNode()));
                 }
             }
 
             // Check that cache is not deployed on new node after undeploy.
             for (int g = 0; g < nodeCount() + 2; g++) {
-                final IgniteKernal kernal0 = (IgniteKernal) grid(g);
+                final IgniteKernal kernal0 = (IgniteKernal)grid(g);
 
                 for (IgniteInternalFuture f : kernal0.context().cache().context().exchange().exchangeFutures())
                     f.get();
@@ -455,7 +554,7 @@ public void testDeployFilter() throws Exception {
                     }, IllegalArgumentException.class, null);
             }
 
-            kernal.context().cache().dynamicDestroyCache(DYNAMIC_CACHE_NAME, true).get();
+            kernal.destroyCache(DYNAMIC_CACHE_NAME);
 
             stopGrid(nodeCount() + 1);
             stopGrid(nodeCount());
@@ -486,6 +585,36 @@ public void testFailWhenConfiguredCacheExists() throws Exception {
         }, CacheExistsException.class, null);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testFailWhenOneOfConfiguredCacheExists() throws Exception {
+        GridTestUtils.assertThrowsInherited(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                final Ignite kernal = grid(0);
+
+                CacheConfiguration ccfgDynamic = new CacheConfiguration();
+                ccfgDynamic.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+                ccfgDynamic.setName(DYNAMIC_CACHE_NAME);
+
+                ccfgDynamic.setNodeFilter(NODE_FILTER);
+
+                CacheConfiguration ccfgStatic = new CacheConfiguration();
+                ccfgStatic.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+                // Cache is already configured, should fail.
+                ccfgStatic.setName(STATIC_CACHE_NAME);
+
+                ccfgStatic.setNodeFilter(NODE_FILTER);
+
+                return kernal.createCaches(F.asList(ccfgDynamic, ccfgStatic));
+            }
+        }, CacheExistsException.class, null);
+
+        assertNull(grid(0).cache(DYNAMIC_CACHE_NAME));
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -522,7 +651,7 @@ public void testClientCache() throws Exception {
             for (int g = 0; g < nodeCount() + 1; g++)
                 assertEquals("1", ignite(g).cache(DYNAMIC_CACHE_NAME).get("1"));
 
-            kernal.context().cache().dynamicDestroyCache(DYNAMIC_CACHE_NAME, true).get();
+            kernal.destroyCache(DYNAMIC_CACHE_NAME);
         }
         finally {
             stopGrid(nodeCount());
@@ -547,7 +676,7 @@ public void testStartFromClientNode() throws Exception {
 
             ccfg.setNodeFilter(NODE_FILTER);
 
-            final IgniteKernal started = (IgniteKernal) grid(nodeCount());
+            final IgniteKernal started = (IgniteKernal)grid(nodeCount());
 
             started.createCache(ccfg);
 
@@ -564,14 +693,13 @@ public void testStartFromClientNode() throws Exception {
             for (int g = 0; g < nodeCount() + 1; g++)
                 assertEquals("1", ignite(g).cache(DYNAMIC_CACHE_NAME).get("1"));
 
-            kernal.context().cache().dynamicDestroyCache(DYNAMIC_CACHE_NAME, true).get();
+            kernal.destroyCache(DYNAMIC_CACHE_NAME);
         }
         finally {
             stopGrid(nodeCount());
         }
     }
 
-
     /**
      * @throws Exception If failed.
      */
@@ -610,7 +738,7 @@ public void testStartNearCacheFromClientNode() throws Exception {
             for (int g = 0; g < nodeCount() + 1; g++)
                 assertEquals("1", ignite(g).cache(DYNAMIC_CACHE_NAME).get("1"));
 
-            kernal.context().cache().dynamicDestroyCache(DYNAMIC_CACHE_NAME, true).get();
+            kernal.destroyCache(DYNAMIC_CACHE_NAME);
         }
         finally {
             stopGrid(nodeCount());
@@ -760,7 +888,7 @@ public void testGetOrCreate() throws Exception {
                 nearGrid.getOrCreateNearCache(DYNAMIC_CACHE_NAME, new NearCacheConfiguration());
 
                 GridCacheContext<Object, Object> nCtx = ((IgniteKernal)nearGrid)
-                        .internalCache(DYNAMIC_CACHE_NAME).context();
+                    .internalCache(DYNAMIC_CACHE_NAME).context();
 
                 assertTrue(nCtx.isNear());
                 assertFalse(nCtx.affinityNode());
@@ -771,11 +899,12 @@ public void testGetOrCreate() throws Exception {
                 clientGrid.getOrCreateCache(cfg);
 
                 GridCacheContext<Object, Object> cCtx = ((IgniteKernal)clientGrid)
-                        .internalCache(DYNAMIC_CACHE_NAME).context();
+                    .internalCache(DYNAMIC_CACHE_NAME).context();
 
                 assertFalse(cCtx.isNear());
                 assertFalse(cCtx.affinityNode());
-            } finally {
+            }
+            finally {
                 stopGrid(nodeCount() + 1);
                 stopGrid(nodeCount());
             }
@@ -785,6 +914,40 @@ public void testGetOrCreate() throws Exception {
         }
     }
 
+    /** {@inheritDoc} */
+    public void testGetOrCreateCollection() throws Exception {
+        final int cacheCnt = 3;
+
+        try {
+            final Collection<CacheConfiguration> ccfgs = new ArrayList<>();
+
+            for (int i = 0; i < cacheCnt; i++) {
+                final CacheConfiguration cfg = new CacheConfiguration();
+
+                cfg.setName(DYNAMIC_CACHE_NAME + Integer.toString(i));
+                cfg.setNodeFilter(NODE_FILTER);
+
+                ccfgs.add(cfg);
+
+                grid(0).getOrCreateCaches(ccfgs);
+            }
+
+            for (int i = 0; i < cacheCnt; i++) {
+                assertNotNull(grid(0).cache(DYNAMIC_CACHE_NAME + Integer.toString(i)));
+
+                IgniteCache<Object, Object> jcache = grid(0).cache(DYNAMIC_CACHE_NAME + Integer.toString(i));
+
+                jcache.put(Integer.toString(i), Integer.toString(i));
+
+                assertEquals(jcache.get(Integer.toString(i)), Integer.toString(i));
+            }
+        }
+        finally {
+            for (int i = 0; i < cacheCnt; i++)
+                grid(0).destroyCache(DYNAMIC_CACHE_NAME + Integer.toString(i));
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -813,7 +976,7 @@ public void testGetOrCreateMultiNode() throws Exception {
             assertNull(err.get());
 
             for (int i = 0; i < nodeCount(); i++) {
-                GridCacheContext<Object, Object> ctx = ((IgniteKernal) ignite(i)).internalCache(DYNAMIC_CACHE_NAME)
+                GridCacheContext<Object, Object> ctx = ((IgniteKernal)ignite(i)).internalCache(DYNAMIC_CACHE_NAME)
                     .context();
 
                 assertTrue(ctx.affinityNode());
@@ -906,7 +1069,7 @@ public void checkGetOrCreateNear(final boolean nearOnly) throws Exception {
                 assertNull(err.get());
 
                 for (int i = 0; i < nodeCount(); i++) {
-                    GridCacheContext<Object, Object> ctx = ((IgniteKernal) ignite(i)).internalCache(DYNAMIC_CACHE_NAME)
+                    GridCacheContext<Object, Object> ctx = ((IgniteKernal)ignite(i)).internalCache(DYNAMIC_CACHE_NAME)
                         .context();
 
                     assertTrue(ctx.affinityNode());
@@ -914,7 +1077,7 @@ public void checkGetOrCreateNear(final boolean nearOnly) throws Exception {
                 }
 
                 for (int i = 0; i < clientCnt; i++) {
-                    GridCacheContext<Object, Object> ctx = ((IgniteKernal) ignite(nodeCount() + i))
+                    GridCacheContext<Object, Object> ctx = ((IgniteKernal)ignite(nodeCount() + i))
                         .internalCache(DYNAMIC_CACHE_NAME).context();
 
                     assertFalse(ctx.affinityNode());
@@ -995,12 +1158,12 @@ public void testServerNodesLeftEvent() throws Exception {
         for (int i = 0; i < nodeCount(); i++) {
             final int idx = i;
 
-                latches[i] = new CountDownLatch(1);
-                lsnrs[i] = new IgnitePredicate<CacheEvent>() {
-                    @Override public boolean apply(CacheEvent e) {
-                        switch (e.type()) {
-                            case EventType.EVT_CACHE_NODES_LEFT:
-                                latches[idx].countDown();
+            latches[i] = new CountDownLatch(1);
+            lsnrs[i] = new IgnitePredicate<CacheEvent>() {
+                @Override public boolean apply(CacheEvent e) {
+                    switch (e.type()) {
+                        case EventType.EVT_CACHE_NODES_LEFT:
+                            latches[idx].countDown();
 
                             break;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java
index c9f77cd3fa553..1b779c2801e02 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java
@@ -290,6 +290,13 @@ public IgfsIgniteMock(@Nullable String name, IgniteFileSystem igfs) {
         return null;
     }
 
+    /** {@inheritDoc} */
+    @Override public Collection<IgniteCache> createCaches(Collection<CacheConfiguration> cacheCfgs) {
+        throwUnsupported();
+
+        return null;
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> createCache(String cacheName) {
         throwUnsupported();
@@ -311,6 +318,13 @@ public IgfsIgniteMock(@Nullable String name, IgniteFileSystem igfs) {
         return null;
     }
 
+    /** {@inheritDoc} */
+    @Override public Collection<IgniteCache> getOrCreateCaches(Collection<CacheConfiguration> cacheCfgs) {
+        throwUnsupported();
+
+        return null;
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> void addCacheConfiguration(CacheConfiguration<K, V> cacheCfg) {
         throwUnsupported();
@@ -353,6 +367,11 @@ public IgfsIgniteMock(@Nullable String name, IgniteFileSystem igfs) {
         throwUnsupported();
     }
 
+    /** {@inheritDoc} */
+    @Override public void destroyCaches(Collection<String> cacheNames) {
+        throwUnsupported();
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> cache(@Nullable String name) {
         throwUnsupported();
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
index b559897705c5b..5722fa3d6056a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
@@ -229,6 +229,11 @@ public IgniteMock(
         return null;
     }
 
+    /** {@inheritDoc} */
+    @Override public Collection<IgniteCache> createCaches(Collection<CacheConfiguration> cacheCfgs) {
+        return null;
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> getOrCreateCache(CacheConfiguration<K, V> cacheCfg) {
         return null;
@@ -260,6 +265,11 @@ public IgniteMock(
         return null;
     }
 
+    /** {@inheritDoc} */
+    @Override public Collection<IgniteCache> getOrCreateCaches(Collection<CacheConfiguration> cacheCfgs) {
+        return null;
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> createCache(String cacheName) {
         return null;
@@ -275,6 +285,11 @@ public IgniteMock(
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void destroyCaches(Collection<String> cacheNames) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public IgniteTransactions transactions() {
         return null;
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
index 2598bc53afc65..21fc28dae7aa7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
@@ -425,6 +425,11 @@ public UUID getId() {
         throw new UnsupportedOperationException("Operation isn't supported yet.");
     }
 
+    /** {@inheritDoc} */
+    @Override public Collection<IgniteCache> createCaches(Collection<CacheConfiguration> cacheCfgs) {
+        throw new UnsupportedOperationException("Operation isn't supported yet.");
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> createCache(String cacheName) {
         throw new UnsupportedOperationException("Operation isn't supported yet.");
@@ -440,6 +445,11 @@ public UUID getId() {
         throw new UnsupportedOperationException("Operation isn't supported yet.");
     }
 
+    /** {@inheritDoc} */
+    @Override public Collection<IgniteCache> getOrCreateCaches(Collection<CacheConfiguration> cacheCfgs) {
+        throw new UnsupportedOperationException("Operation isn't supported yet.");
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> void addCacheConfiguration(CacheConfiguration<K, V> cacheCfg) {
         throw new UnsupportedOperationException("Operation isn't supported yet.");
@@ -476,6 +486,11 @@ public UUID getId() {
         throw new UnsupportedOperationException("Operation isn't supported yet.");
     }
 
+    /** {@inheritDoc} */
+    @Override public void destroyCaches(Collection<String> cacheNames) {
+        throw new UnsupportedOperationException("Operation isn't supported yet.");
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> cache(@Nullable final String name) {
         return new IgniteCacheProcessProxy<>(name, this);
diff --git a/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java b/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
index 7f79c0e90ad56..0a4bf202420e5 100644
--- a/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
+++ b/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
@@ -301,6 +301,13 @@ public ApplicationContext getApplicationContext() throws BeansException {
         return g.createCache(cacheCfg, nearCfg);
     }
 
+    /** {@inheritDoc} */
+    @Override public Collection<IgniteCache> createCaches(Collection<CacheConfiguration> cacheCfgs) {
+        checkIgnite();
+
+        return g.createCaches(cacheCfgs);
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> getOrCreateCache(CacheConfiguration<K, V> cacheCfg, NearCacheConfiguration<K, V> nearCfg) {
         checkIgnite();
@@ -329,6 +336,13 @@ public ApplicationContext getApplicationContext() throws BeansException {
         return g.getOrCreateCache(cacheName);
     }
 
+    /** {@inheritDoc} */
+    @Override public Collection<IgniteCache> getOrCreateCaches(Collection<CacheConfiguration> cacheCfgs) {
+        checkIgnite();
+
+        return g.getOrCreateCaches(cacheCfgs);
+    }
+
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> createCache(String cacheName) {
         checkIgnite();
@@ -350,6 +364,13 @@ public ApplicationContext getApplicationContext() throws BeansException {
         g.destroyCache(cacheName);
     }
 
+    /** {@inheritDoc} */
+    @Override public void destroyCaches(Collection<String> cacheNames) {
+        checkIgnite();
+
+        g.destroyCaches(cacheNames);
+    }
+
     /** {@inheritDoc} */
     @Override public IgniteTransactions transactions() {
         checkIgnite();

From 3e2ccfd30427ba0552eea8667c0129ae5ace9c0b Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Fri, 25 Nov 2016 14:26:54 +0300
Subject: [PATCH 006/446] IGNITE-4299: Fixes for examples.

---
 .../cpp/examples/putget-example/src/putget_example.cpp          | 2 +-
 .../platforms/cpp/examples/query-example/src/query_example.cpp  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/platforms/cpp/examples/putget-example/src/putget_example.cpp b/modules/platforms/cpp/examples/putget-example/src/putget_example.cpp
index 0b57886c462dd..8bf9c8c1feffb 100644
--- a/modules/platforms/cpp/examples/putget-example/src/putget_example.cpp
+++ b/modules/platforms/cpp/examples/putget-example/src/putget_example.cpp
@@ -117,7 +117,7 @@ int main()
     }
 
     std::cout << std::endl;
-    std::cout << ">>> Example finished, press any key to exit ..." << std::endl;
+    std::cout << ">>> Example finished, press 'Enter' to exit ..." << std::endl;
     std::cout << std::endl;
 
     std::cin.get();
diff --git a/modules/platforms/cpp/examples/query-example/src/query_example.cpp b/modules/platforms/cpp/examples/query-example/src/query_example.cpp
index 9bf3e52ee873b..8c2ca0c5569cc 100644
--- a/modules/platforms/cpp/examples/query-example/src/query_example.cpp
+++ b/modules/platforms/cpp/examples/query-example/src/query_example.cpp
@@ -450,7 +450,7 @@ int main()
     }
 
     std::cout << std::endl;
-    std::cout << ">>> Example finished, press any key to exit ..." << std::endl;
+    std::cout << ">>> Example finished, press 'Enter' to exit ..." << std::endl;
     std::cout << std::endl;
 
     std::cin.get();

From 6fbaef45af8f40062a95058df7ec0984c99035b9 Mon Sep 17 00:00:00 2001
From: Konstantin Dudkov <kdudkov@ya.ru>
Date: Fri, 25 Nov 2016 13:58:58 +0300
Subject: [PATCH 007/446] IGNITE-4305 marshalling fix in
 GridNearAtomicSingleUpdateInvokeRequest

---
 .../dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
index 42b51d6161ec3..238db8bc39035 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
@@ -202,8 +202,10 @@ public GridNearAtomicSingleUpdateInvokeRequest() {
         if (!addDepInfo && ctx.deploymentEnabled())
             addDepInfo = true;
 
-        if (entryProcessor != null && entryProcessorBytes == null)
+        if (entryProcessor != null && entryProcessorBytes == null) {
+            prepareObject(entryProcessor, cctx);
             entryProcessorBytes = CU.marshal(cctx, entryProcessor);
+        }
 
         if (invokeArgsBytes == null)
             invokeArgsBytes = marshalInvokeArguments(invokeArgs, cctx);

From 1a2de51f5807a91ce0d5dff28f24ed5bf7abebbc Mon Sep 17 00:00:00 2001
From: Konstantin Dudkov <kdudkov@ya.ru>
Date: Mon, 28 Nov 2016 12:59:02 +0300
Subject: [PATCH 008/446] IGNITE-4305 marshalling fix

---
 .../dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
index 238db8bc39035..df9e38441e79f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
@@ -203,7 +203,8 @@ public GridNearAtomicSingleUpdateInvokeRequest() {
             addDepInfo = true;
 
         if (entryProcessor != null && entryProcessorBytes == null) {
-            prepareObject(entryProcessor, cctx);
+            if (addDepInfo)
+                prepareObject(entryProcessor, cctx);
             entryProcessorBytes = CU.marshal(cctx, entryProcessor);
         }
 

From c06e4017771603df7118974758d3d6b9cadc41b5 Mon Sep 17 00:00:00 2001
From: Eduard Shangareev <eshangareev@gridgain.com>
Date: Wed, 30 Nov 2016 14:34:47 +0300
Subject: [PATCH 009/446] ignite-4332 Usage of cache.getEntry inside
 GridCacheQueryManager.runQuery causes to remote operations

---
 .../cache/query/GridCacheQueryManager.java     | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index ab8bd63ee75da..d4decb47ae489 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.internal.processors.cache.CacheMetricsImpl;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
@@ -1560,9 +1561,12 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                         metrics.addGetTimeNanos(System.nanoTime() - start);
                     }
 
+                    K key0 = null;
+                    V val0 = null;
+
                     if (readEvt) {
-                        K key0 = (K)cctx.unwrapBinaryIfNeeded(key, qry.keepBinary());
-                        V val0 = (V)cctx.unwrapBinaryIfNeeded(val, qry.keepBinary());
+                        key0 = (K)cctx.unwrapBinaryIfNeeded(key, qry.keepBinary());
+                        val0 = (V)cctx.unwrapBinaryIfNeeded(val, qry.keepBinary());
 
                         switch (type) {
                             case SQL:
@@ -1631,12 +1635,12 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                     }
 
                     if (rdc != null || trans != null) {
-                        Cache.Entry<K, V> entry;
+                        if (key0 == null)
+                            key0 = (K)cctx.unwrapBinaryIfNeeded(key, qry.keepBinary());
+                        if (val0 == null)
+                            val0 = (V)cctx.unwrapBinaryIfNeeded(val, qry.keepBinary());
 
-                        if (qry.keepBinary())
-                            entry = cache.<K, V>keepBinary().getEntry(key);
-                        else
-                            entry = cache.<K, V>getEntry(key);
+                        Cache.Entry<K, V> entry = new CacheEntryImpl(key0, val0);
 
                         // Reduce.
                         if (rdc != null) {

From 066691098797be8c01daa0e8dc2ba94d4ad73561 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 1 Dec 2016 17:16:28 +0300
Subject: [PATCH 010/446] ignite-4344 Do not create offheap map on client
 nodes.

---
 .../processors/cache/GridCacheProcessor.java  |  10 +-
 .../cache/OffheapCacheOnClientsTest.java      | 143 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite2.java     |   2 +
 3 files changed, 150 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheOnClientsTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 0e0d76941ae49..0be2072e3766d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1286,10 +1286,12 @@ private GridCacheContext createCache(CacheConfiguration<?, ?> cfg,
 
         U.startLifecycleAware(lifecycleAwares(cfg, cfgStore));
 
+        boolean affNode = CU.affinityNode(ctx.discovery().localNode(), cfg.getNodeFilter());
+
         GridCacheAffinityManager affMgr = new GridCacheAffinityManager();
         GridCacheEventManager evtMgr = new GridCacheEventManager();
-        GridCacheSwapManager swapMgr = new GridCacheSwapManager(cfg.getCacheMode() == LOCAL ||
-            !GridCacheUtils.isNearEnabled(cfg));
+        GridCacheSwapManager swapMgr = new GridCacheSwapManager(
+            affNode && (cfg.getCacheMode() == LOCAL || !GridCacheUtils.isNearEnabled(cfg)));
         GridCacheEvictionManager evictMgr = new GridCacheEvictionManager();
         GridCacheQueryManager qryMgr = queryManager(cfg);
         CacheContinuousQueryManager contQryMgr = new CacheContinuousQueryManager();
@@ -1302,8 +1304,6 @@ private GridCacheContext createCache(CacheConfiguration<?, ?> cfg,
 
         storeMgr.initialize(cfgStore, sesHolders);
 
-        boolean affNode = CU.affinityNode(ctx.discovery().localNode(), cfg.getNodeFilter());
-
         GridCacheContext<?, ?> cacheCtx = new GridCacheContext(
             ctx,
             sharedCtx,
@@ -1427,7 +1427,7 @@ private GridCacheContext createCache(CacheConfiguration<?, ?> cfg,
              * 7. GridCacheTtlManager.
              * ===============================================
              */
-            swapMgr = new GridCacheSwapManager(true);
+            swapMgr = new GridCacheSwapManager(affNode);
             evictMgr = new GridCacheEvictionManager();
             evtMgr = new GridCacheEventManager();
             pluginMgr = new CachePluginManager(ctx, cfg);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheOnClientsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheOnClientsTest.java
new file mode 100644
index 0000000000000..90985b60dce5b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheOnClientsTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.offheap.GridOffHeapProcessor;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheMemoryMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+
+/**
+ *
+ */
+public class OffheapCacheOnClientsTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final String CACHE_NAME = "CACHE_NAME";
+
+    /** */
+    private boolean client;
+
+    /** */
+    private boolean forceSrvMode;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid(0);
+
+        client = true;
+
+        startGrid(1);
+
+        forceSrvMode = true;
+
+        startGrid(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        if (client) {
+            cfg.setClientMode(true);
+
+            ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setForceServerMode(forceSrvMode);
+        }
+
+        return cfg;
+    }
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOffheapCacheOnClient() throws Exception {
+        try {
+            Ignite client = grid(1);
+
+            testStartCacheOnClient(client, OFFHEAP_TIERED);
+            testStartCacheOnClient(client, OFFHEAP_VALUES);
+            testStartCacheOnClient(client, ONHEAP_TIERED);
+
+            client = grid(2);
+
+            testStartCacheOnClient(client, OFFHEAP_TIERED);
+            testStartCacheOnClient(client, OFFHEAP_VALUES);
+            testStartCacheOnClient(client, ONHEAP_TIERED);
+        }
+        finally {
+            grid(0).destroyCache(CACHE_NAME);
+        }
+    }
+
+    /**
+     * @param client Node.
+     * @param memMode Memory mode.
+     * @throws Exception If failed.
+     */
+    private void testStartCacheOnClient(Ignite client, CacheMemoryMode memMode) throws Exception {
+        assertTrue(client.configuration().isClientMode());
+
+        try {
+            client.createCache(new CacheConfiguration(CACHE_NAME)
+                .setCacheMode(REPLICATED)
+                .setOffHeapMaxMemory(1024 * 1024)
+                .setMemoryMode(memMode));
+
+            IgniteCache<Integer, Integer> cache = client.cache(CACHE_NAME);
+
+            assertNotNull(cache);
+
+            cache.put(1, 1);
+            assertEquals((Integer)1, cache.get(1));
+
+            GridOffHeapProcessor offheap = ((IgniteKernal)client).cachex(CACHE_NAME).context().offheap();
+
+            assertNotNull(offheap);
+
+            ConcurrentMap offheapMaps = GridTestUtils.getFieldValue(offheap, "offheap");
+            assertNotNull(offheapMaps);
+
+            assertEquals(0,offheapMaps.size());
+        }
+        finally {
+            client.destroyCache(CACHE_NAME);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
index 6edfd094f7df0..f632f677ff0a0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheIncrementTxTest;
 import org.apache.ignite.internal.processors.cache.IgniteCachePartitionMapUpdateTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheAndNodeStop;
+import org.apache.ignite.internal.processors.cache.OffheapCacheOnClientsTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLoadingConcurrentGridStartSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLoadingConcurrentGridStartSelfTestAllowOverwrite;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLockReleaseNodeLeaveTest;
@@ -266,6 +267,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(IgniteNoCustomEventsOnNodeStart.class));
 
         suite.addTest(new TestSuite(CacheExchangeMessageDuplicatedStateTest.class));
+        suite.addTest(new TestSuite(OffheapCacheOnClientsTest.class));
 
         return suite;
     }

From e9ace7730773a6d4a1d30b271854f1fe8a7ba632 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 2 Dec 2016 16:06:41 +0700
Subject: [PATCH 011/446] Improved exception handling.

---
 .../org/apache/ignite/marshaller/jdk/JdkMarshaller.java   | 4 ++--
 .../ignite/marshaller/optimized/OptimizedMarshaller.java  | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshaller.java b/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshaller.java
index 54172dcc5b052..06b7109fdcd22 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshaller.java
@@ -121,8 +121,8 @@ public class JdkMarshaller extends AbstractNodeNameAwareMarshaller {
         }
         catch (ClassNotFoundException e) {
             throw new IgniteCheckedException("Failed to find class with given class loader for unmarshalling " +
-                "(make sure same versions of all classes are available on all nodes or enable peer-class-loading): " +
-                clsLdr, e);
+                "(make sure same versions of all classes are available on all nodes or enable peer-class-loading) " +
+                "[clsLdr=" + clsLdr + ", cls=" + e.getMessage() + "]", e);
         }
         catch (Exception e) {
             throw new IgniteCheckedException("Failed to deserialize object with given class loader: " + clsLdr, e);
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshaller.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshaller.java
index 467dddf1abd39..6d57864245659 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshaller.java
@@ -219,8 +219,8 @@ public void setPoolSize(int poolSize) {
         }
         catch (ClassNotFoundException e) {
             throw new IgniteCheckedException("Failed to find class with given class loader for unmarshalling " +
-                "(make sure same versions of all classes are available on all nodes or enable peer-class-loading): " +
-                clsLdr, e);
+                "(make sure same versions of all classes are available on all nodes or enable peer-class-loading) " +
+                "[clsLdr=" + clsLdr + ", cls=" + e.getMessage() + "]", e);
         }
         catch (Exception e) {
             throw new IgniteCheckedException("Failed to deserialize object with given class loader: " + clsLdr, e);
@@ -248,8 +248,8 @@ public void setPoolSize(int poolSize) {
         }
         catch (ClassNotFoundException e) {
             throw new IgniteCheckedException("Failed to find class with given class loader for unmarshalling " +
-                "(make sure same version of all classes are available on all nodes or enable peer-class-loading): " +
-                clsLdr, e);
+                "(make sure same version of all classes are available on all nodes or enable peer-class-loading)" +
+                " [clsLdr=" + clsLdr + ", cls=" + e.getMessage() + "]", e);
         }
         catch (Exception e) {
             throw new IgniteCheckedException("Failed to deserialize object with given class loader: " + clsLdr, e);

From 12bdd6a031a33eda004a66e73cee9628f073ed68 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 2 Dec 2016 16:09:29 +0700
Subject: [PATCH 012/446] Updated classnames.properties for running nodes in
 IDE.

---
 .../resources/META-INF/classnames.properties  |  86 +++++++++----
 .../resources/META-INF/classnames.properties  | 114 ++++++++++++++++++
 2 files changed, 176 insertions(+), 24 deletions(-)
 create mode 100644 modules/hadoop/src/main/resources/META-INF/classnames.properties

diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 71d236fdb8ce4..4c9596c854ea0 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -75,11 +75,13 @@ org.apache.ignite.cache.eviction.sorted.SortedEvictionPolicy$HolderComparator
 org.apache.ignite.cache.query.CacheQueryEntryEvent
 org.apache.ignite.cache.query.ContinuousQuery
 org.apache.ignite.cache.query.Query
+org.apache.ignite.cache.query.QueryCancelledException
 org.apache.ignite.cache.query.ScanQuery
 org.apache.ignite.cache.query.SpiQuery
 org.apache.ignite.cache.query.SqlFieldsQuery
 org.apache.ignite.cache.query.SqlQuery
 org.apache.ignite.cache.query.TextQuery
+org.apache.ignite.cache.store.jdbc.CacheAbstractJdbcStore$2
 org.apache.ignite.cache.store.jdbc.CacheAbstractJdbcStore$EntryMapping$1
 org.apache.ignite.cache.store.jdbc.CacheAbstractJdbcStore$EntryMapping$2
 org.apache.ignite.cache.store.jdbc.CacheAbstractJdbcStore$TypeKind
@@ -366,6 +368,7 @@ org.apache.ignite.internal.processors.cache.CacheWeakQueryIteratorsHolder$WeakQu
 org.apache.ignite.internal.processors.cache.CacheWeakQueryIteratorsHolder$WeakReferenceCloseableIterator
 org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch
 org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest
+org.apache.ignite.internal.processors.cache.EntryProcessorResourceInjectorProxy
 org.apache.ignite.internal.processors.cache.GridCacheAdapter
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$10
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$11
@@ -520,10 +523,8 @@ org.apache.ignite.internal.processors.cache.GridCacheUtils$15
 org.apache.ignite.internal.processors.cache.GridCacheUtils$16
 org.apache.ignite.internal.processors.cache.GridCacheUtils$17
 org.apache.ignite.internal.processors.cache.GridCacheUtils$18
-org.apache.ignite.internal.processors.cache.GridCacheUtils$19
 org.apache.ignite.internal.processors.cache.GridCacheUtils$2
 org.apache.ignite.internal.processors.cache.GridCacheUtils$20
-org.apache.ignite.internal.processors.cache.GridCacheUtils$22
 org.apache.ignite.internal.processors.cache.GridCacheUtils$3
 org.apache.ignite.internal.processors.cache.GridCacheUtils$4
 org.apache.ignite.internal.processors.cache.GridCacheUtils$5
@@ -533,6 +534,7 @@ org.apache.ignite.internal.processors.cache.GridCacheUtils$8
 org.apache.ignite.internal.processors.cache.GridCacheUtils$9
 org.apache.ignite.internal.processors.cache.GridCacheValueCollection
 org.apache.ignite.internal.processors.cache.GridCacheValueCollection$1
+org.apache.ignite.internal.processors.cache.GridDeferredAckMessageSender$DeferredAckMessageBuffer
 org.apache.ignite.internal.processors.cache.IgniteCacheProxy
 org.apache.ignite.internal.processors.cache.IgniteCacheProxy$1
 org.apache.ignite.internal.processors.cache.IgniteCacheProxy$10
@@ -545,12 +547,13 @@ org.apache.ignite.internal.processors.cache.IgniteCacheProxy$8
 org.apache.ignite.internal.processors.cache.IgniteCacheProxy$9
 org.apache.ignite.internal.processors.cache.KeyCacheObject
 org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl
+org.apache.ignite.internal.processors.cache.QueryCursorImpl$State
 org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityProxy
 org.apache.ignite.internal.processors.cache.binary.BinaryMetadataKey
 org.apache.ignite.internal.processors.cache.binary.CacheDefaultBinaryAffinityKeyMapper
 org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$1
-org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$4
 org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$5
+org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$6
 org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$MetaDataEntryFilter
 org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$MetaDataPredicate
 org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$MetadataProcessor
@@ -630,6 +633,7 @@ org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal$2
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter$1
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter$2
+org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxOnePhaseCommitAckRequest
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture$1
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture$2
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture$3
@@ -648,6 +652,7 @@ org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedGetFu
 org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedSingleGetFuture$1
 org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedSingleGetFuture$2
 org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedSingleGetFuture$3
+org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateRequest
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$10
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$11
@@ -670,26 +675,30 @@ org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomic
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$27
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$28
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$29
-org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$3
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$30
+org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$31
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$4
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$5
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$6
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$8
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$9
-org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredResponseBuffer
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse
+org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicSingleUpdateRequest
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicUpdateRequest
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicUpdateResponse
+org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractSingleUpdateRequest
+org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateFuture$1
+org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest
+org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicFullUpdateRequest
+org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFilterRequest
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFuture$1
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFuture$2
-org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFuture$3
+org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateInvokeRequest
+org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateRequest
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateFuture$1
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateFuture$2
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateFuture$3
-org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateFuture$4
-org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicFullUpdateRequest
 org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateResponse
 org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache
 org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache$2
@@ -760,6 +769,7 @@ org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$2
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$3
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$4
+org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$LockTimeoutObject$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$MiniFuture$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse
@@ -772,6 +782,7 @@ org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticS
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$2
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$3
+org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$4
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$MiniFuture$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFutureAdapter$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearPessimisticTxPrepareFuture$1
@@ -821,20 +832,21 @@ org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManag
 org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager$7
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryAdapter$1
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryAdapter$ScanQueryFallbackClosableIterator
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryDetailMetricsAdapter
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryFutureAdapter$1
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryFutureAdapter$2
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$1$1
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$1$2
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$11
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$12
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$13
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$14
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$15$1
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$2
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$3
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$4
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$15
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$16
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$17
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$18$1
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$4$1
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$4$2
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$5
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$6
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$7
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$8
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$9
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$CacheSqlIndexMetadata
@@ -896,6 +908,8 @@ org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$12
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$13
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$14
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$15
+org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$16
+org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$17
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$2
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$4
@@ -929,9 +943,9 @@ org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter$Po
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter$PostLockClosure1$4
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter$PostLockClosure2
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter$PostMissClosure
-org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager$2
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager$3
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager$4
+org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager$5
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager$CommitListener
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager$CommittedVersion
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager$NodeFailureTimeoutObject$1
@@ -998,6 +1012,7 @@ org.apache.ignite.internal.processors.continuous.StopRoutineAckDiscoveryMessage
 org.apache.ignite.internal.processors.continuous.StopRoutineDiscoveryMessage
 org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor$3
 org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor$4
+org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor$5
 org.apache.ignite.internal.processors.datastreamer.DataStreamerCacheUpdaters$Batched
 org.apache.ignite.internal.processors.datastreamer.DataStreamerCacheUpdaters$BatchedSorted
 org.apache.ignite.internal.processors.datastreamer.DataStreamerCacheUpdaters$Individual
@@ -1005,9 +1020,9 @@ org.apache.ignite.internal.processors.datastreamer.DataStreamerEntry
 org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$1
 org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$4
 org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$5
-org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$6
 org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$Buffer$1
 org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$Buffer$2
+org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$Buffer$3
 org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$DataStreamerPda
 org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$DefaultIoPolicyResolver
 org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$IsolatedUpdater
@@ -1084,6 +1099,7 @@ org.apache.ignite.internal.processors.datastructures.GridCacheSetProxy
 org.apache.ignite.internal.processors.datastructures.GridSetQueryPredicate
 org.apache.ignite.internal.processors.dr.GridDrType
 org.apache.ignite.internal.processors.dr.IgniteDrDataStreamerCacheUpdater
+org.apache.ignite.internal.processors.hadoop.HadoopDefaultJobInfo
 org.apache.ignite.internal.processors.hadoop.HadoopFileBlock
 org.apache.ignite.internal.processors.hadoop.HadoopInputSplit
 org.apache.ignite.internal.processors.hadoop.HadoopJobId
@@ -1119,7 +1135,6 @@ org.apache.ignite.internal.processors.igfs.IgfsFragmentizerManager$IdentityHashS
 org.apache.ignite.internal.processors.igfs.IgfsFragmentizerRequest
 org.apache.ignite.internal.processors.igfs.IgfsFragmentizerResponse
 org.apache.ignite.internal.processors.igfs.IgfsHandshakeResponse
-org.apache.ignite.internal.processors.igfs.IgfsImpl$12$1
 org.apache.ignite.internal.processors.igfs.IgfsImpl$IgfsGlobalSpaceTask
 org.apache.ignite.internal.processors.igfs.IgfsImpl$IgfsGlobalSpaceTask$1
 org.apache.ignite.internal.processors.igfs.IgfsInputStreamDescriptor
@@ -1155,6 +1170,7 @@ org.apache.ignite.internal.processors.igfs.client.IgfsClientSummaryCallable
 org.apache.ignite.internal.processors.igfs.client.IgfsClientUpdateCallable
 org.apache.ignite.internal.processors.igfs.client.meta.IgfsClientMetaIdsForPathCallable
 org.apache.ignite.internal.processors.igfs.client.meta.IgfsClientMetaInfoForPathCallable
+org.apache.ignite.internal.processors.igfs.client.meta.IgfsClientMetaUnlockCallable
 org.apache.ignite.internal.processors.igfs.data.IgfsDataPutProcessor
 org.apache.ignite.internal.processors.igfs.meta.IgfsMetaDirectoryCreateProcessor
 org.apache.ignite.internal.processors.igfs.meta.IgfsMetaDirectoryListingAddProcessor
@@ -1172,6 +1188,8 @@ org.apache.ignite.internal.processors.igfs.meta.IgfsMetaUpdateTimesProcessor
 org.apache.ignite.internal.processors.job.GridJobProcessor$5
 org.apache.ignite.internal.processors.job.GridJobWorker$3
 org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor$SnapshotReducer
+org.apache.ignite.internal.processors.odbc.OdbcProtocolVersion
+org.apache.ignite.internal.processors.odbc.escape.OdbcEscapeType
 org.apache.ignite.internal.processors.platform.PlatformAbstractConfigurationClosure
 org.apache.ignite.internal.processors.platform.PlatformAbstractPredicate
 org.apache.ignite.internal.processors.platform.PlatformEventFilterListener
@@ -1180,7 +1198,8 @@ org.apache.ignite.internal.processors.platform.PlatformExtendedException
 org.apache.ignite.internal.processors.platform.PlatformJavaObjectFactoryProxy
 org.apache.ignite.internal.processors.platform.PlatformNativeException
 org.apache.ignite.internal.processors.platform.PlatformNoCallbackException
-org.apache.ignite.internal.processors.platform.cache.PlatformCache$1
+org.apache.ignite.internal.processors.platform.PlatformProcessorImpl$1
+org.apache.ignite.internal.processors.platform.cache.PlatformCache$5
 org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryFilter
 org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryFilterImpl
 org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryProcessor
@@ -1211,6 +1230,8 @@ org.apache.ignite.internal.processors.platform.cpp.PlatformCppConfigurationClosu
 org.apache.ignite.internal.processors.platform.datastreamer.PlatformStreamReceiver
 org.apache.ignite.internal.processors.platform.datastreamer.PlatformStreamReceiverImpl
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$1
+org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$10
+org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$11
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$2
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$3
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$4
@@ -1218,6 +1239,7 @@ org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$5
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$6
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$7
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$8
+org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$9
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetConfigurationClosure
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetService
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetServiceImpl
@@ -1232,19 +1254,24 @@ org.apache.ignite.internal.processors.platform.transactions.PlatformTransactions
 org.apache.ignite.internal.processors.platform.utils.PlatformFutureUtils$1
 org.apache.ignite.internal.processors.platform.utils.PlatformFutureUtils$FutureListenable$1
 org.apache.ignite.internal.processors.platform.utils.PlatformFutureUtils$InternalFutureListenable$1
+org.apache.ignite.internal.processors.platform.websession.PlatformDotNetSessionLockProcessor
+org.apache.ignite.internal.processors.platform.websession.PlatformDotNetSessionSetAndUnlockProcessor
 org.apache.ignite.internal.processors.query.GridQueryFieldMetadata
 org.apache.ignite.internal.processors.query.GridQueryIndexType
-org.apache.ignite.internal.processors.query.GridQueryProcessor$2
 org.apache.ignite.internal.processors.query.GridQueryProcessor$3
 org.apache.ignite.internal.processors.query.GridQueryProcessor$4
 org.apache.ignite.internal.processors.query.GridQueryProcessor$5
 org.apache.ignite.internal.processors.query.GridQueryProcessor$6
+org.apache.ignite.internal.processors.query.GridQueryProcessor$7
+org.apache.ignite.internal.processors.query.GridQueryProcessor$8
 org.apache.ignite.internal.processors.query.GridQueryProcessor$IndexType
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageRequest
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageResponse
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryRequest
+org.apache.ignite.internal.processors.resource.GridResourceIoc$AnnotationSet
+org.apache.ignite.internal.processors.resource.GridResourceIoc$ResourceAnnotation
 org.apache.ignite.internal.processors.rest.GridRestCommand
 org.apache.ignite.internal.processors.rest.GridRestProcessor$2$1
 org.apache.ignite.internal.processors.rest.GridRestProcessor$3
@@ -1332,7 +1359,7 @@ org.apache.ignite.internal.processors.service.ServiceDescriptorImpl
 org.apache.ignite.internal.processors.task.GridTaskProcessor$1
 org.apache.ignite.internal.processors.task.GridTaskThreadContextKey
 org.apache.ignite.internal.processors.task.GridTaskWorker$3
-org.apache.ignite.internal.processors.task.GridTaskWorker$4
+org.apache.ignite.internal.processors.task.GridTaskWorker$5
 org.apache.ignite.internal.processors.task.GridTaskWorker$State
 org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException
 org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException
@@ -1356,6 +1383,7 @@ org.apache.ignite.internal.util.GridBoundedConcurrentOrderedMap
 org.apache.ignite.internal.util.GridBoundedConcurrentOrderedSet
 org.apache.ignite.internal.util.GridBoundedLinkedHashMap
 org.apache.ignite.internal.util.GridBoundedLinkedHashSet
+org.apache.ignite.internal.util.GridBoundedPriorityQueue
 org.apache.ignite.internal.util.GridByteArrayList
 org.apache.ignite.internal.util.GridCloseableIteratorAdapter
 org.apache.ignite.internal.util.GridCloseableIteratorAdapterEx
@@ -1407,14 +1435,14 @@ org.apache.ignite.internal.util.IgniteUtils$11
 org.apache.ignite.internal.util.IgniteUtils$12
 org.apache.ignite.internal.util.IgniteUtils$13
 org.apache.ignite.internal.util.IgniteUtils$14
-org.apache.ignite.internal.util.IgniteUtils$16
-org.apache.ignite.internal.util.IgniteUtils$2
-org.apache.ignite.internal.util.IgniteUtils$22
+org.apache.ignite.internal.util.IgniteUtils$15
+org.apache.ignite.internal.util.IgniteUtils$17
 org.apache.ignite.internal.util.IgniteUtils$23
 org.apache.ignite.internal.util.IgniteUtils$24
 org.apache.ignite.internal.util.IgniteUtils$25
 org.apache.ignite.internal.util.IgniteUtils$26
 org.apache.ignite.internal.util.IgniteUtils$27
+org.apache.ignite.internal.util.IgniteUtils$28
 org.apache.ignite.internal.util.IgniteUtils$3
 org.apache.ignite.internal.util.IgniteUtils$4
 org.apache.ignite.internal.util.IgniteUtils$5
@@ -1647,12 +1675,19 @@ org.apache.ignite.internal.visor.cache.VisorCachePartitionsTask
 org.apache.ignite.internal.visor.cache.VisorCachePartitionsTask$VisorCachePartitionsJob
 org.apache.ignite.internal.visor.cache.VisorCacheQueryConfiguration
 org.apache.ignite.internal.visor.cache.VisorCacheQueryConfigurationV2
+org.apache.ignite.internal.visor.cache.VisorCacheQueryDetailMetrics
+org.apache.ignite.internal.visor.cache.VisorCacheQueryDetailMetricsCollectorTask
+org.apache.ignite.internal.visor.cache.VisorCacheQueryDetailMetricsCollectorTask$VisorCacheQueryDetailMetricsCollectorJob
 org.apache.ignite.internal.visor.cache.VisorCacheQueryMetrics
 org.apache.ignite.internal.visor.cache.VisorCacheRebalanceConfiguration
 org.apache.ignite.internal.visor.cache.VisorCacheRebalanceTask
 org.apache.ignite.internal.visor.cache.VisorCacheRebalanceTask$VisorCachesRebalanceJob
 org.apache.ignite.internal.visor.cache.VisorCacheResetMetricsTask
 org.apache.ignite.internal.visor.cache.VisorCacheResetMetricsTask$VisorCacheResetMetricsJob
+org.apache.ignite.internal.visor.cache.VisorCacheResetQueryDetailMetricsTask
+org.apache.ignite.internal.visor.cache.VisorCacheResetQueryDetailMetricsTask$VisorCacheResetQueryDetailMetricsJob
+org.apache.ignite.internal.visor.cache.VisorCacheResetQueryMetricsTask
+org.apache.ignite.internal.visor.cache.VisorCacheResetQueryMetricsTask$VisorCacheResetQueryMetricsJob
 org.apache.ignite.internal.visor.cache.VisorCacheStartTask
 org.apache.ignite.internal.visor.cache.VisorCacheStartTask$VisorCacheStartArg
 org.apache.ignite.internal.visor.cache.VisorCacheStartTask$VisorCacheStartJob
@@ -1682,6 +1717,7 @@ org.apache.ignite.internal.visor.debug.VisorThreadLockInfo
 org.apache.ignite.internal.visor.debug.VisorThreadMonitorInfo
 org.apache.ignite.internal.visor.event.VisorGridDeploymentEvent
 org.apache.ignite.internal.visor.event.VisorGridDiscoveryEvent
+org.apache.ignite.internal.visor.event.VisorGridDiscoveryEventV2
 org.apache.ignite.internal.visor.event.VisorGridEvent
 org.apache.ignite.internal.visor.event.VisorGridEventsLost
 org.apache.ignite.internal.visor.event.VisorGridJobEvent
@@ -1769,6 +1805,7 @@ org.apache.ignite.internal.visor.util.VisorClusterGroupEmptyException
 org.apache.ignite.internal.visor.util.VisorEventMapper
 org.apache.ignite.internal.visor.util.VisorExceptionWrapper
 org.apache.ignite.internal.visor.util.VisorTaskUtils$4
+org.apache.ignite.internal.visor.util.VisorTaskUtils$5
 org.apache.ignite.internal.websession.WebSessionAttributeProcessor
 org.apache.ignite.internal.websession.WebSessionEntity
 org.apache.ignite.lang.IgniteBiClosure
@@ -1798,6 +1835,7 @@ org.apache.ignite.plugin.PluginNotFoundException
 org.apache.ignite.plugin.PluginValidationException
 org.apache.ignite.plugin.extensions.communication.Message
 org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType
+org.apache.ignite.plugin.security.SecurityBasicPermissionSet
 org.apache.ignite.plugin.security.SecurityCredentials
 org.apache.ignite.plugin.security.SecurityException
 org.apache.ignite.plugin.security.SecurityPermission
diff --git a/modules/hadoop/src/main/resources/META-INF/classnames.properties b/modules/hadoop/src/main/resources/META-INF/classnames.properties
new file mode 100644
index 0000000000000..0ac17cf6058dd
--- /dev/null
+++ b/modules/hadoop/src/main/resources/META-INF/classnames.properties
@@ -0,0 +1,114 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.ignite.hadoop.fs.BasicHadoopFileSystemFactory
+org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory
+org.apache.ignite.hadoop.fs.HadoopFileSystemFactory
+org.apache.ignite.hadoop.fs.KerberosHadoopFileSystemFactory
+org.apache.ignite.hadoop.mapreduce.IgniteHadoopWeightedMapReducePlanner$MapperPriority
+org.apache.ignite.hadoop.util.BasicUserNameMapper
+org.apache.ignite.hadoop.util.ChainedUserNameMapper
+org.apache.ignite.hadoop.util.KerberosUserNameMapper
+org.apache.ignite.hadoop.util.KerberosUserNameMapper$State
+org.apache.ignite.hadoop.util.UserNameMapper
+org.apache.ignite.internal.processors.hadoop.HadoopAttributes
+org.apache.ignite.internal.processors.hadoop.HadoopDefaultJobInfo
+org.apache.ignite.internal.processors.hadoop.HadoopExternalSplit
+org.apache.ignite.internal.processors.hadoop.HadoopFileBlock
+org.apache.ignite.internal.processors.hadoop.HadoopInputSplit
+org.apache.ignite.internal.processors.hadoop.HadoopJobId
+org.apache.ignite.internal.processors.hadoop.HadoopJobInfo
+org.apache.ignite.internal.processors.hadoop.HadoopJobPhase
+org.apache.ignite.internal.processors.hadoop.HadoopJobProperty
+org.apache.ignite.internal.processors.hadoop.HadoopJobStatus
+org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan
+org.apache.ignite.internal.processors.hadoop.HadoopSplitWrapper
+org.apache.ignite.internal.processors.hadoop.HadoopTaskCancelledException
+org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo
+org.apache.ignite.internal.processors.hadoop.HadoopTaskType
+org.apache.ignite.internal.processors.hadoop.counter.HadoopCounterAdapter
+org.apache.ignite.internal.processors.hadoop.counter.HadoopCountersImpl
+org.apache.ignite.internal.processors.hadoop.counter.HadoopCountersImpl$CounterKey
+org.apache.ignite.internal.processors.hadoop.counter.HadoopLongCounter
+org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsCommunicationException
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$1
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$10
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$11
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$12
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$13
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$14
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$15
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$16
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$2
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$3
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$4
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$5
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$6
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$8
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsInProc$9
+org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsOutProc$1
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobMetadata
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker$1
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker$CancelJobProcessor
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker$IncrementCountersProcessor
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker$InitializeReducersProcessor
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker$JobLocalState$1
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker$JobLocalState$2
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker$RemoveMappersProcessor
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker$RemoveReducerProcessor
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker$StackedProcessor
+org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker$UpdatePhaseProcessor
+org.apache.ignite.internal.processors.hadoop.message.HadoopMessage
+org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan
+org.apache.ignite.internal.processors.hadoop.proto.HadoopProtocolJobCountersTask
+org.apache.ignite.internal.processors.hadoop.proto.HadoopProtocolJobStatusTask
+org.apache.ignite.internal.processors.hadoop.proto.HadoopProtocolJobStatusTask$1
+org.apache.ignite.internal.processors.hadoop.proto.HadoopProtocolKillJobTask
+org.apache.ignite.internal.processors.hadoop.proto.HadoopProtocolNextTaskIdTask
+org.apache.ignite.internal.processors.hadoop.proto.HadoopProtocolSubmitJobTask
+org.apache.ignite.internal.processors.hadoop.proto.HadoopProtocolTaskAdapter
+org.apache.ignite.internal.processors.hadoop.proto.HadoopProtocolTaskAdapter$Job
+org.apache.ignite.internal.processors.hadoop.proto.HadoopProtocolTaskArguments
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffle$1
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffle$2
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleAck
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleJob$4
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleMessage
+org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopConcurrentHashMultimap$State
+org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopTaskState
+org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopTaskStatus
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopExternalTaskExecutor$1
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopExternalTaskExecutor$2
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopExternalTaskExecutor$4
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopExternalTaskExecutor$HadoopProcess
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopExternalTaskExecutor$HadoopProcess$1
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopJobInfoUpdateRequest
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopPrepareForJobRequest
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopProcessDescriptor
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopProcessStartedAck
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopTaskExecutionRequest
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.HadoopTaskFinishedMessage
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child.HadoopChildProcessRunner$1
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child.HadoopChildProcessRunner$2
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child.HadoopChildProcessRunner$2$1
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child.HadoopChildProcessRunner$3
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child.HadoopChildProcessRunner$MessageListener$1
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.child.HadoopExternalProcessStarter$1
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.HadoopExternalCommunication$HandshakeAndBackpressureFilter$1
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.HadoopExternalCommunication$ProcessHandshakeMessage
+org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.HadoopHandshakeTimeoutException

From 33dda46061aae73e5c138851cfdd5f49a1c254cb Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 2 Dec 2016 12:13:38 +0300
Subject: [PATCH 013/446] ignite-4285 For serializable txs allow multiple
 threads to get read lock for the same key

---
 .../processors/cache/CacheLockCandidates.java |  42 ++
 .../cache/CacheLockCandidatesList.java        |  71 ++
 .../processors/cache/GridCacheEntryEx.java    |   3 +-
 .../processors/cache/GridCacheMapEntry.java   | 117 +++-
 .../processors/cache/GridCacheMvcc.java       | 376 +++++++----
 .../cache/GridCacheMvccCallback.java          |   4 +-
 .../cache/GridCacheMvccCandidate.java         |  80 +--
 .../cache/GridCacheMvccManager.java           |  19 +-
 .../GridDistributedCacheEntry.java            | 303 +++------
 .../distributed/dht/GridDhtCacheEntry.java    |  32 +-
 .../distributed/dht/GridDhtLockFuture.java    |  34 +-
 .../dht/GridDhtTransactionalCacheAdapter.java |   1 -
 .../dht/GridDhtTxPrepareFuture.java           |   5 +-
 .../colocated/GridDhtColocatedLockFuture.java |   8 +-
 .../distributed/near/GridNearCacheEntry.java  |  44 +-
 .../distributed/near/GridNearLockFuture.java  |   3 +-
 .../near/GridNearTransactionalCache.java      |   5 +-
 .../cache/local/GridLocalCacheEntry.java      | 173 ++---
 .../cache/local/GridLocalLockFuture.java      |   2 +-
 .../cache/transactions/IgniteTxManager.java   |   6 +-
 .../CacheSerializableTransactionsTest.java    | 604 +++++++++++++++++-
 .../cache/GridCacheMvccFlagsTest.java         |   8 +-
 .../GridCacheMvccPartitionedSelfTest.java     | 334 ++++++++--
 .../cache/GridCacheMvccSelfTest.java          | 212 +++---
 .../cache/GridCacheTestEntryEx.java           |  77 +--
 .../hashmap/GridHashMapLoadTest.java          |   7 +-
 26 files changed, 1721 insertions(+), 849 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLockCandidates.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLockCandidatesList.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLockCandidates.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLockCandidates.java
new file mode 100644
index 0000000000000..9cf16f4eac55c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLockCandidates.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+
+/**
+ *
+ */
+public interface CacheLockCandidates {
+    /**
+     * @param idx Candidate index.
+     * @return Candidate.
+     */
+    public GridCacheMvccCandidate candidate(int idx);
+
+    /**
+     * @return Number of candidates.
+     */
+    public int size();
+
+    /**
+     * @param ver Candidate version.
+     * @return {@code True} if contains candidate with given version.
+     */
+    public boolean hasCandidate(GridCacheVersion ver);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLockCandidatesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLockCandidatesList.java
new file mode 100644
index 0000000000000..e026bceb97e58
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLockCandidatesList.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+class CacheLockCandidatesList implements CacheLockCandidates {
+    /** */
+    @GridToStringInclude
+    private List<GridCacheMvccCandidate> list = new ArrayList<>();
+
+    /**
+     * @param cand Candidate to add.
+     */
+    void add(GridCacheMvccCandidate cand) {
+        assert !hasCandidate(cand.version()) : cand;
+
+        list.add(cand);
+    }
+
+    /** {@inheritDoc} */
+    @Override public GridCacheMvccCandidate candidate(int idx) {
+        assert idx < list.size() : idx;
+
+        return list.get(idx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size() {
+        return list.size();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean hasCandidate(GridCacheVersion ver) {
+        for (int i = 0; i < list.size(); i++) {
+            GridCacheMvccCandidate cand = list.get(i);
+
+            if (cand.version().equals(ver))
+                return true;
+        }
+
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(CacheLockCandidatesList.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index 176fe77aa8d3c..d8194fcb30de7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -565,6 +565,7 @@ public GridTuple3<Boolean, Object, EntryProcessorResult<Object>> innerUpdateLoca
      * @param timeout Timeout for lock acquisition.
      * @param serOrder Version for serializable transactions ordering.
      * @param serReadVer Optional read entry version for optimistic serializable transaction.
+     * @param read Read lock flag.
      * @return {@code True} if lock was acquired, {@code false} otherwise.
      * @throws GridCacheEntryRemovedException If this entry is obsolete.
      * @throws GridDistributedLockCancelledException If lock has been cancelled.
@@ -573,7 +574,7 @@ public boolean tmLock(IgniteInternalTx tx,
         long timeout,
         @Nullable GridCacheVersion serOrder,
         @Nullable GridCacheVersion serReadVer,
-        boolean keepBinary
+        boolean read
     ) throws GridCacheEntryRemovedException, GridDistributedLockCancelledException;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index aec28bb862c74..31baedacc7956 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -79,9 +79,11 @@
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_EXPIRED;
+import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_LOCKED;
 import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_PUT;
 import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_READ;
 import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_REMOVED;
+import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_UNLOCKED;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
 import static org.apache.ignite.internal.processors.dr.GridDrType.DR_NONE;
 
@@ -827,8 +829,6 @@ private Object innerGet0(
         if (readThrough && !cctx.readThrough())
             readThrough = false;
 
-        GridCacheMvccCandidate owner;
-
         CacheObject ret;
 
         GridCacheVersion startVer;
@@ -841,10 +841,6 @@ private Object innerGet0(
         synchronized (this) {
             checkObsolete();
 
-            GridCacheMvcc mvcc = mvccExtras();
-
-            owner = mvcc == null ? null : mvcc.anyOwner();
-
             boolean valid = valid(tx != null ? tx.topologyVersion() : cctx.affinity().affinityTopologyVersion());
 
             CacheObject val;
@@ -899,11 +895,13 @@ private Object innerGet0(
             if (evt && cctx.events().isRecordable(EVT_CACHE_OBJECT_READ)) {
                 transformClo = EntryProcessorResourceInjectorProxy.unwrap(transformClo);
 
+                GridCacheMvcc mvcc = mvccExtras();
+
                 cctx.events().addEvent(
                     partition(),
                     key,
                     tx,
-                    owner,
+                    mvcc != null ? mvcc.anyOwner() : null,
                     EVT_CACHE_OBJECT_READ,
                     ret,
                     ret != null,
@@ -1010,11 +1008,13 @@ else if (tx.dht()) {
                 if (evt && cctx.events().isRecordable(EVT_CACHE_OBJECT_READ)) {
                     transformClo = EntryProcessorResourceInjectorProxy.unwrap(transformClo);
 
+                    GridCacheMvcc mvcc = mvccExtras();
+
                     cctx.events().addEvent(
                         partition(),
                         key,
                         tx,
-                        owner,
+                        mvcc != null ? mvcc.anyOwner() : null,
                         EVT_CACHE_OBJECT_READ,
                         ret,
                         ret != null,
@@ -3391,14 +3391,14 @@ private boolean checkExpired() throws IgniteCheckedException {
     }
 
     /** {@inheritDoc} */
-    @Override public synchronized boolean hasValue() {
+    @Override public final synchronized boolean hasValue() {
         return hasValueUnlocked();
     }
 
     /**
      * @return {@code True} if this entry has value.
      */
-    protected boolean hasValueUnlocked() {
+    protected final boolean hasValueUnlocked() {
         assert Thread.holdsLock(this);
 
         return val != null || hasOffHeapPointer();
@@ -4318,7 +4318,7 @@ private void evictFailed(@Nullable CacheObject prevVal) throws IgniteCheckedExce
     }
 
     /** {@inheritDoc} */
-    @Override public GridCacheBatchSwapEntry evictInBatchInternal(GridCacheVersion obsoleteVer)
+    @Override public final GridCacheBatchSwapEntry evictInBatchInternal(GridCacheVersion obsoleteVer)
         throws IgniteCheckedException {
         assert Thread.holdsLock(this);
         assert cctx.isSwapOrOffheapEnabled();
@@ -4385,7 +4385,7 @@ private void evictFailed(@Nullable CacheObject prevVal) throws IgniteCheckedExce
      * @param filter Entry filter.
      * @return {@code True} if entry is visitable.
      */
-    public boolean visitable(CacheEntryPredicate[] filter) {
+    public final boolean visitable(CacheEntryPredicate[] filter) {
         boolean rmv = false;
 
         try {
@@ -4440,7 +4440,7 @@ public boolean visitable(CacheEntryPredicate[] filter) {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean deleted() {
+    @Override public final boolean deleted() {
         if (!cctx.deferredDelete())
             return false;
 
@@ -4450,7 +4450,7 @@ public boolean visitable(CacheEntryPredicate[] filter) {
     }
 
     /** {@inheritDoc} */
-    @Override public synchronized boolean obsoleteOrDeleted() {
+    @Override public final synchronized boolean obsoleteOrDeleted() {
         return obsoleteVersionExtras() != null ||
             (cctx.deferredDelete() && (deletedUnlocked() || !hasValueUnlocked()));
     }
@@ -4459,7 +4459,7 @@ public boolean visitable(CacheEntryPredicate[] filter) {
      * @return {@code True} if deleted.
      */
     @SuppressWarnings("SimplifiableIfStatement")
-    protected boolean deletedUnlocked() {
+    protected final boolean deletedUnlocked() {
         assert Thread.holdsLock(this);
 
         if (!cctx.deferredDelete())
@@ -4471,7 +4471,7 @@ protected boolean deletedUnlocked() {
     /**
      * @param deleted {@code True} if deleted.
      */
-    protected void deletedUnlocked(boolean deleted) {
+    protected final void deletedUnlocked(boolean deleted) {
         assert Thread.holdsLock(this);
         assert cctx.deferredDelete();
 
@@ -4508,7 +4508,7 @@ protected void decrementMapPublicSize() {
     /**
      * @return MVCC.
      */
-    @Nullable protected GridCacheMvcc mvccExtras() {
+    @Nullable protected final GridCacheMvcc mvccExtras() {
         return extras != null ? extras.mvcc() : null;
     }
 
@@ -4516,7 +4516,7 @@ protected void decrementMapPublicSize() {
      * @return All MVCC local and non near candidates.
      */
     @SuppressWarnings("ForLoopReplaceableByForEach")
-    @Nullable public synchronized List<GridCacheMvccCandidate> mvccAllLocal() {
+    @Nullable public final synchronized List<GridCacheMvccCandidate> mvccAllLocal() {
         GridCacheMvcc mvcc = extras != null ? extras.mvcc() : null;
 
         if (mvcc == null)
@@ -4542,21 +4542,22 @@ protected void decrementMapPublicSize() {
     /**
      * @param mvcc MVCC.
      */
-    protected void mvccExtras(@Nullable GridCacheMvcc mvcc) {
+    protected final void mvccExtras(@Nullable GridCacheMvcc mvcc) {
         extras = (extras != null) ? extras.mvcc(mvcc) : mvcc != null ? new GridCacheMvccEntryExtras(mvcc) : null;
     }
 
     /**
      * @return Obsolete version.
      */
-    @Nullable protected GridCacheVersion obsoleteVersionExtras() {
+    @Nullable protected final GridCacheVersion obsoleteVersionExtras() {
         return extras != null ? extras.obsoleteVersion() : null;
     }
 
     /**
      * @param obsoleteVer Obsolete version.
+     * @param ext Extras.
      */
-    protected void obsoleteVersionExtras(@Nullable GridCacheVersion obsoleteVer, GridCacheObsoleteEntryExtras ext) {
+    private void obsoleteVersionExtras(@Nullable GridCacheVersion obsoleteVer, GridCacheObsoleteEntryExtras ext) {
         extras = (extras != null) ?
             extras.obsoleteVersion(obsoleteVer) :
             obsoleteVer != null ?
@@ -4564,6 +4565,80 @@ protected void obsoleteVersionExtras(@Nullable GridCacheVersion obsoleteVer, Gri
                 null;
     }
 
+    /**
+     * @param prevOwners Previous owners.
+     * @param owners Current owners.
+     * @param val Entry value.
+     */
+    protected final void checkOwnerChanged(@Nullable CacheLockCandidates prevOwners,
+        @Nullable CacheLockCandidates owners,
+        CacheObject val) {
+        assert !Thread.holdsLock(this);
+
+        if (prevOwners != null && owners == null) {
+            cctx.mvcc().callback().onOwnerChanged(this, null);
+
+            if (cctx.events().isRecordable(EVT_CACHE_OBJECT_UNLOCKED)) {
+                boolean hasVal = hasValue();
+
+                GridCacheMvccCandidate cand = prevOwners.candidate(0);
+
+                cctx.events().addEvent(partition(),
+                    key,
+                    cand.nodeId(),
+                    cand,
+                    EVT_CACHE_OBJECT_UNLOCKED,
+                    val,
+                    hasVal,
+                    val,
+                    hasVal,
+                    null,
+                    null,
+                    null,
+                    true);
+            }
+        }
+
+        if (owners != null) {
+            for (int i = 0; i < owners.size(); i++) {
+                GridCacheMvccCandidate owner = owners.candidate(i);
+
+                boolean locked = prevOwners == null || !prevOwners.hasCandidate(owner.version());
+
+                if (locked) {
+                    cctx.mvcc().callback().onOwnerChanged(this, owner);
+
+                    if (owner.local())
+                        checkThreadChain(owner);
+
+                    if (cctx.events().isRecordable(EVT_CACHE_OBJECT_LOCKED)) {
+                        boolean hasVal = hasValue();
+
+                        // Event notification.
+                        cctx.events().addEvent(partition(),
+                            key,
+                            owner.nodeId(),
+                            owner,
+                            EVT_CACHE_OBJECT_LOCKED,
+                            val,
+                            hasVal,
+                            val,
+                            hasVal,
+                            null,
+                            null,
+                            null,
+                            true);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * @param owner Starting candidate in the chain.
+     */
+    protected abstract void checkThreadChain(GridCacheMvccCandidate owner);
+
     /**
      * Updates metrics.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvcc.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvcc.java
index 507a2c901538f..498584cc36303 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvcc.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvcc.java
@@ -107,7 +107,7 @@ public GridCacheMvcc(GridCacheContext<?, ?> cctx) {
     /**
      * @return Any owner.
      */
-    @Nullable public GridCacheMvccCandidate anyOwner() {
+    @Nullable GridCacheMvccCandidate anyOwner() {
         GridCacheMvccCandidate owner = localOwner();
 
         if (owner == null)
@@ -116,11 +116,23 @@ public GridCacheMvcc(GridCacheContext<?, ?> cctx) {
         return owner;
     }
 
+    /**
+     * @return All owners.
+     */
+    @Nullable public CacheLockCandidates allOwners() {
+        CacheLockCandidates owners = localOwners();
+
+        if (owners == null)
+            owners = remoteOwner();
+
+        return owners;
+    }
+
     /**
      * @return Remote candidate only if it's first in the list and is marked
      *      as <tt>'used'</tt>.
      */
-    @Nullable public GridCacheMvccCandidate remoteOwner() {
+    @Nullable private GridCacheMvccCandidate remoteOwner() {
         if (rmts != null) {
             assert !rmts.isEmpty();
 
@@ -132,11 +144,59 @@ public GridCacheMvcc(GridCacheContext<?, ?> cctx) {
         return null;
     }
 
+    /**
+     * @return All local owners.
+     */
+    @Nullable public CacheLockCandidates localOwners() {
+        if (locs != null) {
+            assert !locs.isEmpty();
+
+            CacheLockCandidates owners = null;
+
+            GridCacheMvccCandidate first = locs.getFirst();
+
+            if (first.read()) {
+                for (GridCacheMvccCandidate cand : locs) {
+                    if (cand.owner()) {
+                        assert cand.read() : this;
+
+                        if (owners != null) {
+                            CacheLockCandidatesList list;
+
+                            if (owners.size() == 1) {
+                                GridCacheMvccCandidate owner = owners.candidate(0);
+
+                                owners = list = new CacheLockCandidatesList();
+
+                                ((CacheLockCandidatesList)owners).add(owner);
+                            }
+                            else
+                                list = ((CacheLockCandidatesList)owners);
+
+                            list.add(cand);
+                        }
+                        else
+                            owners = cand;
+                    }
+
+                    if (!cand.read())
+                        break;
+                }
+            }
+            else if (first.owner())
+                owners = first;
+
+            return owners;
+        }
+
+        return null;
+    }
+
     /**
      * @return Local candidate only if it's first in the list and is marked
      *      as <tt>'owner'</tt>.
      */
-    @Nullable public GridCacheMvccCandidate localOwner() {
+    @Nullable GridCacheMvccCandidate localOwner() {
         if (locs != null) {
             assert !locs.isEmpty();
 
@@ -185,6 +245,29 @@ public GridCacheMvcc(GridCacheContext<?, ?> cctx) {
         return null;
     }
 
+    /**
+     * @param cand Existing candidate.
+     * @param newCand New candidate.
+     * @return {@code False} if new candidate can not be added.
+     */
+    private boolean compareSerializableVersion(GridCacheMvccCandidate cand, GridCacheMvccCandidate newCand) {
+        assert cand.serializable() && newCand.serializable();
+
+        GridCacheVersion candOrder = cand.serializableOrder();
+
+        assert candOrder != null : cand;
+
+        GridCacheVersion newCandOrder = newCand.serializableOrder();
+
+        assert newCandOrder != null : newCand;
+
+        int cmp = SER_VER_COMPARATOR.compare(candOrder, newCandOrder);
+
+        assert cmp != 0;
+
+        return cmp < 0;
+    }
+
     /**
      * @param cand Candidate to add.
      * @return {@code False} if failed to add candidate and transaction should be cancelled.
@@ -200,25 +283,34 @@ private boolean add0(GridCacheMvccCandidate cand) {
             if (!cand.nearLocal()) {
                 if (!locs.isEmpty()) {
                     if (cand.serializable()) {
-                        GridCacheMvccCandidate last = locs.getLast();
-
-                        if (!last.serializable())
-                            return false;
-
-                        GridCacheVersion lastOrder = last.serializableOrder();
+                        Iterator<GridCacheMvccCandidate> it = locs.descendingIterator();
 
-                        assert lastOrder != null : last;
+                        if (cand.read()) {
+                            while (it.hasNext()) {
+                                GridCacheMvccCandidate c = it.next();
 
-                        GridCacheVersion candOrder = cand.serializableOrder();
+                                if (!c.serializable())
+                                    return false;
 
-                        assert candOrder != null : cand;
-
-                        int cmp = SER_VER_COMPARATOR.compare(lastOrder, candOrder);
+                                if (!c.read()) {
+                                    if (compareSerializableVersion(c, cand))
+                                        break;
+                                    else
+                                        return false;
+                                }
+                            }
+                        }
+                        else {
+                            while (it.hasNext()) {
+                                GridCacheMvccCandidate c = it.next();
 
-                        assert cmp != 0;
+                                if (!c.serializable() || !compareSerializableVersion(c, cand))
+                                    return false;
 
-                        if (cmp > 0)
-                            return false;
+                                if (!c.read())
+                                    break;
+                            }
+                        }
 
                         locs.addLast(cand);
 
@@ -284,12 +376,12 @@ private boolean add0(GridCacheMvccCandidate cand) {
         }
         // Remote.
         else {
-            assert !cand.serializable() : cand;
+            assert !cand.serializable() && !cand.read() : cand;
 
             if (rmts == null)
                 rmts = new LinkedList<>();
 
-            assert !cand.owner() || localOwner() == null : "Cannot have local and remote owners " +
+            assert !cand.owner() || localOwners() == null : "Cannot have local and remote owners " +
                 "at the same time [cand=" + cand + ", locs=" + locs + ", rmts=" + rmts + ']';
 
             GridCacheMvccCandidate cur = candidate(rmts, cand.version());
@@ -398,9 +490,8 @@ public boolean isEmpty(GridCacheVersion... exclude) {
      * @param baseVer Base version.
      * @param committedVers Committed versions relative to base.
      * @param rolledbackVers Rolled back versions relative to base.
-     * @return Lock owner.
      */
-    @Nullable public GridCacheMvccCandidate orderCompleted(GridCacheVersion baseVer,
+    public void orderCompleted(GridCacheVersion baseVer,
         Collection<GridCacheVersion> committedVers, Collection<GridCacheVersion> rolledbackVers) {
         assert baseVer != null;
 
@@ -415,10 +506,13 @@ public boolean isEmpty(GridCacheVersion... exclude) {
                 if (!cur.version().equals(baseVer) && committedVers.contains(cur.version())) {
                     cur.setOwner();
 
-                    assert localOwner() == null || localOwner().nearLocal(): "Cannot not have local owner and " +
+                    assert localOwners() == null || localOwner().nearLocal(): "Cannot not have local owner and " +
                         "remote completed transactions at the same time [baseVer=" + baseVer +
-                        ", committedVers=" + committedVers + ", rolledbackVers=" + rolledbackVers +
-                        ", localOwner=" + localOwner() + ", locs=" + locs + ", rmts=" + rmts + ']';
+                        ", committedVers=" + committedVers +
+                        ", rolledbackVers=" + rolledbackVers +
+                        ", localOwner=" + localOwner() +
+                        ", locs=" + locs +
+                        ", rmts=" + rmts + ']';
 
                     if (maxIdx < 0)
                         maxIdx = it.nextIndex();
@@ -462,8 +556,6 @@ else if (maxIdx >= 0 && cur.version().isGreaterEqual(baseVer)) {
                     rmts = null;
             }
         }
-
-        return anyOwner();
     }
 
     /**
@@ -471,11 +563,10 @@ else if (maxIdx >= 0 && cur.version().isGreaterEqual(baseVer)) {
      *
      * @param baseVer Base version.
      * @param owned Owned list.
-     * @return Current owner.
      */
-    @Nullable public GridCacheMvccCandidate markOwned(GridCacheVersion baseVer, GridCacheVersion owned) {
+    public void markOwned(GridCacheVersion baseVer, GridCacheVersion owned) {
         if (owned == null)
-            return anyOwner();
+            return;
 
         if (rmts != null) {
             GridCacheMvccCandidate baseCand = candidate(rmts, baseVer);
@@ -483,8 +574,6 @@ else if (maxIdx >= 0 && cur.version().isGreaterEqual(baseVer)) {
             if (baseCand != null)
                 baseCand.ownerVersion(owned);
         }
-
-        return anyOwner();
     }
 
     /**
@@ -495,6 +584,7 @@ else if (maxIdx >= 0 && cur.version().isGreaterEqual(baseVer)) {
      * @param reenter Reentry flag ({@code true} if reentry is allowed).
      * @param tx Transaction flag.
      * @param implicitSingle Implicit transaction flag.
+     * @param read Read lock flag.
      * @return New lock candidate if lock was added, or current owner if lock was reentered,
      *      or <tt>null</tt> if lock was owned by another thread and timeout is negative.
      */
@@ -505,7 +595,8 @@ else if (maxIdx >= 0 && cur.version().isGreaterEqual(baseVer)) {
         long timeout,
         boolean reenter,
         boolean tx,
-        boolean implicitSingle) {
+        boolean implicitSingle,
+        boolean read) {
         return addLocal(
             parent,
             /*nearNodeId*/null,
@@ -517,7 +608,8 @@ else if (maxIdx >= 0 && cur.version().isGreaterEqual(baseVer)) {
             reenter,
             tx,
             implicitSingle,
-            /*dht-local*/false
+            /*dht-local*/false,
+            /*read*/read
         );
     }
 
@@ -533,6 +625,7 @@ else if (maxIdx >= 0 && cur.version().isGreaterEqual(baseVer)) {
      * @param tx Transaction flag.
      * @param implicitSingle Implicit flag.
      * @param dhtLoc DHT local flag.
+     * @param read Read lock flag.
      * @return New lock candidate if lock was added, or current owner if lock was reentered,
      *      or <tt>null</tt> if lock was owned by another thread and timeout is negative.
      */
@@ -547,7 +640,8 @@ else if (maxIdx >= 0 && cur.version().isGreaterEqual(baseVer)) {
         boolean reenter,
         boolean tx,
         boolean implicitSingle,
-        boolean dhtLoc) {
+        boolean dhtLoc,
+        boolean read) {
         if (log.isDebugEnabled())
             log.debug("Adding local candidate [mvcc=" + this + ", parent=" + parent + ", threadId=" + threadId +
                 ", ver=" + ver + ", timeout=" + timeout + ", reenter=" + reenter + ", tx=" + tx + "]");
@@ -582,14 +676,14 @@ else if (maxIdx >= 0 && cur.version().isGreaterEqual(baseVer)) {
             nearVer,
             threadId,
             ver,
-            timeout,
             /*local*/true,
             /*reenter*/false,
             tx,
             implicitSingle,
             /*near-local*/false,
             dhtLoc,
-            serOrder
+            serOrder,
+            read
         );
 
         if (serOrder == null) {
@@ -613,7 +707,6 @@ else if (maxIdx >= 0 && cur.version().isGreaterEqual(baseVer)) {
      * @param otherNodeId Other node ID.
      * @param threadId Thread ID.
      * @param ver Lock version.
-     * @param timeout Lock acquire timeout.
      * @param tx Transaction flag.
      * @param implicitSingle Implicit flag.
      * @param nearLoc Near local flag.
@@ -625,7 +718,6 @@ public GridCacheMvccCandidate addRemote(
         @Nullable UUID otherNodeId,
         long threadId,
         GridCacheVersion ver,
-        long timeout,
         boolean tx,
         boolean implicitSingle,
         boolean nearLoc) {
@@ -636,14 +728,14 @@ public GridCacheMvccCandidate addRemote(
             null,
             threadId,
             ver,
-            timeout,
             /*local*/false,
             /*reentry*/false,
             tx,
             implicitSingle,
             nearLoc,
             false,
-            null
+            null,
+            /*read*/false
         );
 
         addRemote(cand);
@@ -659,9 +751,9 @@ public GridCacheMvccCandidate addRemote(
      * @param otherNodeId Other node ID.
      * @param threadId Thread ID.
      * @param ver Lock version.
-     * @param timeout Lock acquire timeout.
      * @param tx Transaction flag.
      * @param implicitSingle Implicit flag.
+     * @param read Read lock flag.
      * @return Add remote candidate.
      */
     public GridCacheMvccCandidate addNearLocal(GridCacheEntryEx parent,
@@ -669,23 +761,23 @@ public GridCacheMvccCandidate addNearLocal(GridCacheEntryEx parent,
         @Nullable UUID otherNodeId,
         long threadId,
         GridCacheVersion ver,
-        long timeout,
         boolean tx,
-        boolean implicitSingle) {
+        boolean implicitSingle,
+        boolean read) {
         GridCacheMvccCandidate cand = new GridCacheMvccCandidate(parent,
             nodeId,
             otherNodeId,
             null,
             threadId,
             ver,
-            timeout,
             /*local*/true,
             /*reentry*/false,
             tx,
             implicitSingle,
             /*near loc*/true,
             /*dht loc*/false,
-            null);
+            null,
+            /*read*/read);
 
         add0(cand);
 
@@ -695,7 +787,7 @@ public GridCacheMvccCandidate addNearLocal(GridCacheEntryEx parent,
     /**
      * @param cand Remote candidate.
      */
-    public void addRemote(GridCacheMvccCandidate cand) {
+    private void addRemote(GridCacheMvccCandidate cand) {
         assert !cand.local();
 
         if (log.isDebugEnabled())
@@ -710,11 +802,11 @@ public void addRemote(GridCacheMvccCandidate cand) {
      * @param ver Lock version to acquire or set to ready.
      * @return Current owner.
      */
-    @Nullable public GridCacheMvccCandidate readyLocal(GridCacheVersion ver) {
+    @Nullable public CacheLockCandidates readyLocal(GridCacheVersion ver) {
         GridCacheMvccCandidate cand = candidate(ver);
 
         if (cand == null)
-            return anyOwner();
+            return allOwners();
 
         assert cand.local();
 
@@ -725,14 +817,14 @@ public void addRemote(GridCacheMvccCandidate cand) {
      * @param cand Local candidate added in any of the {@code addLocal(..)} methods.
      * @return Current lock owner.
      */
-    @Nullable public GridCacheMvccCandidate readyLocal(GridCacheMvccCandidate cand) {
+    @Nullable public CacheLockCandidates readyLocal(GridCacheMvccCandidate cand) {
         assert cand.local();
 
         cand.setReady();
 
         reassign();
 
-        return anyOwner();
+        return allOwners();
     }
 
     /**
@@ -751,9 +843,12 @@ public void addRemote(GridCacheMvccCandidate cand) {
      * @param pending Pending dht versions that are not owned and which version is less then mapped.
      * @return Lock owner after reassignment.
      */
-    @Nullable public GridCacheMvccCandidate readyNearLocal(GridCacheVersion ver, GridCacheVersion mappedVer,
-        Collection<GridCacheVersion> committedVers, Collection<GridCacheVersion> rolledBackVers,
-        Collection<GridCacheVersion> pending) {
+    @Nullable public CacheLockCandidates readyNearLocal(GridCacheVersion ver,
+        GridCacheVersion mappedVer,
+        Collection<GridCacheVersion> committedVers,
+        Collection<GridCacheVersion> rolledBackVers,
+        Collection<GridCacheVersion> pending)
+    {
         GridCacheMvccCandidate cand = candidate(locs, ver);
 
         if (cand != null) {
@@ -785,7 +880,7 @@ public void addRemote(GridCacheMvccCandidate cand) {
                     if (c.owner())
                         continue;
 
-                    assert !c.ready() :
+                    assert !c.ready() || (c.read() && cand.read()):
                         "Cannot have more then one ready near-local candidate [c=" + c + ", cand=" + cand +
                             ", mvcc=" + this + ']';
 
@@ -819,7 +914,7 @@ public void addRemote(GridCacheMvccCandidate cand) {
             reassign();
         }
 
-        return anyOwner();
+        return allOwners();
     }
 
     /**
@@ -831,7 +926,7 @@ public void addRemote(GridCacheMvccCandidate cand) {
      * @param rolledback Rolledback versions.
      * @return Lock owner.
      */
-    @Nullable public GridCacheMvccCandidate doneRemote(
+    @Nullable public CacheLockCandidates doneRemote(
         GridCacheVersion ver,
         Collection<GridCacheVersion> pending,
         Collection<GridCacheVersion> committed,
@@ -879,7 +974,7 @@ else if (!committed.contains(c.version()) && !rolledback.contains(c.version()) &
             }
         }
 
-        return anyOwner();
+        return allOwners();
     }
 
     /**
@@ -942,19 +1037,39 @@ private void reassign() {
         if (locs != null) {
             boolean first = true;
 
-            for (ListIterator<GridCacheMvccCandidate> it = locs.listIterator(); it.hasNext(); ) {
+            ListIterator<GridCacheMvccCandidate> it = locs.listIterator();
+
+            while (it.hasNext()) {
                 GridCacheMvccCandidate cand = it.next();
 
-                if (first && cand.serializable()) {
-                    if (cand.owner() || !cand.ready())
+                if (first) {
+                    if (cand.read()) {
+                        if (cand.ready() && !cand.owner())
+                            cand.setOwner();
+
+                        while (it.hasNext()) {
+                            cand = it.next();
+
+                            if (!cand.read())
+                                break;
+
+                            if (cand.ready() && !cand.owner())
+                                cand.setOwner();
+                        }
+
                         return;
+                    }
+                    else if (cand.serializable()) {
+                        if (cand.owner() || !cand.ready())
+                            return;
 
-                    cand.setOwner();
+                        cand.setOwner();
 
-                    return;
-                }
+                        return;
+                    }
 
-                first = false;
+                    first = false;
+                }
 
                 if (cand.owner())
                     return;
@@ -1036,6 +1151,8 @@ private void reassign() {
                     }
 
                     if (assigned) {
+                        assert !cand.serializable() : cand;
+
                         it.remove();
 
                         // Owner must be first in the list.
@@ -1066,15 +1183,16 @@ private void reassign() {
      *
      * @return Owner.
      */
-    @Nullable public GridCacheMvccCandidate recheck() {
+    @Nullable public CacheLockCandidates recheck() {
         reassign();
 
-        return anyOwner();
+        return allOwners();
     }
 
     /**
      * Local local release.
-     * @return Removed lock candidate or <tt>null</tt> if candidate was not removed.
+     *
+     * @return Removed candidate.
      */
     @Nullable public GridCacheMvccCandidate releaseLocal() {
         return releaseLocal(Thread.currentThread().getId());
@@ -1084,32 +1202,45 @@ private void reassign() {
      * Local release.
      *
      * @param threadId ID of the thread.
-     * @return Current owner.
+     * @return Removed candidate.
      */
     @Nullable public GridCacheMvccCandidate releaseLocal(long threadId) {
-        GridCacheMvccCandidate owner = localOwner();
+        CacheLockCandidates owners = localOwners();
 
-        if (owner == null || owner.threadId() != threadId)
-            // Release had no effect.
-            return owner;
+        // Release had no effect.
+        if (owners == null)
+            return null;
 
-        owner.setUsed();
+        GridCacheMvccCandidate owner = null;
 
-        remove0(owner.version(), true);
+        for (int i = 0; i < owners.size(); i++) {
+            GridCacheMvccCandidate owner0 = owners.candidate(i);
 
-        return anyOwner();
+            if (owner0.threadId() == threadId) {
+                owner = owner0;
+
+                break;
+            }
+        }
+
+        if (owner != null) {
+            owner.setUsed();
+
+            remove0(owner.version(), true);
+
+            return owner;
+        }
+        else
+            return null;
     }
 
     /**
      * Removes lock even if it is not owner.
      *
      * @param ver Lock version.
-     * @return Current owner.
      */
-    @Nullable public GridCacheMvccCandidate remove(GridCacheVersion ver) {
+    public void remove(GridCacheVersion ver) {
         remove0(ver, false);
-
-        return anyOwner();
     }
 
     /**
@@ -1118,7 +1249,7 @@ private void reassign() {
      * @param nodeId Node ID.
      * @return Current owner.
      */
-    @Nullable public GridCacheMvccCandidate removeExplicitNodeCandidates(UUID nodeId) {
+    @Nullable public CacheLockCandidates removeExplicitNodeCandidates(UUID nodeId) {
         if (rmts != null) {
             for (Iterator<GridCacheMvccCandidate> it = rmts.iterator(); it.hasNext(); ) {
                 GridCacheMvccCandidate cand = it.next();
@@ -1153,7 +1284,7 @@ private void reassign() {
 
         reassign();
 
-        return anyOwner();
+        return allOwners();
     }
 
     /**
@@ -1177,7 +1308,7 @@ private void reassign() {
      * @param threadId Thread ID.
      * @return Candidate or <tt>null</tt> if there is no candidate for given ID.
      */
-    @Nullable public GridCacheMvccCandidate localCandidate(long threadId) {
+    @Nullable GridCacheMvccCandidate localCandidate(long threadId) {
         // Don't return reentries.
         return localCandidate(threadId, false);
     }
@@ -1187,7 +1318,7 @@ private void reassign() {
      * @param threadId Thread ID.
      * @return Remote candidate.
      */
-    @Nullable public GridCacheMvccCandidate remoteCandidate(UUID nodeId, long threadId) {
+    @Nullable GridCacheMvccCandidate remoteCandidate(UUID nodeId, long threadId) {
         if (rmts != null)
             for (GridCacheMvccCandidate c : rmts)
                 if (c.nodeId().equals(nodeId) && c.threadId() == threadId)
@@ -1217,7 +1348,7 @@ private void reassign() {
      * @param ver Version.
      * @return {@code True} if candidate with given version exists.
      */
-    public boolean hasCandidate(GridCacheVersion ver) {
+    boolean hasCandidate(GridCacheVersion ver) {
         return candidate(ver) != null;
     }
 
@@ -1283,41 +1414,25 @@ private List<GridCacheMvccCandidate> candidates(List<GridCacheMvccCandidate> col
         return cands;
     }
 
-    /**
-     * @return {@code True} if lock is owner by current thread.
-     */
-    public boolean isLocallyOwnedByCurrentThread() {
-        return isLocallyOwnedByThread(Thread.currentThread().getId(), true);
-    }
-
     /**
      * @param threadId Thread ID to check.
      * @param exclude Versions to ignore.
      * @return {@code True} if lock is owned by the thread with given ID.
      */
-    public boolean isLocallyOwnedByThread(long threadId, boolean allowDhtLoc, GridCacheVersion... exclude) {
-        GridCacheMvccCandidate owner = localOwner();
-
-        return owner != null && owner.threadId() == threadId && owner.nodeId().equals(cctx.nodeId()) &&
-            (allowDhtLoc || !owner.dhtLocal()) && !U.containsObjectArray(exclude, owner.version());
-    }
+    boolean isLocallyOwnedByThread(long threadId, boolean allowDhtLoc, GridCacheVersion... exclude) {
+        CacheLockCandidates owners = localOwners();
 
-    /**
-     * @param threadId Thread ID.
-     * @param nodeId Node ID.
-     * @return {@code True} if lock is held by given thread and node IDs.
-     */
-    public boolean isLockedByThread(long threadId, UUID nodeId) {
-        GridCacheMvccCandidate owner = anyOwner();
+        if (owners != null) {
+            for (int i = 0; i < owners.size(); i++) {
+                GridCacheMvccCandidate owner = owners.candidate(i);
 
-        return owner != null && owner.threadId() == threadId && owner.nodeId().equals(nodeId);
-    }
+                if (owner.threadId() == threadId && owner.nodeId().equals(cctx.nodeId()) &&
+                    (allowDhtLoc || !owner.dhtLocal()) && !U.containsObjectArray(exclude, owner.version()))
+                    return true;
+            }
+        }
 
-    /**
-     * @return {@code True} if lock is owned by any thread or node.
-     */
-    public boolean isOwnedByAny() {
-        return anyOwner() != null;
+        return false;
     }
 
     /**
@@ -1325,10 +1440,10 @@ public boolean isOwnedByAny() {
      * @param lockVer ID of lock candidate.
      * @return {@code True} if candidate is owner.
      */
-    public boolean isLocallyOwned(GridCacheVersion lockVer) {
-        GridCacheMvccCandidate owner = localOwner();
+    boolean isLocallyOwned(GridCacheVersion lockVer) {
+        CacheLockCandidates owners = localOwners();
 
-        return owner != null && owner.version().equals(lockVer);
+        return owners != null && owners.hasCandidate(lockVer);
     }
 
     /**
@@ -1336,30 +1451,25 @@ public boolean isLocallyOwned(GridCacheVersion lockVer) {
      * @param threadId Thread ID.
      * @return {@code True} if locked by lock ID or thread ID.
      */
-    public boolean isLocallyOwnedByIdOrThread(GridCacheVersion lockVer, long threadId) {
-        GridCacheMvccCandidate owner = localOwner();
+    boolean isLocallyOwnedByIdOrThread(GridCacheVersion lockVer, long threadId) {
+        CacheLockCandidates owners = localOwners();
 
-        return owner != null && (owner.version().equals(lockVer) || owner.threadId() == threadId);
-    }
+        if (owners != null) {
+            for (int i = 0; i < owners.size(); i++) {
+                GridCacheMvccCandidate owner = owners.candidate(i);
 
-    /**
-     * @return First remote entry or <tt>null</tt>.
-     */
-    @Nullable public GridCacheMvccCandidate firstRemote() {
-        return rmts == null ? null : rmts.getFirst();
-    }
+                if ((owner.version().equals(lockVer) || owner.threadId() == threadId))
+                    return true;
+            }
+        }
 
-    /**
-     * @return First local entry or <tt>null</tt>.
-     */
-    @Nullable public GridCacheMvccCandidate firstLocal() {
-        return locs == null ? null : locs.getFirst();
+        return false;
     }
 
     /**
      * @return Local MVCC candidates.
      */
-    @Nullable public List<GridCacheMvccCandidate> allLocal() {
+    @Nullable List<GridCacheMvccCandidate> allLocal() {
         return locs;
     }
 
@@ -1367,10 +1477,10 @@ public boolean isLocallyOwnedByIdOrThread(GridCacheVersion lockVer, long threadI
      * @param ver Version to check for ownership.
      * @return {@code True} if lock is owned by the specified version.
      */
-    public boolean isOwnedBy(GridCacheVersion ver) {
-        GridCacheMvccCandidate cand = anyOwner();
+    boolean isOwnedBy(GridCacheVersion ver) {
+        CacheLockCandidates owners = allOwners();
 
-        return cand != null && cand.version().equals(ver);
+        return owners != null && owners.hasCandidate(ver);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCallback.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCallback.java
index fc1faf7566732..2ba41f73244c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCallback.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCallback.java
@@ -37,11 +37,9 @@ public interface GridCacheMvccCallback {
      * can be made from this call.
      *
      * @param entry Entry.
-     * @param prev Previous candidate.
      * @param owner Current owner.
      */
-    public void onOwnerChanged(GridCacheEntryEx entry, GridCacheMvccCandidate prev,
-        GridCacheMvccCandidate owner);
+    public void onOwnerChanged(GridCacheEntryEx entry, GridCacheMvccCandidate owner);
 
     /**
      * Called when entry has no more candidates. This call happens
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java
index f1c1b83a51d66..e9dd455901e44 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java
@@ -43,6 +43,7 @@
 import static org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate.Mask.LOCAL;
 import static org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate.Mask.NEAR_LOCAL;
 import static org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate.Mask.OWNER;
+import static org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate.Mask.READ;
 import static org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate.Mask.READY;
 import static org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate.Mask.REENTRY;
 import static org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate.Mask.REMOVED;
@@ -54,7 +55,7 @@
  * Lock candidate.
  */
 public class GridCacheMvccCandidate implements Externalizable,
-    Comparable<GridCacheMvccCandidate> {
+    Comparable<GridCacheMvccCandidate>, CacheLockCandidates {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -69,14 +70,6 @@ public class GridCacheMvccCandidate implements Externalizable,
     @GridToStringInclude
     private GridCacheVersion ver;
 
-    /** Maximum wait time. */
-    @GridToStringInclude
-    private long timeout;
-
-    /** Candidate timestamp. */
-    @GridToStringInclude
-    private long ts;
-
     /** Thread ID. */
     @GridToStringInclude
     private long threadId;
@@ -143,7 +136,6 @@ public GridCacheMvccCandidate() {
      * @param otherVer Other version.
      * @param threadId Requesting thread ID.
      * @param ver Cache version.
-     * @param timeout Maximum wait time.
      * @param loc {@code True} if the lock is local.
      * @param reentry {@code True} if candidate is for reentry.
      * @param tx Transaction flag.
@@ -151,6 +143,7 @@ public GridCacheMvccCandidate() {
      * @param nearLoc Near-local flag.
      * @param dhtLoc DHT local flag.
      * @param serOrder Version for serializable transactions ordering.
+     * @param read Read lock flag.
      */
     public GridCacheMvccCandidate(
         GridCacheEntryEx parent,
@@ -159,14 +152,14 @@ public GridCacheMvccCandidate(
         @Nullable GridCacheVersion otherVer,
         long threadId,
         GridCacheVersion ver,
-        long timeout,
         boolean loc,
         boolean reentry,
         boolean tx,
         boolean singleImplicit,
         boolean nearLoc,
         boolean dhtLoc,
-        @Nullable GridCacheVersion serOrder
+        @Nullable GridCacheVersion serOrder,
+        boolean read
     ) {
         assert nodeId != null;
         assert ver != null;
@@ -178,7 +171,6 @@ public GridCacheMvccCandidate(
         this.otherVer = otherVer;
         this.threadId = threadId;
         this.ver = ver;
-        this.timeout = timeout;
         this.serOrder = serOrder;
 
         mask(LOCAL, loc);
@@ -187,8 +179,7 @@ public GridCacheMvccCandidate(
         mask(SINGLE_IMPLICIT, singleImplicit);
         mask(NEAR_LOCAL, nearLoc);
         mask(DHT_LOCAL, dhtLoc);
-
-        ts = U.currentTimeMillis();
+        mask(READ, read);
 
         id = IDGEN.incrementAndGet();
     }
@@ -245,14 +236,14 @@ public GridCacheMvccCandidate reenter() {
             otherVer,
             threadId,
             ver,
-            timeout,
             local(),
             /*reentry*/true,
             tx(),
             singleImplicit(),
             nearLocal(),
             dhtLocal(),
-            serializableOrder());
+            serializableOrder(),
+            read());
 
         reentry.topVer = topVer;
 
@@ -410,20 +401,6 @@ public GridCacheVersion version() {
         return ver;
     }
 
-    /**
-     * @return Maximum wait time.
-     */
-    public long timeout() {
-        return timeout;
-    }
-
-    /**
-     * @return Timestamp at the time of entering pending set.
-     */
-    public long timestamp() {
-        return ts;
-    }
-
     /**
      * @return {@code True} if lock is local.
      */
@@ -473,6 +450,13 @@ public boolean serializable() {
         return serOrder;
     }
 
+    /**
+     * @return Read lock flag.
+     */
+    public boolean read() {
+        return READ.get(flags());
+    }
+
     /**
      * @return {@code True} if this candidate is a reentry.
      */
@@ -586,16 +570,21 @@ public IgniteTxKey key() {
         return parent0.txKey();
     }
 
-    /**
-     * Checks if this candidate matches version or thread-nodeId combination.
-     *
-     * @param nodeId Node ID to check.
-     * @param ver Version to check.
-     * @param threadId Thread ID to check.
-     * @return {@code True} if matched.
-     */
-    public boolean matches(GridCacheVersion ver, UUID nodeId, long threadId) {
-        return ver.equals(this.ver) || (nodeId.equals(this.nodeId) && threadId == this.threadId);
+    /** {@inheritDoc} */
+    @Override public GridCacheMvccCandidate candidate(int idx) {
+        assert idx == 0 : idx;
+
+        return this;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean hasCandidate(GridCacheVersion ver) {
+        return this.ver.equals(ver);
     }
 
     /** {@inheritDoc} */
@@ -610,7 +599,6 @@ public boolean matches(GridCacheVersion ver, UUID nodeId, long threadId) {
             ver.writeExternal(out);
         }
 
-        out.writeLong(timeout);
         out.writeLong(threadId);
         out.writeLong(id);
         out.writeShort(flags());
@@ -626,7 +614,6 @@ public boolean matches(GridCacheVersion ver, UUID nodeId, long threadId) {
             ver.readExternal(in);
         }
 
-        timeout = in.readLong();
         threadId = in.readLong();
         id = in.readLong();
 
@@ -635,8 +622,6 @@ public boolean matches(GridCacheVersion ver, UUID nodeId, long threadId) {
         mask(OWNER, OWNER.get(flags));
         mask(USED, USED.get(flags));
         mask(TX, TX.get(flags));
-
-        ts = U.currentTimeMillis();
     }
 
     /** {@inheritDoc} */
@@ -719,7 +704,10 @@ enum Mask {
         NEAR_LOCAL(0x200),
 
         /** */
-        REMOVED(0x400);
+        REMOVED(0x400),
+
+        /** */
+        READ(0x800);
 
         /** All mask values. */
         private static final Mask[] MASKS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
index c57e17cf8846f..0d0e9ee319d4d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
@@ -140,15 +140,14 @@ public class GridCacheMvccManager extends GridCacheSharedManagerAdapter {
     private final GridCacheMvccCallback cb = new GridCacheMvccCallback() {
         /** {@inheritDoc} */
         @SuppressWarnings({"unchecked"})
-        @Override public void onOwnerChanged(final GridCacheEntryEx entry, final GridCacheMvccCandidate prev,
-            final GridCacheMvccCandidate owner) {
+        @Override public void onOwnerChanged(final GridCacheEntryEx entry, final GridCacheMvccCandidate owner) {
             int nested = nestedLsnrCalls.get();
 
             if (nested < MAX_NESTED_LSNR_CALLS) {
                 nestedLsnrCalls.set(nested + 1);
 
                 try {
-                    notifyOwnerChanged(entry, prev, owner);
+                    notifyOwnerChanged(entry, owner);
                 }
                 finally {
                     nestedLsnrCalls.set(nested);
@@ -157,7 +156,7 @@ public class GridCacheMvccManager extends GridCacheSharedManagerAdapter {
             else {
                 cctx.kernalContext().closure().runLocalSafe(new GridPlainRunnable() {
                     @Override public void run() {
-                        notifyOwnerChanged(entry, prev, owner);
+                        notifyOwnerChanged(entry, owner);
                     }
                 }, true);
             }
@@ -182,19 +181,13 @@ public class GridCacheMvccManager extends GridCacheSharedManagerAdapter {
 
     /**
      * @param entry Entry to notify callback for.
-     * @param prev Previous lock owner.
      * @param owner Current lock owner.
      */
-    private void notifyOwnerChanged(final GridCacheEntryEx entry, final GridCacheMvccCandidate prev,
-        final GridCacheMvccCandidate owner) {
+    private void notifyOwnerChanged(final GridCacheEntryEx entry, final GridCacheMvccCandidate owner) {
         assert entry != null;
-        assert owner != prev : "New and previous owner are identical instances: " + owner;
-        assert owner == null || prev == null || !owner.version().equals(prev.version()) :
-            "New and previous owners have identical versions [owner=" + owner + ", prev=" + prev + ']';
 
         if (log.isDebugEnabled())
-            log.debug("Received owner changed callback [" + entry.key() + ", owner=" + owner + ", prev=" +
-                prev + ']');
+            log.debug("Received owner changed callback [" + entry.key() + ", owner=" + owner + ']');
 
         if (owner != null && (owner.local() || owner.nearLocal())) {
             Collection<GridCacheMvccFuture<?>> futCol = mvccFuts.get(owner.version());
@@ -226,7 +219,7 @@ private void notifyOwnerChanged(final GridCacheEntryEx entry, final GridCacheMvc
 
         if (log.isDebugEnabled())
             log.debug("Lock future not found for owner change callback (will try transaction futures) [owner=" +
-                owner + ", prev=" + prev + ", entry=" + entry + ']');
+                owner + ", entry=" + entry + ']');
 
         // If no future was found, delegate to transaction manager.
         if (cctx.tm().onOwnerChanged(entry, owner)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java
index 2d1b02ed4f94b..3d55f31c7aa34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.UUID;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheLockCandidates;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
@@ -66,7 +67,7 @@ public GridDistributedCacheEntry(
     /**
      *
      */
-    protected void refreshRemotes() {
+    private void refreshRemotes() {
         GridCacheMvcc mvcc = mvccExtras();
 
         rmts = mvcc == null ? Collections.<GridCacheMvccCandidate>emptyList() : mvcc.remoteCandidates();
@@ -82,6 +83,7 @@ protected void refreshRemotes() {
      * @param reenter Reentry flag.
      * @param tx Transaction flag.
      * @param implicitSingle Implicit flag.
+     * @param read Read lock flag.
      * @return New candidate.
      * @throws GridCacheEntryRemovedException If entry has been removed.
      */
@@ -92,10 +94,11 @@ protected void refreshRemotes() {
         long timeout,
         boolean reenter,
         boolean tx,
-        boolean implicitSingle) throws GridCacheEntryRemovedException {
+        boolean implicitSingle,
+        boolean read) throws GridCacheEntryRemovedException {
         GridCacheMvccCandidate cand;
-        GridCacheMvccCandidate prev;
-        GridCacheMvccCandidate owner;
+        CacheLockCandidates prev;
+        CacheLockCandidates owner;
 
         CacheObject val;
 
@@ -110,16 +113,23 @@ protected void refreshRemotes() {
                 mvccExtras(mvcc);
             }
 
-            prev = mvcc.anyOwner();
+            prev = mvcc.allOwners();
 
             boolean emptyBefore = mvcc.isEmpty();
 
-            cand = mvcc.addLocal(this, threadId, ver, timeout, reenter, tx, implicitSingle);
+            cand = mvcc.addLocal(this,
+                threadId,
+                ver,
+                timeout,
+                reenter,
+                tx,
+                implicitSingle,
+                read);
 
             if (cand != null)
                 cand.topologyVersion(topVer);
 
-            owner = mvcc.anyOwner();
+            owner = mvcc.allOwners();
 
             boolean emptyAfter = mvcc.isEmpty();
 
@@ -168,7 +178,6 @@ protected void refreshRemotes() {
      * @param otherNodeId Other node ID.
      * @param threadId Thread ID.
      * @param ver Lock version.
-     * @param timeout Lock acquire timeout.
      * @param tx Transaction flag.
      * @param implicitSingle Implicit flag.
      * @param owned Owned candidate version.
@@ -180,13 +189,12 @@ public void addRemote(
         @Nullable UUID otherNodeId,
         long threadId,
         GridCacheVersion ver,
-        long timeout,
         boolean tx,
         boolean implicitSingle,
         @Nullable GridCacheVersion owned
     ) throws GridDistributedLockCancelledException, GridCacheEntryRemovedException {
-        GridCacheMvccCandidate prev;
-        GridCacheMvccCandidate owner;
+        CacheLockCandidates prev;
+        CacheLockCandidates owner;
 
         CacheObject val;
 
@@ -204,7 +212,7 @@ public void addRemote(
                 mvccExtras(mvcc);
             }
 
-            prev = mvcc.anyOwner();
+            prev = mvcc.allOwners();
 
             boolean emptyBefore = mvcc.isEmpty();
 
@@ -214,7 +222,6 @@ public void addRemote(
                 otherNodeId,
                 threadId,
                 ver,
-                timeout,
                 tx,
                 implicitSingle,
                 /*near-local*/false
@@ -223,62 +230,7 @@ public void addRemote(
             if (owned != null)
                 mvcc.markOwned(ver, owned);
 
-            owner = mvcc.anyOwner();
-
-            boolean emptyAfter = mvcc.isEmpty();
-
-            checkCallbacks(emptyBefore, emptyAfter);
-
-            val = this.val;
-
-            refreshRemotes();
-
-            if (emptyAfter)
-                mvccExtras(null);
-        }
-
-        // This call must be outside of synchronization.
-        checkOwnerChanged(prev, owner, val);
-    }
-
-    /**
-     * Adds new lock candidate.
-     *
-     * @param cand Remote lock candidate.
-     * @throws GridDistributedLockCancelledException If lock has been canceled.
-     * @throws GridCacheEntryRemovedException If this entry is obsolete.
-     */
-    public void addRemote(GridCacheMvccCandidate cand) throws GridDistributedLockCancelledException,
-        GridCacheEntryRemovedException {
-
-        CacheObject val;
-
-        GridCacheMvccCandidate prev;
-        GridCacheMvccCandidate owner;
-
-        synchronized (this) {
-            cand.parent(this);
-
-            // Check removed locks prior to obsolete flag.
-            checkRemoved(cand.version());
-
-            checkObsolete();
-
-            GridCacheMvcc mvcc = mvccExtras();
-
-            if (mvcc == null) {
-                mvcc = new GridCacheMvcc(cctx);
-
-                mvccExtras(mvcc);
-            }
-
-            boolean emptyBefore = mvcc.isEmpty();
-
-            prev = mvcc.anyOwner();
-
-            mvcc.addRemote(cand);
-
-            owner = mvcc.anyOwner();
+            owner = mvcc.allOwners();
 
             boolean emptyAfter = mvcc.isEmpty();
 
@@ -303,8 +255,8 @@ public void addRemote(GridCacheMvccCandidate cand) throws GridDistributedLockCan
      * @throws GridCacheEntryRemovedException If entry was removed.
      */
     public void removeExplicitNodeLocks(UUID nodeId) throws GridCacheEntryRemovedException {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         CacheObject val = null;
 
@@ -314,7 +266,7 @@ public void removeExplicitNodeLocks(UUID nodeId) throws GridCacheEntryRemovedExc
             GridCacheMvcc mvcc = mvccExtras();
 
             if (mvcc != null) {
-                prev = mvcc.anyOwner();
+                prev = mvcc.allOwners();
 
                 boolean emptyBefore = mvcc.isEmpty();
 
@@ -346,8 +298,9 @@ public void removeExplicitNodeLocks(UUID nodeId) throws GridCacheEntryRemovedExc
      * @return Removed candidate, or <tt>null</tt> if thread still holds the lock.
      */
     @Nullable public GridCacheMvccCandidate removeLock() {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+        GridCacheMvccCandidate rmvd = null;
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         CacheObject val;
 
@@ -355,11 +308,11 @@ public void removeExplicitNodeLocks(UUID nodeId) throws GridCacheEntryRemovedExc
             GridCacheMvcc mvcc = mvccExtras();
 
             if (mvcc != null) {
-                prev = mvcc.anyOwner();
+                prev = mvcc.allOwners();
 
                 boolean emptyBefore = mvcc.isEmpty();
 
-                owner = mvcc.releaseLocal();
+                rmvd = mvcc.releaseLocal();
 
                 boolean emptyAfter = mvcc.isEmpty();
 
@@ -367,28 +320,38 @@ public void removeExplicitNodeLocks(UUID nodeId) throws GridCacheEntryRemovedExc
 
                 if (emptyAfter)
                     mvccExtras(null);
+                else
+                    owner = mvcc.allOwners();
             }
 
             val = this.val;
         }
 
-        if (log.isDebugEnabled())
-            log.debug("Released local candidate from entry [owner=" + owner + ", prev=" + prev +
+        if (log.isDebugEnabled()) {
+            log.debug("Released local candidate from entry [owner=" + owner +
+                ", prev=" + prev +
+                ", rmvd=" + rmvd +
                 ", entry=" + this + ']');
+        }
+
+        if (prev != null) {
+            for (int i = 0; i < prev.size(); i++) {
+                GridCacheMvccCandidate cand = prev.candidate(i);
 
-        if (prev != null && owner != prev)
-            checkThreadChain(prev);
+                checkThreadChain(cand);
+            }
+        }
 
         // This call must be outside of synchronization.
         checkOwnerChanged(prev, owner, val);
 
-        return owner != prev ? prev : null;
+        return rmvd;
     }
 
     /** {@inheritDoc} */
     @Override public boolean removeLock(GridCacheVersion ver) throws GridCacheEntryRemovedException {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         GridCacheMvccCandidate doomed;
 
@@ -408,13 +371,11 @@ public void removeExplicitNodeLocks(UUID nodeId) throws GridCacheEntryRemovedExc
                 checkObsolete();
 
             if (doomed != null) {
-                assert mvcc != null;
-
-                prev = mvcc.anyOwner();
+                prev = mvcc.allOwners();
 
                 boolean emptyBefore = mvcc.isEmpty();
 
-                owner = mvcc.remove(doomed.version());
+                mvcc.remove(doomed.version());
 
                 boolean emptyAfter = mvcc.isEmpty();
 
@@ -425,6 +386,8 @@ public void removeExplicitNodeLocks(UUID nodeId) throws GridCacheEntryRemovedExc
 
                 if (emptyAfter)
                     mvccExtras(null);
+                else
+                    owner = mvcc.allOwners();
             }
 
             val = this.val;
@@ -477,10 +440,10 @@ public boolean addRemoved(GridCacheVersion ver) {
      * @return Owner.
      * @throws GridCacheEntryRemovedException If entry is removed.
      */
-    @Nullable public GridCacheMvccCandidate readyLock(GridCacheVersion ver)
+    @Nullable public CacheLockCandidates readyLock(GridCacheVersion ver)
         throws GridCacheEntryRemovedException {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         CacheObject val;
 
@@ -490,13 +453,13 @@ public boolean addRemoved(GridCacheVersion ver) {
             GridCacheMvcc mvcc = mvccExtras();
 
             if (mvcc != null) {
-                prev = mvcc.anyOwner();
+                prev = mvcc.allOwners();
 
                 boolean emptyBefore = mvcc.isEmpty();
 
                 owner = mvcc.readyLocal(ver);
 
-                assert owner == null || owner.owner() : "Owner flag not set for owner: " + owner;
+                assert owner == null || owner.candidate(0).owner() : "Owner flag not set for owner: " + owner;
 
                 boolean emptyAfter = mvcc.isEmpty();
 
@@ -523,16 +486,16 @@ public boolean addRemoved(GridCacheVersion ver) {
      * @param committed Committed versions.
      * @param rolledBack Rolled back versions.
      * @param pending Pending locks on dht node with version less then mapped.
-     * @return Current lock owner.
      *
      * @throws GridCacheEntryRemovedException If entry is removed.
      */
-    @Nullable public GridCacheMvccCandidate readyNearLock(GridCacheVersion ver, GridCacheVersion mapped,
+    public void readyNearLock(GridCacheVersion ver, GridCacheVersion mapped,
         Collection<GridCacheVersion> committed,
         Collection<GridCacheVersion> rolledBack,
-        Collection<GridCacheVersion> pending) throws GridCacheEntryRemovedException {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+        Collection<GridCacheVersion> pending) throws GridCacheEntryRemovedException
+    {
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         CacheObject val;
 
@@ -542,13 +505,13 @@ public boolean addRemoved(GridCacheVersion ver) {
             GridCacheMvcc mvcc = mvccExtras();
 
             if (mvcc != null) {
-                prev = mvcc.anyOwner();
+                prev = mvcc.allOwners();
 
                 boolean emptyBefore = mvcc.isEmpty();
 
                 owner = mvcc.readyNearLocal(ver, mapped, committed, rolledBack, pending);
 
-                assert owner == null || owner.owner() : "Owner flag is not set for owner: " + owner;
+                assert owner == null || owner.candidate(0).owner() : "Owner flag is not set for owner: " + owner;
 
                 boolean emptyAfter = mvcc.isEmpty();
 
@@ -563,75 +526,6 @@ public boolean addRemoved(GridCacheVersion ver) {
 
         // This call must be made outside of synchronization.
         checkOwnerChanged(prev, owner, val);
-
-        return owner;
-    }
-
-    /**
-     * Reorders completed versions.
-     *
-     * @param baseVer Base version for reordering.
-     * @param committedVers Completed versions.
-     * @param rolledbackVers Rolled back versions.
-     * @throws GridCacheEntryRemovedException If entry has been removed.
-     */
-    public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion> committedVers,
-        Collection<GridCacheVersion> rolledbackVers)
-        throws GridCacheEntryRemovedException {
-        if (!F.isEmpty(committedVers) || !F.isEmpty(rolledbackVers)) {
-            GridCacheMvccCandidate prev = null;
-            GridCacheMvccCandidate owner = null;
-
-            CacheObject val;
-
-            synchronized (this) {
-                checkObsolete();
-
-                GridCacheMvcc mvcc = mvccExtras();
-
-                if (mvcc != null) {
-                    prev = mvcc.anyOwner();
-
-                    boolean emptyBefore = mvcc.isEmpty();
-
-                    owner = mvcc.orderCompleted(baseVer, committedVers, rolledbackVers);
-
-                    boolean emptyAfter = mvcc.isEmpty();
-
-                    checkCallbacks(emptyBefore, emptyAfter);
-
-                    if (emptyAfter)
-                        mvccExtras(null);
-                }
-
-                val = this.val;
-            }
-
-            // This call must be made outside of synchronization.
-            checkOwnerChanged(prev, owner, val);
-        }
-    }
-
-    /**
-     *
-     * @param lockVer Done version.
-     * @param baseVer Base version.
-     * @param committedVers Completed versions for reordering.
-     * @param rolledbackVers Rolled back versions for reordering.
-     * @param sysInvalidate Flag indicating if this entry is done from invalidated transaction (in case of tx
-     *      salvage). In this case all locks before salvaged lock will marked as used and corresponding
-     *      transactions will be invalidated.
-     * @throws GridCacheEntryRemovedException If entry has been removed.
-     * @return Owner.
-     */
-    @Nullable public GridCacheMvccCandidate doneRemote(
-        GridCacheVersion lockVer,
-        GridCacheVersion baseVer,
-        Collection<GridCacheVersion> committedVers,
-        Collection<GridCacheVersion> rolledbackVers,
-        boolean sysInvalidate) throws GridCacheEntryRemovedException {
-        return doneRemote(lockVer, baseVer, Collections.<GridCacheVersion>emptySet(), committedVers,
-            rolledbackVers, sysInvalidate);
     }
 
     /**
@@ -645,17 +539,16 @@ public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion
      *      salvage). In this case all locks before salvaged lock will marked as used and corresponding
      *      transactions will be invalidated.
      * @throws GridCacheEntryRemovedException If entry has been removed.
-     * @return Owner.
      */
-    @Nullable public GridCacheMvccCandidate doneRemote(
+    public void doneRemote(
         GridCacheVersion lockVer,
         GridCacheVersion baseVer,
         @Nullable Collection<GridCacheVersion> pendingVers,
         Collection<GridCacheVersion> committedVers,
         Collection<GridCacheVersion> rolledbackVers,
         boolean sysInvalidate) throws GridCacheEntryRemovedException {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         CacheObject val;
 
@@ -665,7 +558,7 @@ public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion
             GridCacheMvcc mvcc = mvccExtras();
 
             if (mvcc != null) {
-                prev = mvcc.anyOwner();
+                prev = mvcc.allOwners();
 
                 boolean emptyBefore = mvcc.isEmpty();
 
@@ -680,7 +573,9 @@ public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion
                 if (sysInvalidate && baseVer != null)
                     mvcc.salvageRemote(baseVer);
 
-                owner = mvcc.doneRemote(lockVer, maskNull(pendingVers), maskNull(committedVers),
+                owner = mvcc.doneRemote(lockVer,
+                    maskNull(pendingVers),
+                    maskNull(committedVers),
                     maskNull(rolledbackVers));
 
                 boolean emptyAfter = mvcc.isEmpty();
@@ -696,18 +591,14 @@ public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion
 
         // This call must be made outside of synchronization.
         checkOwnerChanged(prev, owner, val);
-
-        return owner;
     }
 
     /**
      * Rechecks if lock should be reassigned.
-     *
-     * @return Current owner.
      */
-    @Nullable public GridCacheMvccCandidate recheck() {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+    public void recheck() {
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         CacheObject val;
 
@@ -715,7 +606,7 @@ public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion
             GridCacheMvcc mvcc = mvccExtras();
 
             if (mvcc != null) {
-                prev = mvcc.anyOwner();
+                prev = mvcc.allOwners();
 
                 boolean emptyBefore = mvcc.isEmpty();
 
@@ -734,8 +625,6 @@ public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion
 
         // This call must be made outside of synchronization.
         checkOwnerChanged(prev, owner, val);
-
-        return owner;
     }
 
     /** {@inheritDoc} */
@@ -743,7 +632,7 @@ public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion
         long timeout,
         @Nullable GridCacheVersion serOrder,
         GridCacheVersion serReadVer,
-        boolean keepBinary
+        boolean read
     ) throws GridCacheEntryRemovedException, GridDistributedLockCancelledException {
         if (tx.local())
             // Null is returned if timeout is negative and there is other lock owner.
@@ -754,7 +643,8 @@ public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion
                 timeout,
                 /*reenter*/false,
                 /*tx*/true,
-                tx.implicitSingle()) != null;
+                tx.implicitSingle(),
+                read) != null;
 
         try {
             addRemote(
@@ -762,7 +652,6 @@ public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion
                 tx.otherNodeId(),
                 tx.threadId(),
                 tx.xidVersion(),
-                tx.timeout(),
                 /*tx*/true,
                 tx.implicitSingle(),
                 tx.ownedVersion(txKey())
@@ -779,7 +668,7 @@ public void orderCompleted(GridCacheVersion baseVer, Collection<GridCacheVersion
     }
 
     /** {@inheritDoc} */
-    @Override public void txUnlock(IgniteInternalTx tx) throws GridCacheEntryRemovedException {
+    @Override public final void txUnlock(IgniteInternalTx tx) throws GridCacheEntryRemovedException {
         removeLock(tx.xidVersion());
     }
 
@@ -799,42 +688,8 @@ protected void checkCallbacks(boolean emptyBefore, boolean emptyAfter) {
         }
     }
 
-    /**
-     * @param prev Previous owner.
-     * @param owner Current owner.
-     * @param val Entry value.
-     */
-    protected void checkOwnerChanged(GridCacheMvccCandidate prev, GridCacheMvccCandidate owner, CacheObject val) {
-        assert !Thread.holdsLock(this);
-
-        if (owner != prev) {
-            cctx.mvcc().callback().onOwnerChanged(this, prev, owner);
-
-            if (owner != null && owner.local())
-                checkThreadChain(owner);
-
-            if (prev != null && cctx.events().isRecordable(EVT_CACHE_OBJECT_UNLOCKED)) {
-                boolean hasVal = hasValue();
-
-                // Event notification.
-                cctx.events().addEvent(partition(), key, prev.nodeId(), prev, EVT_CACHE_OBJECT_UNLOCKED, val, hasVal,
-                    val, hasVal, null, null, null, true);
-            }
-
-            if (owner != null && cctx.events().isRecordable(EVT_CACHE_OBJECT_LOCKED)) {
-                boolean hasVal = hasValue();
-
-                // Event notification.
-                cctx.events().addEvent(partition(), key, owner.nodeId(), owner, EVT_CACHE_OBJECT_LOCKED, val, hasVal,
-                    val, hasVal, null, null, null, true);
-            }
-        }
-    }
-
-    /**
-     * @param owner Starting candidate in the chain.
-     */
-    protected void checkThreadChain(GridCacheMvccCandidate owner) {
+    /** {@inheritDoc} */
+    @Override final protected void checkThreadChain(GridCacheMvccCandidate owner) {
         assert !Thread.holdsLock(this);
 
         assert owner != null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
index b0b0a7e71fa5c..cf4085ba02335 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheLockCandidates;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
@@ -135,7 +136,7 @@ public GridDhtCacheEntry(
      * @return Local candidate by near version.
      * @throws GridCacheEntryRemovedException If removed.
      */
-    @Nullable public synchronized GridCacheMvccCandidate localCandidateByNearVersion(GridCacheVersion nearVer,
+    @Nullable synchronized GridCacheMvccCandidate localCandidateByNearVersion(GridCacheVersion nearVer,
         boolean rmv) throws GridCacheEntryRemovedException {
         checkObsolete();
 
@@ -165,34 +166,33 @@ public GridDhtCacheEntry(
      * @param threadId Owning thread ID.
      * @param ver Lock version.
      * @param serOrder Version for serializable transactions ordering.
-     * @param serReadVer Optional read entry version for optimistic serializable transaction.
      * @param timeout Timeout to acquire lock.
      * @param reenter Reentry flag.
      * @param tx Tx flag.
      * @param implicitSingle Implicit flag.
+     * @param read Read lock flag.
      * @return New candidate.
      * @throws GridCacheEntryRemovedException If entry has been removed.
      * @throws GridDistributedLockCancelledException If lock was cancelled.
      */
-    @Nullable public GridCacheMvccCandidate addDhtLocal(
+    @Nullable GridCacheMvccCandidate addDhtLocal(
         UUID nearNodeId,
         GridCacheVersion nearVer,
         AffinityTopologyVersion topVer,
         long threadId,
         GridCacheVersion ver,
         @Nullable GridCacheVersion serOrder,
-        @Nullable GridCacheVersion serReadVer,
         long timeout,
         boolean reenter,
         boolean tx,
-        boolean implicitSingle)
+        boolean implicitSingle,
+        boolean read)
         throws GridCacheEntryRemovedException, GridDistributedLockCancelledException {
-        assert serReadVer == null || serOrder != null;
         assert !reenter || serOrder == null;
 
         GridCacheMvccCandidate cand;
-        GridCacheMvccCandidate prev;
-        GridCacheMvccCandidate owner;
+        CacheLockCandidates prev;
+        CacheLockCandidates owner;
 
         CacheObject val;
 
@@ -211,7 +211,7 @@ public GridDhtCacheEntry(
                 mvccExtras(mvcc);
             }
 
-            prev = mvcc.anyOwner();
+            prev = mvcc.allOwners();
 
             boolean emptyBefore = mvcc.isEmpty();
 
@@ -226,7 +226,8 @@ public GridDhtCacheEntry(
                 reenter,
                 tx,
                 implicitSingle,
-                /*dht-local*/true
+                /*dht-local*/true,
+                read
             );
 
             if (cand == null)
@@ -234,10 +235,10 @@ public GridDhtCacheEntry(
 
             cand.topologyVersion(topVer);
 
-            owner = mvcc.anyOwner();
+            owner = mvcc.allOwners();
 
             if (owner != null)
-                cand.ownerVersion(owner.version());
+                cand.ownerVersion(owner.candidate(0).version());
 
             boolean emptyAfter = mvcc.isEmpty();
 
@@ -264,7 +265,7 @@ public GridDhtCacheEntry(
         long timeout,
         @Nullable GridCacheVersion serOrder,
         GridCacheVersion serReadVer,
-        boolean keepBinary
+        boolean read
     ) throws GridCacheEntryRemovedException, GridDistributedLockCancelledException {
         if (tx.local()) {
             GridDhtTxLocalAdapter dhtTx = (GridDhtTxLocalAdapter)tx;
@@ -277,11 +278,11 @@ public GridDhtCacheEntry(
                 tx.threadId(),
                 tx.xidVersion(),
                 serOrder,
-                serReadVer,
                 timeout,
                 /*reenter*/false,
                 /*tx*/true,
-                tx.implicitSingle()) != null;
+                tx.implicitSingle(),
+                read) != null;
         }
 
         try {
@@ -290,7 +291,6 @@ public GridDhtCacheEntry(
                 tx.otherNodeId(),
                 tx.threadId(),
                 tx.xidVersion(),
-                tx.timeout(),
                 /*tx*/true,
                 tx.implicit(),
                 null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
index b005b2951fdad..d77933e9f5d47 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
+import org.apache.ignite.internal.processors.cache.CacheLockCandidates;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
@@ -345,25 +346,6 @@ private boolean isInvalidate() {
         return tx == null ? null : tx.isolation();
     }
 
-    /**
-     * @param cached Entry.
-     * @return {@code True} if locked.
-     * @throws GridCacheEntryRemovedException If removed.
-     */
-    private boolean locked(GridCacheEntryEx cached) throws GridCacheEntryRemovedException {
-        return (cached.lockedLocally(lockVer) && filter(cached)); // If filter failed, lock is failed.
-    }
-
-    /**
-     * @param cached Entry.
-     * @param owner Lock owner.
-     * @return {@code True} if locked.
-     */
-    private boolean locked(GridCacheEntryEx cached, GridCacheMvccCandidate owner) {
-        // Reentry-aware check (if filter failed, lock is failed).
-        return owner != null && owner.matches(lockVer, cctx.nodeId(), threadId) && filter(cached);
-    }
-
     /**
      * Adds entry to future.
      *
@@ -392,11 +374,11 @@ private boolean locked(GridCacheEntryEx cached, GridCacheMvccCandidate owner) {
             threadId,
             lockVer,
             null,
-            null,
             timeout,
             /*reenter*/false,
             inTx(),
-            implicitSingle()
+            implicitSingle(),
+            false
         );
 
         if (c == null && timeout < 0) {
@@ -575,10 +557,10 @@ private void readyLocks() {
                     break; // While.
 
                 try {
-                    GridCacheMvccCandidate owner = entry.readyLock(lockVer);
+                    CacheLockCandidates owners = entry.readyLock(lockVer);
 
                     if (timeout < 0) {
-                        if (owner == null || !owner.version().equals(lockVer)) {
+                        if (owners == null || !owners.hasCandidate(lockVer)) {
                             // We did not send any requests yet.
                             onFailed(false);
 
@@ -587,9 +569,9 @@ private void readyLocks() {
                     }
 
                     if (log.isDebugEnabled()) {
-                        if (!locked(entry, owner))
-                            log.debug("Entry is not locked (will keep waiting) [entry=" + entry +
-                                ", fut=" + this + ']');
+                        log.debug("Current lock owners [entry=" + entry +
+                            ", owners=" + owners +
+                            ", fut=" + this + ']');
                     }
 
                     break; // Inner while loop.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index 6b437b11dba23..01bc4e028955f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -273,7 +273,6 @@ protected GridDhtTransactionalCacheAdapter(GridCacheContext<K, V> ctx, GridCache
                             nodeId,
                             req.threadId(),
                             req.version(),
-                            req.timeout(),
                             tx != null,
                             tx != null && tx.implicitSingle(),
                             null
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index 1dbda69f30dbc..a759194453a78 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheInvokeEntry;
+import org.apache.ignite.internal.processors.cache.CacheLockCandidates;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
@@ -629,10 +630,10 @@ private void readyLocks(Iterable<IgniteTxEntry> checkEntries) {
                 try {
                     assert txEntry.explicitVersion() == null || entry.lockedBy(txEntry.explicitVersion());
 
-                    GridCacheMvccCandidate c = entry.readyLock(tx.xidVersion());
+                    CacheLockCandidates owners = entry.readyLock(tx.xidVersion());
 
                     if (log.isDebugEnabled())
-                        log.debug("Current lock owner for entry [owner=" + c + ", entry=" + entry + ']');
+                        log.debug("Current lock owners for entry [owner=" + owners + ", entry=" + entry + ']');
 
                     break; // While.
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index b0eea0191e808..5557d3423e201 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -312,14 +312,14 @@ private boolean implicitTx() {
                     null,
                     threadId,
                     lockVer,
-                    timeout,
                     true,
                     tx.entry(txKey).locked(),
                     inTx(),
                     inTx() && tx.implicitSingle(),
                     false,
                     false,
-                    null);
+                    null,
+                    false);
 
                 cand.topologyVersion(topVer);
             }
@@ -332,14 +332,14 @@ private boolean implicitTx() {
                     null,
                     threadId,
                     lockVer,
-                    timeout,
                     true,
                     false,
                     inTx(),
                     inTx() && tx.implicitSingle(),
                     false,
                     false,
-                    null);
+                    null,
+                    false);
 
                 cand.topologyVersion(topVer);
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index d495f833c683b..30fc213fc63e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheLockCandidates;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
@@ -444,7 +445,8 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
         long timeout,
         boolean reenter,
         boolean tx,
-        boolean implicitSingle) throws GridCacheEntryRemovedException {
+        boolean implicitSingle,
+        boolean read) throws GridCacheEntryRemovedException {
         return addNearLocal(
             null,
             threadId,
@@ -453,7 +455,8 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
             timeout,
             reenter,
             tx,
-            implicitSingle
+            implicitSingle,
+            read
         );
     }
 
@@ -468,10 +471,11 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
      * @param reenter Reentry flag.
      * @param tx Transaction flag.
      * @param implicitSingle Implicit flag.
+     * @param read Read lock flag.
      * @return New candidate.
      * @throws GridCacheEntryRemovedException If entry has been removed.
      */
-    @Nullable public GridCacheMvccCandidate addNearLocal(
+    @Nullable GridCacheMvccCandidate addNearLocal(
         @Nullable UUID dhtNodeId,
         long threadId,
         GridCacheVersion ver,
@@ -479,10 +483,11 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
         long timeout,
         boolean reenter,
         boolean tx,
-        boolean implicitSingle)
+        boolean implicitSingle,
+        boolean read)
         throws GridCacheEntryRemovedException {
-        GridCacheMvccCandidate prev;
-        GridCacheMvccCandidate owner;
+        CacheLockCandidates prev;
+        CacheLockCandidates owner = null;
         GridCacheMvccCandidate cand;
 
         CacheObject val;
@@ -505,7 +510,7 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
             if (c != null)
                 return reenter ? c.reenter() : null;
 
-            prev = mvcc.anyOwner();
+            prev = mvcc.allOwners();
 
             boolean emptyBefore = mvcc.isEmpty();
 
@@ -520,14 +525,12 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
                 dhtNodeId,
                 threadId,
                 ver,
-                timeout,
                 tx,
-                implicitSingle);
+                implicitSingle,
+                read);
 
             cand.topologyVersion(topVer);
 
-            owner = mvcc.anyOwner();
-
             boolean emptyAfter = mvcc.isEmpty();
 
             checkCallbacks(emptyBefore, emptyAfter);
@@ -536,6 +539,8 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
 
             if (emptyAfter)
                 mvccExtras(null);
+            else
+                owner = mvcc.allOwners();
         }
 
         // This call must be outside of synchronization.
@@ -572,8 +577,8 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
      * @return Removed candidate, or <tt>null</tt> if thread still holds the lock.
      */
     @Nullable @Override public GridCacheMvccCandidate removeLock() {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         CacheObject val;
 
@@ -585,7 +590,7 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
             GridCacheMvcc mvcc = mvccExtras();
 
             if (mvcc != null) {
-                prev = mvcc.anyOwner();
+                prev = mvcc.allOwners();
 
                 boolean emptyBefore = mvcc.isEmpty();
 
@@ -605,7 +610,7 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
 
                     mvcc.remove(cand.version());
 
-                    owner = mvcc.anyOwner();
+                    owner = mvcc.allOwners();
                 }
                 else
                     return null;
@@ -630,13 +635,12 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
 
         cctx.mvcc().removeExplicitLock(cand);
 
-        if (prev != null && owner != prev)
-            checkThreadChain(prev);
+        checkThreadChain(cand);
 
         // This call must be outside of synchronization.
         checkOwnerChanged(prev, owner, val);
 
-        return owner != prev ? prev : null;
+        return cand;
     }
 
     /** {@inheritDoc} */
@@ -648,7 +652,7 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
     /**
      * @throws GridCacheEntryRemovedException If entry was removed.
      */
-    public synchronized void reserveEviction() throws GridCacheEntryRemovedException {
+    synchronized void reserveEviction() throws GridCacheEntryRemovedException {
         checkObsolete();
 
         evictReservations++;
@@ -657,7 +661,7 @@ public synchronized void reserveEviction() throws GridCacheEntryRemovedException
     /**
      *
      */
-    public synchronized void releaseEviction() {
+    synchronized void releaseEviction() {
         assert evictReservations > 0 : this;
         assert !obsolete() : this;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index 3d9b6ab322aff..24313791da10b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -325,7 +325,8 @@ private boolean locked(GridCacheEntryEx cached) throws GridCacheEntryRemovedExce
             timeout,
             !inTx(),
             inTx(),
-            implicitSingleTx()
+            implicitSingleTx(),
+            false
         );
 
         if (inTx()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
index cf5d2e2ea36fa..7ac3295e6cf8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
@@ -356,7 +356,6 @@ else if (log.isDebugEnabled())
                                 nodeId,
                                 req.threadId(),
                                 req.version(),
-                                req.timeout(),
                                 tx != null,
                                 tx != null && tx.implicitSingle(),
                                 req.owned(entry.key())
@@ -577,9 +576,7 @@ protected boolean evictNearEntry(GridCacheEntryEx e, GridCacheVersion obsoleteVe
                                     if (!primary.isLocal()) {
                                         assert req != null;
 
-                                        req.addKey(
-                                            entry.key(),
-                                            ctx);
+                                        req.addKey(entry.key(), ctx);
                                     }
                                     else
                                         locKeys.add(cacheKey);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java
index 10fa116452d1d..bc61333514ff8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.local;
 
+import org.apache.ignite.internal.processors.cache.CacheLockCandidates;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
@@ -29,9 +30,6 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
 
-import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_LOCKED;
-import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_UNLOCKED;
-
 /**
  * Cache entry for local caches.
  */
@@ -46,7 +44,7 @@ public class GridLocalCacheEntry extends GridCacheMapEntry {
      * @param hash Key hash value.
      * @param val Entry value.
      */
-    public GridLocalCacheEntry(
+    GridLocalCacheEntry(
         GridCacheContext ctx,
         KeyCacheObject key,
         int hash,
@@ -71,10 +69,11 @@ public GridLocalCacheEntry(
      * @param reenter Reentry flag.
      * @param tx Transaction flag.
      * @param implicitSingle Implicit transaction flag.
+     * @param read Read lock flag.
      * @return New candidate.
      * @throws GridCacheEntryRemovedException If entry has been removed.
      */
-    @Nullable public GridCacheMvccCandidate addLocal(
+    @Nullable GridCacheMvccCandidate addLocal(
         long threadId,
         GridCacheVersion ver,
         @Nullable GridCacheVersion serOrder,
@@ -83,14 +82,14 @@ public GridLocalCacheEntry(
         boolean reenter,
         boolean tx,
         boolean implicitSingle,
-        boolean keepBinary
+        boolean read
     ) throws GridCacheEntryRemovedException {
-        GridCacheMvccCandidate prev;
-        GridCacheMvccCandidate cand;
-        GridCacheMvccCandidate owner;
+        assert serReadVer == null || serOrder != null;
 
         CacheObject val;
-        boolean hasVal;
+        GridCacheMvccCandidate cand;
+        CacheLockCandidates prev;
+        CacheLockCandidates owner = null;
 
         synchronized (this) {
             checkObsolete();
@@ -108,7 +107,7 @@ public GridLocalCacheEntry(
                 mvccExtras(mvcc);
             }
 
-            prev = mvcc.localOwner();
+            prev = mvcc.localOwners();
 
             cand = mvcc.addLocal(
                 this,
@@ -121,59 +120,50 @@ public GridLocalCacheEntry(
                 reenter,
                 tx,
                 implicitSingle,
-                /*dht-local*/false
+                /*dht-local*/false,
+                read
             );
 
-            owner = mvcc.localOwner();
-
-            val = this.val;
-
-            hasVal = hasValueUnlocked();
-
             if (mvcc.isEmpty())
                 mvccExtras(null);
-        }
-
-        if (cand != null) {
-            if (!cand.reentry())
-                cctx.mvcc().addNext(cctx, cand);
+            else
+                owner = mvcc.localOwners();
 
-            // Event notification.
-            if (cctx.events().isRecordable(EVT_CACHE_OBJECT_LOCKED))
-                cctx.events().addEvent(partition(), key, cand.nodeId(), cand, EVT_CACHE_OBJECT_LOCKED, val, hasVal,
-                    val, hasVal, null, null, null, keepBinary);
+            val = this.val;
         }
 
-        checkOwnerChanged(prev, owner);
+        if (cand != null && !cand.reentry())
+            cctx.mvcc().addNext(cctx, cand);
+
+        checkOwnerChanged(prev, owner, val);
 
         return cand;
     }
 
     /**
-     *
      * @param cand Candidate.
-     * @return Current owner.
      */
-    @Nullable public GridCacheMvccCandidate readyLocal(GridCacheMvccCandidate cand) {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+    void readyLocal(GridCacheMvccCandidate cand) {
+        CacheObject val;
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         synchronized (this) {
             GridCacheMvcc mvcc = mvccExtras();
 
             if (mvcc != null) {
-                prev = mvcc.localOwner();
+                prev = mvcc.localOwners();
 
                 owner = mvcc.readyLocal(cand);
 
                 if (mvcc.isEmpty())
                     mvccExtras(null);
             }
-        }
 
-        checkOwnerChanged(prev, owner);
+            val = this.val;
+        }
 
-        return owner;
+        checkOwnerChanged(prev, owner, val);
     }
 
     /** {@inheritDoc} */
@@ -181,7 +171,7 @@ public GridLocalCacheEntry(
         long timeout,
         @Nullable GridCacheVersion serOrder,
         GridCacheVersion serReadVer,
-        boolean keepBinary)
+        boolean read)
         throws GridCacheEntryRemovedException {
         GridCacheMvccCandidate cand = addLocal(
             tx.threadId(),
@@ -192,7 +182,7 @@ public GridLocalCacheEntry(
             /*reenter*/false,
             /*tx*/true,
             tx.implicitSingle(),
-            keepBinary
+            read
         );
 
         if (cand != null) {
@@ -206,50 +196,32 @@ public GridLocalCacheEntry(
 
     /**
      * Rechecks if lock should be reassigned.
-     *
-     * @return Current owner.
      */
-    @Nullable public GridCacheMvccCandidate recheck() {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+    public void recheck() {
+        CacheObject val;
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         synchronized (this) {
             GridCacheMvcc mvcc = mvccExtras();
 
             if (mvcc != null) {
-                prev = mvcc.localOwner();
+                prev = mvcc.allOwners();
 
                 owner = mvcc.recheck();
 
                 if (mvcc.isEmpty())
                     mvccExtras(null);
             }
-        }
-
-        checkOwnerChanged(prev, owner);
-
-        return owner;
-    }
 
-    /**
-     * @param prev Previous owner.
-     * @param owner Current owner.
-     */
-    private void checkOwnerChanged(GridCacheMvccCandidate prev, GridCacheMvccCandidate owner) {
-        assert !Thread.holdsLock(this);
-
-        if (owner != prev) {
-            cctx.mvcc().callback().onOwnerChanged(this, prev, owner);
-
-            if (owner != null)
-                checkThreadChain(owner);
+            val = this.val;
         }
+
+        checkOwnerChanged(prev, owner, val);
     }
 
-    /**
-     * @param owner Starting candidate in the chain.
-     */
-    private void checkThreadChain(GridCacheMvccCandidate owner) {
+    /** {@inheritDoc} */
+    @Override protected void checkThreadChain(GridCacheMvccCandidate owner) {
         assert !Thread.holdsLock(this);
 
         assert owner != null;
@@ -298,62 +270,50 @@ void releaseLocal() {
      *
      * @param threadId Thread ID.
      */
-    void releaseLocal(long threadId) {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
-
+    private void releaseLocal(long threadId) {
         CacheObject val;
-        boolean hasVal;
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         synchronized (this) {
             GridCacheMvcc mvcc = mvccExtras();
 
             if (mvcc != null) {
-                prev = mvcc.localOwner();
+                prev = mvcc.localOwners();
 
-                owner = mvcc.releaseLocal(threadId);
+                mvcc.releaseLocal(threadId);
 
                 if (mvcc.isEmpty())
                     mvccExtras(null);
+                else
+                    owner = mvcc.allOwners();
             }
 
             val = this.val;
-            hasVal = hasValueUnlocked();
         }
 
-        if (prev != null && owner != prev) {
-            checkThreadChain(prev);
+        if (prev != null) {
+            for (int i = 0; i < prev.size(); i++) {
+                GridCacheMvccCandidate cand = prev.candidate(i);
 
-            // Event notification.
-            if (cctx.events().isRecordable(EVT_CACHE_OBJECT_UNLOCKED))
-                cctx.events().addEvent(partition(), key, prev.nodeId(), prev, EVT_CACHE_OBJECT_UNLOCKED, val, hasVal,
-                    val, hasVal, null, null, null, true);
-        }
+                boolean unlocked = owner == null || !owner.hasCandidate(cand.version());
 
-        checkOwnerChanged(prev, owner);
-    }
+                if (unlocked)
+                    checkThreadChain(cand);
+            }
+        }
 
-    /**
-     * Removes candidate regardless if it is owner or not.
-     *
-     * @param cand Candidate to remove.
-     * @throws GridCacheEntryRemovedException If the entry was removed by version other
-     *      than one passed in.
-     */
-    void removeLock(GridCacheMvccCandidate cand) throws GridCacheEntryRemovedException {
-        removeLock(cand.version());
+        checkOwnerChanged(prev, owner, val);
     }
 
     /** {@inheritDoc} */
     @Override public boolean removeLock(GridCacheVersion ver) throws GridCacheEntryRemovedException {
-        GridCacheMvccCandidate prev = null;
-        GridCacheMvccCandidate owner = null;
+        CacheObject val;
+        CacheLockCandidates prev = null;
+        CacheLockCandidates owner = null;
 
         GridCacheMvccCandidate doomed;
 
-        CacheObject val;
-        boolean hasVal;
-
         synchronized (this) {
             GridCacheVersion obsoleteVer = obsoleteVersionExtras();
 
@@ -365,28 +325,23 @@ void removeLock(GridCacheMvccCandidate cand) throws GridCacheEntryRemovedExcepti
             doomed = mvcc == null ? null : mvcc.candidate(ver);
 
             if (doomed != null) {
-                prev = mvcc.localOwner();
+                prev = mvcc.allOwners();
 
-                owner = mvcc.remove(ver);
+                mvcc.remove(ver);
 
                 if (mvcc.isEmpty())
                     mvccExtras(null);
+                else
+                    owner = mvcc.allOwners();
             }
 
             val = this.val;
-            hasVal = hasValueUnlocked();
         }
 
-        if (doomed != null) {
+        if (doomed != null)
             checkThreadChain(doomed);
 
-            // Event notification.
-            if (cctx.events().isRecordable(EVT_CACHE_OBJECT_UNLOCKED))
-                cctx.events().addEvent(partition(), key, doomed.nodeId(), doomed, EVT_CACHE_OBJECT_UNLOCKED,
-                    val, hasVal, val, hasVal, null, null, null, true);
-        }
-
-        checkOwnerChanged(prev, owner);
+        checkOwnerChanged(prev, owner, val);
 
         return doomed != null;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java
index c5bd71a61f044..8e224c8b66111 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java
@@ -231,7 +231,7 @@ private boolean locked(GridCacheEntryEx cached) throws GridCacheEntryRemovedExce
             !inTx(),
             inTx(),
             implicitSingle(),
-            true
+            false
         );
 
         entries.add(entry);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index c72d7f781d938..2c02f967d00be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -53,7 +53,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheReturnCompletableWrapper;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
 import org.apache.ignite.internal.processors.cache.GridDeferredAckMessageSender;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheMappedVersion;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxFinishSync;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryFuture;
@@ -105,6 +104,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.GridTopic.TOPIC_TX;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.READ;
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled;
 import static org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx.FinalizationStatus.RECOVERY_FINISH;
 import static org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx.FinalizationStatus.USER_FINISH;
@@ -1615,7 +1615,9 @@ private boolean lockMultiple(IgniteInternalTx tx, Iterable<IgniteTxEntry> entrie
 
                     assert serReadVer == null || (tx.optimistic() && tx.serializable()) : txEntry1;
 
-                    if (!entry1.tmLock(tx, timeout, serOrder, serReadVer, txEntry1.keepBinary())) {
+                    boolean read = serOrder != null && txEntry1.op() == READ;
+
+                    if (!entry1.tmLock(tx, timeout, serOrder, serReadVer, read)) {
                         // Unlock locks locked so far.
                         for (IgniteTxEntry txEntry2 : entries) {
                             if (txEntry2 == txEntry1)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheSerializableTransactionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheSerializableTransactionsTest.java
index 6a73f79dfc303..e9be108c5e8a0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheSerializableTransactionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheSerializableTransactionsTest.java
@@ -590,7 +590,7 @@ public void testTxCommitReadOnlyGetEntries() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testTxCommitReadOnlyGetAll(boolean needVer) throws Exception {
+    private void testTxCommitReadOnlyGetAll(boolean needVer) throws Exception {
         Ignite ignite0 = ignite(0);
 
         final IgniteTransactions txs = ignite0.transactions();
@@ -924,6 +924,71 @@ private void txConflictReadWrite(boolean noVal, boolean rmv, boolean needVer) th
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxConflictReadWrite3() throws Exception {
+        Ignite ignite0 = ignite(0);
+
+        final IgniteTransactions txs = ignite0.transactions();
+
+        for (CacheConfiguration<Integer, Integer> ccfg : cacheConfigurations()) {
+            logCacheInfo(ccfg);
+
+            IgniteCache<Integer, Integer> cache = ignite0.createCache(ccfg);
+
+            List<Integer> readKeys = new ArrayList<>();
+            List<Integer> writeKeys = new ArrayList<>();
+
+            readKeys.add(primaryKey(cache));
+            writeKeys.add(primaryKeys(cache, 1, 1000_0000).get(0));
+
+            if (ccfg.getBackups() > 0) {
+                readKeys.add(backupKey(cache));
+                writeKeys.add(backupKeys(cache, 1, 1000_0000).get(0));
+            }
+
+            if (ccfg.getCacheMode() == PARTITIONED) {
+                readKeys.add(nearKey(cache));
+                writeKeys.add(nearKeys(cache, 1, 1000_0000).get(0));
+            }
+
+            try {
+                for (Integer readKey : readKeys) {
+                    for (Integer writeKey : writeKeys) {
+                        try {
+                            try (Transaction tx = txs.txStart(OPTIMISTIC, SERIALIZABLE)) {
+                                cache.get(readKey);
+
+                                cache.put(writeKey, writeKey);
+
+                                updateKey(cache, readKey, 0);
+
+                                tx.commit();
+                            }
+
+                            fail();
+                        }
+                        catch (TransactionOptimisticException e) {
+                            // Expected exception.
+                        }
+
+                        try (Transaction tx = txs.txStart(OPTIMISTIC, SERIALIZABLE)) {
+                            cache.get(readKey);
+
+                            cache.put(writeKey, writeKey);
+
+                            tx.commit();
+                        }
+                    }
+                }
+            }
+            finally {
+                destroyCache(ccfg.getName());
+            }
+        }
+    }
+
     /**
      * @throws Exception If failed
      */
@@ -2443,6 +2508,529 @@ private void rollbackIfLockedPartialLock(boolean locKey) throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoReadLockConflict() throws Exception {
+        checkNoReadLockConflict(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoReadLockConflictGetEntry() throws Exception {
+        checkNoReadLockConflict(true);
+    }
+
+    /**
+     * @param entry If {@code true} then uses 'getEntry' to read value, otherwise uses 'get'.
+     * @throws Exception If failed.
+     */
+    private void checkNoReadLockConflict(final boolean entry) throws Exception {
+        Ignite ignite0 = ignite(0);
+
+        for (CacheConfiguration<Integer, Integer> ccfg : cacheConfigurations()) {
+            logCacheInfo(ccfg);
+
+            final AtomicInteger putKey = new AtomicInteger(1_000_000);
+
+            ignite0.createCache(ccfg);
+
+            CacheConfiguration<Integer, Integer> readCacheCcfg = new CacheConfiguration<>(ccfg);
+
+            readCacheCcfg.setName(ccfg.getName() + "-read");
+
+            ignite0.createCache(readCacheCcfg);
+
+            try {
+                checkNoReadLockConflict(ignite(0), ccfg.getName(), ccfg.getName(), entry, putKey);
+
+                checkNoReadLockConflict(ignite(1), ccfg.getName(), ccfg.getName(), entry, putKey);
+
+                checkNoReadLockConflict(ignite(SRVS), ccfg.getName(), ccfg.getName(), entry, putKey);
+
+                checkNoReadLockConflict(ignite(0), readCacheCcfg.getName(), ccfg.getName(), entry, putKey);
+
+                checkNoReadLockConflict(ignite(1), readCacheCcfg.getName(), ccfg.getName(), entry, putKey);
+
+                checkNoReadLockConflict(ignite(SRVS), readCacheCcfg.getName(), ccfg.getName(), entry, putKey);
+            }
+            finally {
+                destroyCache(ccfg.getName());
+
+                destroyCache(readCacheCcfg.getName());
+            }
+        }
+    }
+
+    /**
+     * @param ignite Node.
+     * @param readCacheName Cache name for get.
+     * @param writeCacheName Cache name for put.
+     * @param entry If {@code true} then uses 'getEntry' to read value, otherwise uses 'get'.
+     * @param putKey Write key counter.
+     * @throws Exception If failed.
+     */
+    private void checkNoReadLockConflict(final Ignite ignite,
+        String readCacheName,
+        String writeCacheName,
+        final boolean entry,
+        final AtomicInteger putKey) throws Exception
+    {
+        final int THREADS = 64;
+
+        final IgniteCache<Integer, Integer> readCache = ignite.cache(readCacheName);
+        final IgniteCache<Integer, Integer> writeCache = ignite.cache(writeCacheName);
+
+        List<Integer> readKeys = testKeys(readCache);
+
+        for (final Integer readKey : readKeys) {
+            final CyclicBarrier barrier = new CyclicBarrier(THREADS);
+
+            readCache.put(readKey, Integer.MIN_VALUE);
+
+            GridTestUtils.runMultiThreaded(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) {
+                        if (entry)
+                            readCache.get(readKey);
+                        else
+                            readCache.getEntry(readKey);
+
+                        barrier.await();
+
+                        writeCache.put(putKey.incrementAndGet(), 0);
+
+                        tx.commit();
+                    }
+
+                    return null;
+                }
+            }, THREADS, "test-thread");
+
+            assertEquals((Integer)Integer.MIN_VALUE, readCache.get(readKey));
+
+            readCache.put(readKey, readKey);
+
+            assertEquals(readKey, readCache.get(readKey));
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoReadLockConflictMultiNode() throws Exception {
+        Ignite ignite0 = ignite(0);
+
+        for (final CacheConfiguration<Integer, Integer> ccfg : cacheConfigurations()) {
+            logCacheInfo(ccfg);
+
+            final AtomicInteger putKey = new AtomicInteger(1_000_000);
+
+            ignite0.createCache(ccfg);
+
+            try {
+                final int THREADS = 64;
+
+                IgniteCache<Integer, Integer> cache0 = ignite0.cache(ccfg.getName());
+
+                List<Integer> readKeys = testKeys(cache0);
+
+                for (final Integer readKey : readKeys) {
+                    final CyclicBarrier barrier = new CyclicBarrier(THREADS);
+
+                    cache0.put(readKey, Integer.MIN_VALUE);
+
+                    final AtomicInteger idx = new AtomicInteger();
+
+                    GridTestUtils.runMultiThreaded(new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            Ignite ignite = ignite(idx.incrementAndGet() % (CLIENTS + SRVS));
+
+                            IgniteCache<Integer, Integer> cache = ignite.cache(ccfg.getName());
+
+                            try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) {
+                                cache.get(readKey);
+
+                                barrier.await();
+
+                                cache.put(putKey.incrementAndGet(), 0);
+
+                                tx.commit();
+                            }
+
+                            return null;
+                        }
+                    }, THREADS, "test-thread");
+
+                    assertEquals((Integer)Integer.MIN_VALUE, cache0.get(readKey));
+
+                    cache0.put(readKey, readKey);
+
+                    assertEquals(readKey, cache0.get(readKey));
+                }
+            }
+            finally {
+                destroyCache(ccfg.getName());
+            }
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("UnnecessaryLocalVariable")
+    public void testReadLockPessimisticTxConflict() throws Exception {
+        Ignite ignite0 = ignite(0);
+
+        for (CacheConfiguration<Integer, Integer> ccfg : cacheConfigurations()) {
+            logCacheInfo(ccfg);
+
+            ignite0.createCache(ccfg);
+
+            try {
+                Ignite ignite = ignite0;
+
+                IgniteCache<Integer, Integer> cache = ignite.cache(ccfg.getName());
+
+                Integer writeKey = Integer.MAX_VALUE;
+
+                List<Integer> readKeys = testKeys(cache);
+
+                for (Integer readKey : readKeys) {
+                    CountDownLatch latch = new CountDownLatch(1);
+
+                    IgniteInternalFuture<?> fut = lockKey(latch, cache, readKey);
+
+                    try {
+                        // No conflict for write, conflict with pessimistic tx for read.
+                        try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) {
+                            cache.put(writeKey, writeKey);
+
+                            cache.get(readKey);
+
+                            tx.commit();
+                        }
+
+                        fail();
+                    }
+                    catch (TransactionOptimisticException e) {
+                        log.info("Expected exception: " + e);
+                    }
+                    finally {
+                        latch.countDown();
+                    }
+
+                    fut.get();
+                }
+            }
+            finally {
+                destroyCache(ccfg.getName());
+            }
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("UnnecessaryLocalVariable")
+    public void testReadWriteTxConflict() throws Exception {
+        Ignite ignite0 = ignite(0);
+
+        for (CacheConfiguration<Integer, Integer> ccfg : cacheConfigurations()) {
+            logCacheInfo(ccfg);
+
+            ignite0.createCache(ccfg);
+
+            try {
+                Ignite ignite = ignite0;
+
+                IgniteCache<Integer, Integer> cache = ignite.cache(ccfg.getName());
+
+                Integer writeKey = Integer.MAX_VALUE;
+
+                List<Integer> readKeys = testKeys(cache);
+
+                for (Integer readKey : readKeys) {
+                    try {
+                        // No conflict for read, conflict for write.
+                        try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) {
+                            cache.getAndPut(writeKey, writeKey);
+
+                            cache.get(readKey);
+
+                            updateKey(cache, writeKey, writeKey + readKey);
+
+                            tx.commit();
+                        }
+
+                        fail();
+                    }
+                    catch (TransactionOptimisticException e) {
+                        log.info("Expected exception: " + e);
+                    }
+
+                    assertEquals((Integer)(writeKey + readKey), cache.get(writeKey));
+                    assertNull(cache.get(readKey));
+
+                    cache.put(readKey, readKey);
+
+                    assertEquals(readKey, cache.get(readKey));
+                }
+            }
+            finally {
+                destroyCache(ccfg.getName());
+            }
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReadWriteTransactionsNoDeadlock() throws Exception {
+        checkReadWriteTransactionsNoDeadlock(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReadWriteTransactionsNoDeadlockMultinode() throws Exception {
+        checkReadWriteTransactionsNoDeadlock(true);
+    }
+
+    /**
+     * @param multiNode Multi-node test flag.
+     * @throws Exception If failed.
+     */
+    private void checkReadWriteTransactionsNoDeadlock(final boolean multiNode) throws Exception {
+        final Ignite ignite0 = ignite(0);
+
+        for (final CacheConfiguration<Integer, Integer> ccfg : cacheConfigurations()) {
+            logCacheInfo(ccfg);
+
+            ignite0.createCache(ccfg);
+
+            try {
+                final long stopTime = U.currentTimeMillis() + 10_000;
+
+                final AtomicInteger idx = new AtomicInteger();
+
+                GridTestUtils.runMultiThreaded(new Callable<Void>() {
+                    @Override public Void call() throws Exception {
+                        Ignite ignite = multiNode ? ignite(idx.incrementAndGet() % (SRVS + CLIENTS)) : ignite0;
+
+                        IgniteCache<Integer, Integer> cache = ignite.cache(ccfg.getName());
+
+                        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                        while (U.currentTimeMillis() < stopTime) {
+                            try {
+                                try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) {
+                                    for (int i = 0; i < 10; i++) {
+                                        Integer key = rnd.nextInt(30);
+
+                                        if (rnd.nextBoolean())
+                                            cache.get(key);
+                                        else
+                                            cache.put(key, key);
+                                    }
+
+                                    tx.commit();
+                                }
+                            }
+                            catch (TransactionOptimisticException ignore) {
+                                // No-op.
+                            }
+                        }
+
+                        return null;
+                    }
+                }, 32, "test-thread");
+            }
+            finally {
+                destroyCache(ccfg.getName());
+            }
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReadWriteAccountTx() throws Exception {
+        final CacheConfiguration<Integer, Integer> ccfg = cacheConfiguration(PARTITIONED,
+            FULL_SYNC,
+            1,
+            false,
+            false);
+
+        ignite(0).createCache(ccfg);
+
+        try {
+            final int ACCOUNTS = 50;
+            final int VAL_PER_ACCOUNT = 1000;
+
+            IgniteCache<Integer, Account> cache0 = ignite(0).cache(ccfg.getName());
+
+            final Set<Integer> keys = new HashSet<>();
+
+            for (int i = 0; i < ACCOUNTS; i++) {
+                cache0.put(i, new Account(VAL_PER_ACCOUNT));
+
+                keys.add(i);
+            }
+
+            final List<Ignite> clients = clients();
+
+            final AtomicBoolean stop = new AtomicBoolean();
+
+            final AtomicInteger idx = new AtomicInteger();
+
+            IgniteInternalFuture<?> readFut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    try {
+                        int threadIdx = idx.getAndIncrement();
+
+                        int nodeIdx = threadIdx % (SRVS + CLIENTS);
+
+                        Ignite node = ignite(nodeIdx);
+
+                        IgniteCache<Integer, Account> cache = node.cache(ccfg.getName());
+
+                        IgniteTransactions txs = node.transactions();
+
+                        Integer putKey = ACCOUNTS + threadIdx;
+
+                        while (!stop.get()) {
+                            int sum;
+
+                            while (true) {
+                                sum = 0;
+
+                                try (Transaction tx = txs.txStart(OPTIMISTIC, SERIALIZABLE)) {
+                                    Map<Integer, Account> data = cache.getAll(keys);
+
+                                    for (int i = 0; i < ACCOUNTS; i++) {
+                                        Account account = data.get(i);
+
+                                        assertNotNull(account);
+
+                                        sum += account.value();
+                                    }
+
+                                    cache.put(putKey, new Account(sum));
+
+                                    tx.commit();
+                                }
+                                catch (TransactionOptimisticException e) {
+                                    continue;
+                                }
+
+                                break;
+                            }
+
+                            assertEquals(ACCOUNTS * VAL_PER_ACCOUNT, sum);
+                        }
+
+                        return null;
+                    }
+                    catch (Throwable e) {
+                        stop.set(true);
+
+                        log.error("Unexpected error: " + e);
+
+                        throw e;
+                    }
+                }
+            }, (SRVS + CLIENTS) * 2, "update-thread");
+
+            IgniteInternalFuture<?> updateFut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    try {
+                        int nodeIdx = idx.getAndIncrement() % clients.size();
+
+                        Ignite node = clients.get(nodeIdx);
+
+                        IgniteCache<Integer, Account> cache = node.cache(ccfg.getName());
+
+                        IgniteTransactions txs = node.transactions();
+
+                        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                        while (!stop.get()) {
+                            int id1 = rnd.nextInt(ACCOUNTS);
+
+                            int id2 = rnd.nextInt(ACCOUNTS);
+
+                            while (id2 == id1)
+                                id2 = rnd.nextInt(ACCOUNTS);
+
+                            while (true) {
+                                try (Transaction tx = txs.txStart(OPTIMISTIC, SERIALIZABLE)) {
+                                    Account a1 = cache.get(id1);
+                                    Account a2 = cache.get(id2);
+
+                                    assertNotNull(a1);
+                                    assertNotNull(a2);
+
+                                    if (a1.value() > 0) {
+                                        a1 = new Account(a1.value() - 1);
+                                        a2 = new Account(a2.value() + 1);
+                                    }
+
+                                    cache.put(id1, a1);
+                                    cache.put(id2, a2);
+
+                                    tx.commit();
+                                }
+                                catch (TransactionOptimisticException e) {
+                                    continue;
+                                }
+
+                                break;
+                            }
+                        }
+
+                        return null;
+                    }
+                    catch (Throwable e) {
+                        stop.set(true);
+
+                        log.error("Unexpected error: " + e);
+
+                        throw e;
+                    }
+                }
+            }, 2, "update-thread");
+
+            try {
+                U.sleep(15_000);
+            }
+            finally {
+                stop.set(true);
+            }
+
+            readFut.get();
+            updateFut.get();
+            int sum = 0;
+
+            for (int i = 0; i < ACCOUNTS; i++) {
+                Account a = cache0.get(i);
+
+                assertNotNull(a);
+                assertTrue(a.value() >= 0);
+
+                log.info("Account: " + a.value());
+
+                sum += a.value();
+            }
+
+            assertEquals(ACCOUNTS * VAL_PER_ACCOUNT, sum);
+        }
+        finally {
+            ignite(0).destroyCache(ccfg.getName());
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -4189,13 +4777,17 @@ private List<Integer> testKeys(IgniteCache<Integer, Integer> cache) throws Excep
 
         List<Integer> keys = new ArrayList<>();
 
-        if (ccfg.getCacheMode() == PARTITIONED)
-            keys.add(nearKey(cache));
+        if (!cache.unwrap(Ignite.class).configuration().isClientMode()) {
+            if (ccfg.getCacheMode() == PARTITIONED)
+                keys.add(nearKey(cache));
 
-        keys.add(primaryKey(cache));
+            keys.add(primaryKey(cache));
 
-        if (ccfg.getBackups() != 0)
-            keys.add(backupKey(cache));
+            if (ccfg.getBackups() != 0)
+                keys.add(backupKey(cache));
+        }
+        else
+            keys.add(nearKey(cache));
 
         return keys;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccFlagsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccFlagsTest.java
index 234f3627990f6..ff2d62dbf591b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccFlagsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccFlagsTest.java
@@ -82,14 +82,14 @@ public void testAllTrueFlags() {
             ver,
             1,
             ver,
-            0,
             true,
             true,
             true,
             true,
             true,
             true,
-            null
+            null,
+            false
         );
 
         c.setOwner();
@@ -123,14 +123,14 @@ public void testAllFalseFlags() {
             ver,
             1,
             ver,
-            0,
             false,
             false,
             false,
             false,
             false,
             false,
-            null
+            null,
+            false
         );
 
         short flags = c.flags();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccPartitionedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccPartitionedSelfTest.java
index 1b97663941b00..11a91b5fe8eb8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccPartitionedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccPartitionedSelfTest.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
@@ -37,6 +38,9 @@
  * Test cases for multi-threaded tests in partitioned cache.
  */
 public class GridCacheMvccPartitionedSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final UUID nodeId = UUID.randomUUID();
+
     /** Grid. */
     private IgniteKernal grid;
 
@@ -94,8 +98,8 @@ public void testNearLocalsWithPending() {
         GridCacheVersion ver1 = version(1);
         GridCacheVersion ver2 = version(2);
 
-        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, 0, false, true);
-        GridCacheMvccCandidate c2 = entry.addNearLocal(node1, 1, ver2, 0, true);
+        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, false, true);
+        GridCacheMvccCandidate c2 = entry.addNearLocal(node1, 1, ver2, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
         Collection<GridCacheMvccCandidate> nearLocCands = entry.localCandidates();
@@ -128,8 +132,8 @@ public void testNearLocalsWithCommitted() {
         GridCacheVersion ver1 = version(1);
         GridCacheVersion ver2 = version(2);
 
-        GridCacheMvccCandidate c1 = entry.addNearLocal(node1, 1, ver1, 0, true);
-        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, 0, false, true);
+        GridCacheMvccCandidate c1 = entry.addNearLocal(node1, 1, ver1, true);
+        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, false, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
         Collection<GridCacheMvccCandidate> nearLocCands = entry.localCandidates();
@@ -161,8 +165,8 @@ public void testNearLocalsWithRolledback() {
         GridCacheVersion ver1 = version(1);
         GridCacheVersion ver2 = version(2);
 
-        GridCacheMvccCandidate c1 = entry.addNearLocal(node1, 1, ver1, 0, true);
-        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, 0, false, true);
+        GridCacheMvccCandidate c1 = entry.addNearLocal(node1, 1, ver1, true);
+        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, false, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
         Collection<GridCacheMvccCandidate> nearLocCands = entry.localCandidates();
@@ -194,8 +198,8 @@ public void testNearLocals() {
         GridCacheVersion ver1 = version(1);
         GridCacheVersion ver2 = version(2);
 
-        GridCacheMvccCandidate c1 = entry.addNearLocal(node1, 1, ver1, 0, true);
-        GridCacheMvccCandidate c2 = entry.addNearLocal(node1, 1, ver2, 0, true);
+        GridCacheMvccCandidate c1 = entry.addNearLocal(node1, 1, ver1, true);
+        GridCacheMvccCandidate c2 = entry.addNearLocal(node1, 1, ver2, true);
 
         entry.readyNearLocal(ver2, ver2,  empty(), empty(), empty());
 
@@ -224,8 +228,8 @@ public void testNearLocalsWithOwned() {
         GridCacheVersion ver1 = version(1);
         GridCacheVersion ver2 = version(2);
 
-        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, 0, false, true);
-        GridCacheMvccCandidate c2 = entry.addNearLocal(node1, 1, ver2, 0, true);
+        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, false, true);
+        GridCacheMvccCandidate c2 = entry.addNearLocal(node1, 1, ver2, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
         Collection<GridCacheMvccCandidate> nearLocCands = entry.localCandidates();
@@ -263,11 +267,11 @@ public void testAddPendingRemote0() throws Exception {
         GridCacheVersion ver0 = version(0);
         GridCacheVersion ver1 = version(1);
 
-        entry.addNearLocal(node1, 1, ver1, 0, true);
+        entry.addNearLocal(node1, 1, ver1, true);
 
         entry.readyNearLocal(ver1, ver1, empty(), empty(), Collections.singletonList(ver0));
 
-        entry.addRemote(node1, 1, ver0, 0, false, true);
+        entry.addRemote(node1, 1, ver0, false, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
         Collection<GridCacheMvccCandidate> nearLocCands = entry.localCandidates();
@@ -297,13 +301,13 @@ public void testAddPendingRemote1() throws Exception {
         GridCacheVersion ver2 = version(2);
         GridCacheVersion ver3 = version(3);
 
-        GridCacheMvccCandidate c3 = entry.addNearLocal(node1, 1, ver3, 0, true);
+        GridCacheMvccCandidate c3 = entry.addNearLocal(node1, 1, ver3, true);
 
         entry.readyNearLocal(ver3, ver3, empty(), empty(), Arrays.asList(ver0, ver1, ver2));
 
-        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, 0, false, true);
-        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, 0, false, true);
-        GridCacheMvccCandidate c0 = entry.addRemote(node1, 1, ver0, 0, false, true);
+        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, false, true);
+        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, false, true);
+        GridCacheMvccCandidate c0 = entry.addRemote(node1, 1, ver0, false, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
 
@@ -340,13 +344,13 @@ public void testAddPendingRemote2() throws Exception {
         GridCacheVersion ver2 = version(2);
         GridCacheVersion ver3 = version(3);
 
-        GridCacheMvccCandidate c3 = entry.addNearLocal(node1, 1, ver3, 0, true);
-        entry.addNearLocal(node1, 1, ver2, 0, true);
+        GridCacheMvccCandidate c3 = entry.addNearLocal(node1, 1, ver3, true);
+        entry.addNearLocal(node1, 1, ver2, true);
 
         entry.readyNearLocal(ver3, ver3, empty(), empty(), Arrays.asList(ver0, ver1, ver2));
 
-        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, 0, false, true);
-        GridCacheMvccCandidate c0 = entry.addRemote(node1, 1, ver0, 0, false, true);
+        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, false, true);
+        GridCacheMvccCandidate c0 = entry.addRemote(node1, 1, ver0, false, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
 
@@ -386,12 +390,12 @@ public void testSalvageRemote() {
         GridCacheVersion ver5 = version(5);
         GridCacheVersion ver6 = version(6);
 
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addRemote(node1, 1, ver2, 0, false, true);
-        GridCacheMvccCandidate c3 = entry.addNearLocal(node1, 1, ver3, 0, true);
-        GridCacheMvccCandidate c4 = entry.addRemote(node1, 1, ver4, 0, false, true);
-        entry.addRemote(node1, 1, ver5, 0, false, true);
-        entry.addRemote(node1, 1, ver6, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addRemote(node1, 1, ver2, false, true);
+        GridCacheMvccCandidate c3 = entry.addNearLocal(node1, 1, ver3, true);
+        GridCacheMvccCandidate c4 = entry.addRemote(node1, 1, ver4, false, true);
+        entry.addRemote(node1, 1, ver5, false, true);
+        entry.addRemote(node1, 1, ver6, false, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
 
@@ -442,9 +446,9 @@ public void testNearRemoteConsistentOrdering0() throws Exception {
         GridCacheVersion ver2 = version(20);
         GridCacheVersion ver3 = version(30);
 
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addNearLocal(node1, 1, nearVer2, 0, true);
-        entry.addRemote(node1, 1, ver3, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addNearLocal(node1, 1, nearVer2, true);
+        entry.addRemote(node1, 1, ver3, false, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
         Collection<GridCacheMvccCandidate> nearLocCands = entry.localCandidates();
@@ -480,9 +484,9 @@ public void testNearRemoteConsistentOrdering1() throws Exception {
         GridCacheVersion ver2 = version(20);
         GridCacheVersion ver3 = version(30);
 
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addNearLocal(node1, 1, nearVer2, 0, true);
-        entry.addRemote(node1, 1, ver3, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addNearLocal(node1, 1, nearVer2, true);
+        entry.addRemote(node1, 1, ver3, false, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
         Collection<GridCacheMvccCandidate> nearLocCands = entry.localCandidates();
@@ -525,9 +529,9 @@ public void testNearRemoteConsistentOrdering2() throws Exception {
         GridCacheVersion ver2 = version(20);
         GridCacheVersion ver3 = version(30);
 
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addNearLocal(node1, 1, nearVer2, 0, true);
-        entry.addRemote(node1, 1, ver3, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addNearLocal(node1, 1, nearVer2, true);
+        entry.addRemote(node1, 1, ver3, false, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
         Collection<GridCacheMvccCandidate> nearLocCands = entry.localCandidates();
@@ -570,9 +574,9 @@ public void testNearRemoteConsistentOrdering3() throws Exception {
         GridCacheVersion ver2 = version(20);
         GridCacheVersion ver3 = version(30);
 
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addNearLocal(node1, 1, nearVer2, 0, true);
-        entry.addRemote(node1, 1, ver3, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addNearLocal(node1, 1, nearVer2, true);
+        entry.addRemote(node1, 1, ver3, false, true);
 
         Collection<GridCacheMvccCandidate> rmtCands = entry.remoteMvccSnapshot();
         Collection<GridCacheMvccCandidate> nearLocCands = entry.localCandidates();
@@ -594,6 +598,222 @@ public void testNearRemoteConsistentOrdering3() throws Exception {
         assertEquals(ver1, rmtCands.iterator().next().version());
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSerializableReadLocksAdd() throws Exception {
+        GridCacheAdapter<String, String> cache = grid.internalCache();
+
+        GridCacheVersion serOrder1 = new GridCacheVersion(0, 0, 10, 1);
+        GridCacheVersion serOrder2 = new GridCacheVersion(0, 0, 20, 1);
+        GridCacheVersion serOrder3 = new GridCacheVersion(0, 0, 15, 1);
+
+        {
+            GridCacheMvcc mvcc = new GridCacheMvcc(cache.context());
+
+            GridCacheTestEntryEx e = new GridCacheTestEntryEx(cache.context(), "1");
+
+            GridCacheMvccCandidate cand1 = addLocal(mvcc, e, version(1), serOrder1, true);
+
+            assertNotNull(cand1);
+
+            GridCacheMvccCandidate cand2 = addLocal(mvcc, e, version(2), serOrder2, true);
+
+            assertNotNull(cand2);
+
+            GridCacheMvccCandidate cand3 = addLocal(mvcc, e, version(3), serOrder3, false);
+
+            assertNull(cand3);
+
+            cand3 = addLocal(mvcc, e, version(3), serOrder3, true);
+
+            assertNotNull(cand3);
+        }
+
+        {
+            GridCacheMvcc mvcc = new GridCacheMvcc(cache.context());
+
+            GridCacheTestEntryEx e = new GridCacheTestEntryEx(cache.context(), "1");
+
+            GridCacheMvccCandidate cand1 = addLocal(mvcc, e, version(1), serOrder2, true);
+
+            assertNotNull(cand1);
+
+            GridCacheMvccCandidate cand2 = addLocal(mvcc, e, version(2), serOrder1, true);
+
+            assertNotNull(cand2);
+
+            GridCacheMvccCandidate cand3 = addLocal(mvcc, e, version(3), serOrder3, false);
+
+            assertNull(cand3);
+
+            cand3 = addLocal(mvcc, e, version(3), serOrder3, true);
+
+            assertNotNull(cand3);
+        }
+
+        {
+            GridCacheMvcc mvcc = new GridCacheMvcc(cache.context());
+
+            GridCacheTestEntryEx e = new GridCacheTestEntryEx(cache.context(), "1");
+
+            GridCacheMvccCandidate cand1 = addLocal(mvcc, e, version(1), serOrder3, false);
+
+            assertNotNull(cand1);
+
+            GridCacheMvccCandidate cand2 = addLocal(mvcc, e, version(2), serOrder2, true);
+
+            assertNotNull(cand2);
+
+            GridCacheMvccCandidate cand3 = addLocal(mvcc, e, version(3), serOrder1, true);
+
+            assertNull(cand3);
+
+            cand3 = addLocal(mvcc, e, version(3), serOrder1, false);
+
+            assertNull(cand3);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSerializableReadLocksAssign() throws Exception {
+        GridCacheAdapter<String, String> cache = grid.internalCache();
+
+        GridCacheVersion serOrder1 = new GridCacheVersion(0, 0, 10, 1);
+        GridCacheVersion serOrder2 = new GridCacheVersion(0, 0, 20, 1);
+        GridCacheVersion serOrder3 = new GridCacheVersion(0, 0, 15, 1);
+
+        {
+            GridCacheMvcc mvcc = new GridCacheMvcc(cache.context());
+
+            GridCacheTestEntryEx e = new GridCacheTestEntryEx(cache.context(), "1");
+
+            GridCacheMvccCandidate cand1 = addLocal(mvcc, e, version(1), serOrder1, true);
+
+            assertNotNull(cand1);
+
+            GridCacheMvccCandidate cand2 = addLocal(mvcc, e, version(2), serOrder2, true);
+
+            assertNotNull(cand2);
+
+            GridCacheMvccCandidate cand3 = addLocal(mvcc, e, version(3), serOrder3, false);
+
+            assertNull(cand3);
+
+            cand3 = addLocal(mvcc, e, version(3), serOrder3, true);
+
+            assertNotNull(cand3);
+
+            CacheLockCandidates owners = mvcc.recheck();
+
+            assertNull(owners);
+
+            cand1.setReady();
+
+            owners = mvcc.recheck();
+
+            assertSame(cand1, owners);
+            checkCandidates(owners, cand1.version());
+
+            cand2.setReady();
+
+            owners = mvcc.recheck();
+            checkCandidates(owners, cand1.version(), cand2.version());
+
+            mvcc.remove(cand1.version());
+
+            owners = mvcc.recheck();
+            assertSame(cand2, owners);
+            checkCandidates(owners, cand2.version());
+        }
+
+        {
+            GridCacheMvcc mvcc = new GridCacheMvcc(cache.context());
+
+            GridCacheTestEntryEx e = new GridCacheTestEntryEx(cache.context(), "1");
+
+            GridCacheMvccCandidate cand1 = addLocal(mvcc, e, version(1), serOrder1, true);
+
+            assertNotNull(cand1);
+
+            GridCacheMvccCandidate cand2 = addLocal(mvcc, e, version(2), serOrder2, true);
+
+            assertNotNull(cand2);
+
+            GridCacheMvccCandidate cand3 = addLocal(mvcc, e, version(3), serOrder3, false);
+
+            assertNull(cand3);
+
+            cand3 = addLocal(mvcc, e, version(3), serOrder3, true);
+
+            assertNotNull(cand3);
+
+            CacheLockCandidates owners = mvcc.recheck();
+
+            assertNull(owners);
+
+            cand2.setReady();
+
+            owners = mvcc.recheck();
+
+            assertSame(cand2, owners);
+            checkCandidates(owners, cand2.version());
+
+            cand1.setReady();
+
+            owners = mvcc.recheck();
+            checkCandidates(owners, cand1.version(), cand2.version());
+
+            mvcc.remove(cand2.version());
+
+            owners = mvcc.recheck();
+            assertSame(cand1, owners);
+            checkCandidates(owners, cand1.version());
+        }
+    }
+
+    /**
+     * @param all Candidates list.
+     * @param vers Expected candidates.
+     */
+    private void checkCandidates(CacheLockCandidates all, GridCacheVersion...vers) {
+        assertNotNull(all);
+        assertEquals(vers.length, all.size());
+
+        for (GridCacheVersion ver : vers)
+            assertTrue(all.hasCandidate(ver));
+    }
+
+    /**
+     * @param mvcc Mvcc.
+     * @param e Entry.
+     * @param ver Version.
+     * @param serOrder Serializable tx version.
+     * @param read Read lock flag.
+     * @return Candidate.
+     */
+    @Nullable private GridCacheMvccCandidate addLocal(GridCacheMvcc mvcc,
+        GridCacheEntryEx e,
+        GridCacheVersion ver,
+        GridCacheVersion serOrder,
+        boolean read) {
+        return mvcc.addLocal(e,
+            nodeId,
+            null,
+            1,
+            ver,
+            0,
+            serOrder,
+            false,
+            true,
+            false,
+            true,
+            read
+        );
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -627,7 +847,8 @@ private void checkNonSerializableConflict() throws Exception {
             false,
             true,
             false,
-            true
+            true,
+            false
         );
 
         assertNotNull(cand1);
@@ -642,7 +863,8 @@ private void checkNonSerializableConflict() throws Exception {
             false,
             true,
             false,
-            true
+            true,
+            false
         );
 
         assertNull(cand2);
@@ -681,7 +903,8 @@ private void checkSerializableAdd(boolean incVer) throws Exception {
             false,
             true,
             false,
-            true
+            true,
+            false
             );
 
         assertNotNull(cand1);
@@ -696,7 +919,8 @@ private void checkSerializableAdd(boolean incVer) throws Exception {
             false,
             true,
             false,
-            true
+            true,
+            false
         );
 
         assertNotNull(cand2);
@@ -711,7 +935,8 @@ private void checkSerializableAdd(boolean incVer) throws Exception {
             false,
             true,
             false,
-            true
+            true,
+            false
         );
 
         assertNull(cand3);
@@ -726,36 +951,37 @@ private void checkSerializableAdd(boolean incVer) throws Exception {
             false,
             true,
             false,
-            true
+            true,
+            false
         );
 
         assertNotNull(cand4);
 
-        GridCacheMvccCandidate owner = mvcc.recheck();
+        CacheLockCandidates owners = mvcc.recheck();
 
-        assertNull(owner);
+        assertNull(owners);
 
         cand2.setReady();
 
-        owner = mvcc.recheck();
+        owners = mvcc.recheck();
 
-        assertNull(owner);
+        assertNull(owners);
 
         cand1.setReady();
 
-        owner = mvcc.recheck();
+        owners = mvcc.recheck();
 
-        assertSame(cand1, owner);
+        assertSame(cand1, owners);
 
-        owner = mvcc.recheck();
+        owners = mvcc.recheck();
 
-        assertSame(cand1, owner);
+        assertSame(cand1, owners);
 
         mvcc.remove(cand1.version());
 
-        owner = mvcc.recheck();
+        owners = mvcc.recheck();
 
-        assertSame(cand2, owner);
+        assertSame(cand2, owners);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccSelfTest.java
index 59f9a9d0d7154..f46b2904785b5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccSelfTest.java
@@ -95,14 +95,14 @@ public void testMarshalUnmarshalCandidate() throws Exception {
             version(1),
             123,
             version(2),
-            123,
             /*local*/false,
             /*reentry*/false,
             true,
             false,
             false,
             false,
-            null
+            null,
+            false
         );
 
         Marshaller marshaller = getTestResources().getMarshaller();
@@ -129,14 +129,14 @@ public void testRemotes() {
         GridCacheVersion ver4 = version(4);
         GridCacheVersion ver5 = version(5);
 
-        entry.addRemote(node1, 1, ver1, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
 
         Collection<GridCacheMvccCandidate> cands = entry.remoteMvccSnapshot();
 
         assert cands.size() == 1;
         assert cands.iterator().next().version().equals(ver1);
 
-        entry.addRemote(node2, 5, ver5, 0, false, true);
+        entry.addRemote(node2, 5, ver5, false, true);
 
         cands = entry.remoteMvccSnapshot();
 
@@ -147,7 +147,7 @@ public void testRemotes() {
         // Check order.
         checkOrder(cands, ver1, ver5);
 
-        entry.addRemote(node1, 3, ver3, 0, false, true);
+        entry.addRemote(node1, 3, ver3, false, true);
 
         cands = entry.remoteMvccSnapshot();
 
@@ -162,7 +162,7 @@ public void testRemotes() {
 
         checkDone(entry.candidate(ver3));
 
-        entry.addRemote(node1, 2, ver2, 0, false, true);
+        entry.addRemote(node1, 2, ver2, false, true);
 
         cands = entry.remoteMvccSnapshot();
 
@@ -197,7 +197,7 @@ public void testRemotes() {
 
         checkDone(entry.candidate(ver5));
 
-        entry.addRemote(node1, 4, ver4, 0, false, true);
+        entry.addRemote(node1, 4, ver4, false, true);
 
         cands = entry.remoteMvccSnapshot();
 
@@ -278,10 +278,10 @@ public void testNearRemoteWithOwned() {
         GridCacheVersion ver3 = version(3);
         GridCacheVersion ver4 = version(4);
 
-        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, 0, false, true);
-        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, 0, false, true);
-        GridCacheMvccCandidate c3 = entry.addRemote(node1, 1, ver3, 0, false, true);
-        GridCacheMvccCandidate c4 = entry.addRemote(node1, 1, ver4, 0, false, true);
+        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, false, true);
+        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, false, true);
+        GridCacheMvccCandidate c3 = entry.addRemote(node1, 1, ver3, false, true);
+        GridCacheMvccCandidate c4 = entry.addRemote(node1, 1, ver4, false, true);
 
         GridCacheMvccCandidate[] candArr = new GridCacheMvccCandidate[] {c1, c2, c3, c4};
 
@@ -322,12 +322,12 @@ public void testNearRemoteWithOwned1() {
         GridCacheVersion ver5 = version(5);
         GridCacheVersion ver6 = version(6);
 
-        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, 0, false, true);
-        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, 0, false, true);
-        GridCacheMvccCandidate c3 = entry.addRemote(node1, 1, ver3, 0, false, true);
-        GridCacheMvccCandidate c4 = entry.addRemote(node1, 1, ver4, 0, false, true);
-        GridCacheMvccCandidate c5 = entry.addRemote(node1, 1, ver5, 0, false, true);
-        GridCacheMvccCandidate c6 = entry.addRemote(node1, 1, ver6, 0, false, true);
+        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, false, true);
+        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, false, true);
+        GridCacheMvccCandidate c3 = entry.addRemote(node1, 1, ver3, false, true);
+        GridCacheMvccCandidate c4 = entry.addRemote(node1, 1, ver4, false, true);
+        GridCacheMvccCandidate c5 = entry.addRemote(node1, 1, ver5, false, true);
+        GridCacheMvccCandidate c6 = entry.addRemote(node1, 1, ver6, false, true);
 
         GridCacheMvccCandidate[] candArr = new GridCacheMvccCandidate[] {c1, c2, c3, c4, c5, c6};
 
@@ -369,13 +369,13 @@ public void testNearRemoteWithOwned2() {
         GridCacheVersion ver5 = version(5);
         GridCacheVersion ver6 = version(6);
 
-        GridCacheMvccCandidate c0 = entry.addRemote(node1, 1, ver0, 0, false, true);
-        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, 0, false, true);
-        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, 0, false, true);
-        GridCacheMvccCandidate c3 = entry.addRemote(node1, 1, ver3, 0, false, true);
-        GridCacheMvccCandidate c4 = entry.addRemote(node1, 1, ver4, 0, false, true);
-        GridCacheMvccCandidate c5 = entry.addRemote(node1, 1, ver5, 0, false, true);
-        GridCacheMvccCandidate c6 = entry.addRemote(node1, 1, ver6, 0, false, true);
+        GridCacheMvccCandidate c0 = entry.addRemote(node1, 1, ver0, false, true);
+        GridCacheMvccCandidate c1 = entry.addRemote(node1, 1, ver1, false, true);
+        GridCacheMvccCandidate c2 = entry.addRemote(node1, 1, ver2, false, true);
+        GridCacheMvccCandidate c3 = entry.addRemote(node1, 1, ver3, false, true);
+        GridCacheMvccCandidate c4 = entry.addRemote(node1, 1, ver4, false, true);
+        GridCacheMvccCandidate c5 = entry.addRemote(node1, 1, ver5, false, true);
+        GridCacheMvccCandidate c6 = entry.addRemote(node1, 1, ver6, false, true);
 
         GridCacheMvccCandidate[] candArr = new GridCacheMvccCandidate[] {c0, c1, c2, c3, c4, c5, c6};
 
@@ -486,7 +486,7 @@ public void testLocalWithRemote() {
         GridCacheVersion ver2 = version(2);
         GridCacheVersion ver3 = version(3);
 
-        entry.addRemote(nodeId, 1, ver2, 0, false, true);
+        entry.addRemote(nodeId, 1, ver2, false, true);
 
         entry.addLocal(3, ver3, 0, false, true);
 
@@ -529,15 +529,15 @@ public void testCompletedWithBaseInTheMiddle() {
         GridCacheVersion ver7 = version(7);
         GridCacheVersion ver8 = version(8);
 
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addRemote(node2, 2, ver2, 0, false, true);
-        entry.addRemote(node1, 3, ver3, 0, false, true);
-        entry.addRemote(node2, 4, ver4, 0, false, true);
-        entry.addRemote(node1, 5, ver5, 0, false, true);
-        entry.addRemote(node2, 7, ver7, 0, false, true);
-        entry.addRemote(node2, 8, ver8, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addRemote(node2, 2, ver2, false, true);
+        entry.addRemote(node1, 3, ver3, false, true);
+        entry.addRemote(node2, 4, ver4, false, true);
+        entry.addRemote(node1, 5, ver5, false, true);
+        entry.addRemote(node2, 7, ver7, false, true);
+        entry.addRemote(node2, 8, ver8, false, true);
 
-        GridCacheMvccCandidate doomed = entry.addRemote(node2, 6, ver6, 0, false, true);
+        GridCacheMvccCandidate doomed = entry.addRemote(node2, 6, ver6, false, true);
 
         // No reordering happens.
         checkOrder(entry.remoteMvccSnapshot(), ver1, ver2, ver3, ver4, ver5, ver7, ver8, ver6);
@@ -581,13 +581,13 @@ public void testCompletedWithCompletedBaseInTheMiddle() {
         GridCacheVersion ver6 = version(6);
         GridCacheVersion ver7 = version(7);
 
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addRemote(node2, 2, ver2, 0, false, true);
-        entry.addRemote(node1, 3, ver3, 0, false, true);
-        entry.addRemote(node2, 4, ver4, 0, false, true);
-        entry.addRemote(node1, 5, ver5, 0, false, true);
-        entry.addRemote(node2, 6, ver6, 0, false, true);
-        entry.addRemote(node2, 7, ver7, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addRemote(node2, 2, ver2, false, true);
+        entry.addRemote(node1, 3, ver3, false, true);
+        entry.addRemote(node2, 4, ver4, false, true);
+        entry.addRemote(node1, 5, ver5, false, true);
+        entry.addRemote(node2, 6, ver6, false, true);
+        entry.addRemote(node2, 7, ver7, false, true);
 
         List<GridCacheVersion> committed = Arrays.asList(ver4, ver6, ver2);
 
@@ -623,13 +623,13 @@ public void testCompletedTwiceWithBaseInTheMiddle() {
         GridCacheVersion ver6 = version(6);
         GridCacheVersion ver7 = version(7);
 
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addRemote(node2, 2, ver2, 0, false, true);
-        entry.addRemote(node1, 3, ver3, 0, false, true);
-        entry.addRemote(node2, 4, ver4, 0, false, true);
-        entry.addRemote(node1, 5, ver5, 0, false, true);
-        entry.addRemote(node2, 6, ver6, 0, false, true);
-        entry.addRemote(node2, 7, ver7, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addRemote(node2, 2, ver2, false, true);
+        entry.addRemote(node1, 3, ver3, false, true);
+        entry.addRemote(node2, 4, ver4, false, true);
+        entry.addRemote(node1, 5, ver5, false, true);
+        entry.addRemote(node2, 6, ver6, false, true);
+        entry.addRemote(node2, 7, ver7, false, true);
 
         List<GridCacheVersion> completed = Arrays.asList(ver4, ver6);
 
@@ -669,11 +669,11 @@ public void testCompletedWithBaseInTheMiddleNoChange() {
         GridCacheVersion ver6 = version(6);
         GridCacheVersion ver7 = version(7);
 
-        entry.addRemote(node1, 1, ver1, 0, false, false);
-        entry.addRemote(node2, 2, ver2, 0, false, false);
-        entry.addRemote(node1, 3, ver3, 0, false, false);
-        entry.addRemote(node2, 4, ver4, 0, false, false);
-        entry.addRemote(node1, 5, ver5, 0, false, false);
+        entry.addRemote(node1, 1, ver1, false, false);
+        entry.addRemote(node2, 2, ver2, false, false);
+        entry.addRemote(node1, 3, ver3, false, false);
+        entry.addRemote(node2, 4, ver4, false, false);
+        entry.addRemote(node1, 5, ver5, false, false);
 
         List<GridCacheVersion> committed = Arrays.asList(ver6, ver7);
 
@@ -708,13 +708,13 @@ public void testCompletedWithBaseInTheBeginning() {
         GridCacheVersion ver6 = version(6);
         GridCacheVersion ver7 = version(7);
 
-        entry.addRemote(node1, 1, ver1, 0, false, false);
-        entry.addRemote(node2, 2, ver2, 0, false, false);
-        entry.addRemote(node1, 3, ver3, 0, false, false);
-        entry.addRemote(node2, 4, ver4, 0, false, false);
-        entry.addRemote(node1, 5, ver5, 0, false, false);
-        entry.addRemote(node2, 6, ver6, 0, false, false);
-        entry.addRemote(node2, 7, ver7, 0, false, false);
+        entry.addRemote(node1, 1, ver1, false, false);
+        entry.addRemote(node2, 2, ver2, false, false);
+        entry.addRemote(node1, 3, ver3, false, false);
+        entry.addRemote(node2, 4, ver4, false, false);
+        entry.addRemote(node1, 5, ver5, false, false);
+        entry.addRemote(node2, 6, ver6, false, false);
+        entry.addRemote(node2, 7, ver7, false, false);
 
         List<GridCacheVersion> committed = Arrays.asList(ver4, ver6, ver3);
 
@@ -750,11 +750,11 @@ public void testCompletedWithBaseInTheBeginningNoChange() {
         GridCacheVersion ver6 = version(6);
         GridCacheVersion ver7 = version(7);
 
-        entry.addRemote(node1, 1, ver1, 0, false, false);
-        entry.addRemote(node2, 2, ver2, 0, false, false);
-        entry.addRemote(node1, 3, ver3, 0, false, false);
-        entry.addRemote(node2, 4, ver4, 0, false, false);
-        entry.addRemote(node1, 5, ver5, 0, false, false);
+        entry.addRemote(node1, 1, ver1, false, false);
+        entry.addRemote(node2, 2, ver2, false, false);
+        entry.addRemote(node1, 3, ver3, false, false);
+        entry.addRemote(node2, 4, ver4, false, false);
+        entry.addRemote(node1, 5, ver5, false, false);
 
         List<GridCacheVersion> committed = Arrays.asList(ver6, ver7);
 
@@ -789,11 +789,11 @@ public void testCompletedWithBaseInTheEndNoChange() {
         GridCacheVersion ver6 = version(6);
         GridCacheVersion ver7 = version(7);
 
-        entry.addRemote(node1, 1, ver1, 0, false, false);
-        entry.addRemote(node2, 2, ver2, 0, false, false);
-        entry.addRemote(node1, 3, ver3, 0, false, false);
-        entry.addRemote(node2, 4, ver4, 0, false, false);
-        entry.addRemote(node1, 5, ver5, 0, false, false);
+        entry.addRemote(node1, 1, ver1, false, false);
+        entry.addRemote(node2, 2, ver2, false, false);
+        entry.addRemote(node1, 3, ver3, false, false);
+        entry.addRemote(node2, 4, ver4, false, false);
+        entry.addRemote(node1, 5, ver5, false, false);
 
         List<GridCacheVersion> committed = Arrays.asList(ver6, ver7);
 
@@ -829,12 +829,12 @@ public void testCompletedWithBaseNotPresentInTheMiddle() {
         GridCacheVersion ver7 = version(7);
 
         // Don't add version 2.
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addRemote(node1, 3, ver3, 0, false, true);
-        entry.addRemote(node2, 4, ver4, 0, false, true);
-        entry.addRemote(node1, 5, ver5, 0, false, true);
-        entry.addRemote(node2, 6, ver6, 0, false, true);
-        entry.addRemote(node2, 7, ver7, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addRemote(node1, 3, ver3, false, true);
+        entry.addRemote(node2, 4, ver4, false, true);
+        entry.addRemote(node1, 5, ver5, false, true);
+        entry.addRemote(node2, 6, ver6, false, true);
+        entry.addRemote(node2, 7, ver7, false, true);
 
         List<GridCacheVersion> committed = Arrays.asList(ver6, ver4);
 
@@ -870,9 +870,9 @@ public void testCompletedWithBaseNotPresentInTheMiddleNoChange() {
         GridCacheVersion ver7 = version(7);
 
         // Don't add versions 2, 5, 6, 7.
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addRemote(node1, 3, ver3, 0, false, true);
-        entry.addRemote(node2, 4, ver4, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addRemote(node1, 3, ver3, false, true);
+        entry.addRemote(node2, 4, ver4, false, true);
 
         List<GridCacheVersion> committed = Arrays.asList(ver6, ver5, ver7);
 
@@ -905,12 +905,12 @@ public void testCompletedWithBaseNotPresentInTheBeginning() {
         GridCacheVersion ver7 = version(7);
 
         // Don't add version 1.
-        entry.addRemote(node1, 2, ver2, 0, false, true);
-        entry.addRemote(node1, 3, ver3, 0, false, true);
-        entry.addRemote(node2, 4, ver4, 0, false, true);
-        entry.addRemote(node1, 5, ver5, 0, false, true);
-        entry.addRemote(node2, 6, ver6, 0, false, true);
-        entry.addRemote(node2, 7, ver7, 0, false, true);
+        entry.addRemote(node1, 2, ver2, false, true);
+        entry.addRemote(node1, 3, ver3, false, true);
+        entry.addRemote(node2, 4, ver4, false, true);
+        entry.addRemote(node1, 5, ver5, false, true);
+        entry.addRemote(node2, 6, ver6, false, true);
+        entry.addRemote(node2, 7, ver7, false, true);
 
         List<GridCacheVersion> committed = Arrays.asList(ver4, ver6, ver3);
 
@@ -946,12 +946,12 @@ public void testCompletedWithBaseNotPresentInTheBeginningNoChange() {
         GridCacheVersion ver7 = version(7);
 
         // Don't add version 6, 7
-        entry.addRemote(node1, 2, ver2, 0, false, true);
-        entry.addRemote(node1, 3, ver3, 0, false, true);
-        entry.addRemote(node2, 4, ver4, 0, false, true);
-        entry.addRemote(node1, 5, ver5, 0, false, true);
-        entry.addRemote(node1, 6, ver6, 0, false, true);
-        entry.addRemote(node1, 7, ver7, 0, false, true);
+        entry.addRemote(node1, 2, ver2, false, true);
+        entry.addRemote(node1, 3, ver3, false, true);
+        entry.addRemote(node2, 4, ver4, false, true);
+        entry.addRemote(node1, 5, ver5, false, true);
+        entry.addRemote(node1, 6, ver6, false, true);
+        entry.addRemote(node1, 7, ver7, false, true);
 
         List<GridCacheVersion> committed = Arrays.asList(ver2, ver3);
 
@@ -987,10 +987,10 @@ public void testCompletedWithBaseNotPresentInTheEndNoChange() {
         GridCacheVersion ver7 = version(7);
 
         // Don't add version 5, 6, 7
-        entry.addRemote(node1, 1, ver1, 0, false, true);
-        entry.addRemote(node1, 2, ver2, 0, false, true);
-        entry.addRemote(node1, 3, ver3, 0, false, true);
-        entry.addRemote(node2, 4, ver4, 0, false, true);
+        entry.addRemote(node1, 1, ver1, false, true);
+        entry.addRemote(node1, 2, ver2, false, true);
+        entry.addRemote(node1, 3, ver3, false, true);
+        entry.addRemote(node2, 4, ver4, false, true);
 
         List<GridCacheVersion> committed = Arrays.asList(ver6, ver7);
 
@@ -1021,7 +1021,7 @@ public void testLocalAndRemote() {
         GridCacheVersion ver4 = version(4);
         GridCacheVersion ver5 = version(5);
 
-        entry.addRemote(node1, 1, ver1, 0, false, false);
+        entry.addRemote(node1, 1, ver1, false, false);
         entry.addLocal(2, ver2, 0, true, true);
 
         Collection<GridCacheMvccCandidate> cands = entry.remoteMvccSnapshot();
@@ -1029,7 +1029,7 @@ public void testLocalAndRemote() {
         assert cands.size() == 1;
         assert cands.iterator().next().version().equals(ver1);
 
-        entry.addRemote(node2, 5, ver5, 0, false, false);
+        entry.addRemote(node2, 5, ver5, false, false);
 
         cands = entry.remoteMvccSnapshot();
 
@@ -1040,7 +1040,7 @@ public void testLocalAndRemote() {
         checkOrder(cands, ver1, ver5);
         checkOrder(entry.localCandidates(true), ver2);
 
-        entry.addRemote(node1, 3, ver3, 0, false, true);
+        entry.addRemote(node1, 3, ver3, false, true);
         entry.addLocal(4, ver4, 0, /*reenter*/true, false);
 
         cands = entry.remoteMvccSnapshot();
@@ -1174,11 +1174,11 @@ public void testMultipleLocalAndRemoteLocks1() throws Exception {
 
         linkCandidates(ctx, c13, c33);
 
-        entry2.addRemote(nodeId, 3, ver2, 0, false, true);
+        entry2.addRemote(nodeId, 3, ver2, false, true);
 
         checkLocal(entry2.candidate(ver1), ver1, true, false, false);
 
-        entry3.addRemote(nodeId, 3, ver2, 0, false, false);
+        entry3.addRemote(nodeId, 3, ver2, false, false);
 
         entry3.readyLocal(ver3);
 
@@ -1245,11 +1245,11 @@ public void testMultipleLocalAndRemoteLocks2() throws Exception {
 
         linkCandidates(ctx, c13, c33);
 
-        entry2.addRemote(UUID.randomUUID(), 3, ver1, 0, false, true);
+        entry2.addRemote(UUID.randomUUID(), 3, ver1, false, true);
 
         checkLocal(entry2.candidate(ver2), ver2, true, false, false);
 
-        entry3.addRemote(UUID.randomUUID(), 3, ver1, 0, false, true);
+        entry3.addRemote(UUID.randomUUID(), 3, ver1, false, true);
 
         entry3.readyLocal(ver3);
 
@@ -1421,7 +1421,7 @@ public void testReverseOrder1() {
 
         checkLocal(c1k1, ver2, true, true, false);
 
-        GridCacheMvccCandidate c2k1 = entry1.addRemote(id, 2, ver1, 0, false, true);
+        GridCacheMvccCandidate c2k1 = entry1.addRemote(id, 2, ver1, false, true);
 
         // Force recheck of assignments.
         entry1.recheckLock();
@@ -1437,7 +1437,7 @@ public void testReverseOrder1() {
 
         assert c1k2.previous() == c1k1;
 
-        GridCacheMvccCandidate c2k2 = entry2.addRemote(id, 3, ver1, 0, false, true);
+        GridCacheMvccCandidate c2k2 = entry2.addRemote(id, 3, ver1, false, true);
 
         entry2.readyLocal(c1k2);
 
@@ -1479,8 +1479,8 @@ public void testReverseOrder2() throws Exception {
         checkLocal(v3k2, ver3, false, false, false);
 
         // Remote locks.
-        GridCacheMvccCandidate v2k1 = entry1.addRemote(id, 3, ver2, 0, false, false);
-        GridCacheMvccCandidate v2k2 = entry2.addRemote(id, 3, ver2, 0, false, false);
+        GridCacheMvccCandidate v2k1 = entry1.addRemote(id, 3, ver2, false, false);
+        GridCacheMvccCandidate v2k2 = entry2.addRemote(id, 3, ver2, false, false);
 
         checkRemote(v2k1, ver2, false, false);
         checkRemote(v2k2, ver2, false, false);
@@ -1602,7 +1602,7 @@ public void testReverseOrder4() throws Exception {
         checkLocal(v1k1, ver1, true, false, false);
         checkLocal(v1k2, ver1, true, false, false);
 
-        GridCacheMvccCandidate v2k2 = entry2.addRemote(id, 5, ver2, 0, false, false);
+        GridCacheMvccCandidate v2k2 = entry2.addRemote(id, 5, ver2, false, false);
 
         checkRemote(v2k2, ver2, false, false);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 396629ac4401a..48621afc1a8b8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -64,24 +64,12 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
      * @param ctx Context.
      * @param key Key.
      */
-    public GridCacheTestEntryEx(GridCacheContext ctx, Object key) {
+    GridCacheTestEntryEx(GridCacheContext ctx, Object key) {
         mvcc = new GridCacheMvcc(ctx);
 
         this.key = ctx.toCacheKeyObject(key);
     }
 
-    /**
-     * @param ctx Context.
-     * @param key Key.
-     * @param val Value.
-     */
-    public GridCacheTestEntryEx(GridCacheContext ctx, Object key, Object val) {
-        mvcc = new GridCacheMvcc(ctx);
-
-        this.key = ctx.toCacheKeyObject(key);
-        this.val = ctx.toCacheObject(val);
-    }
-
     /** {@inheritDoc} */
     @Override public int memorySize() throws IgniteCheckedException {
         return 1024;
@@ -146,7 +134,7 @@ public GridCacheTestEntryEx(GridCacheContext ctx, Object key, Object val) {
      * @return New lock candidate if lock was added, or current owner if lock was reentered,
      *      or <tt>null</tt> if lock was owned by another thread and timeout is negative.
      */
-    @Nullable public GridCacheMvccCandidate addLocal(
+    @Nullable GridCacheMvccCandidate addLocal(
         long threadId,
         GridCacheVersion ver,
         long timeout,
@@ -159,6 +147,7 @@ public GridCacheTestEntryEx(GridCacheContext ctx, Object key, Object val) {
             timeout,
             reenter,
             tx,
+            false,
             false
         );
     }
@@ -169,14 +158,13 @@ public GridCacheTestEntryEx(GridCacheContext ctx, Object key, Object val) {
      * @param nodeId Node ID.
      * @param threadId Thread ID.
      * @param ver Lock version.
-     * @param timeout Lock acquire timeout.
      * @param ec Not used.
      * @param tx Transaction flag.
      * @return Remote candidate.
      */
-    public GridCacheMvccCandidate addRemote(UUID nodeId, long threadId, GridCacheVersion ver, long timeout,
+    GridCacheMvccCandidate addRemote(UUID nodeId, long threadId, GridCacheVersion ver,
         boolean ec, boolean tx) {
-        return mvcc.addRemote(this, nodeId, null, threadId, ver, timeout, tx, true, false);
+        return mvcc.addRemote(this, nodeId, null, threadId, ver, tx, true, false);
     }
 
     /**
@@ -185,20 +173,19 @@ public GridCacheMvccCandidate addRemote(UUID nodeId, long threadId, GridCacheVer
      * @param nodeId Node ID.
      * @param threadId Thread ID.
      * @param ver Lock version.
-     * @param timeout Lock acquire timeout.
      * @param tx Transaction flag.
      * @return Remote candidate.
      */
-    public GridCacheMvccCandidate addNearLocal(UUID nodeId, long threadId, GridCacheVersion ver, long timeout,
+    GridCacheMvccCandidate addNearLocal(UUID nodeId, long threadId, GridCacheVersion ver,
         boolean tx) {
-        return mvcc.addNearLocal(this, nodeId, null, threadId, ver, timeout, tx, true);
+        return mvcc.addNearLocal(this, nodeId, null, threadId, ver, tx, true, false);
     }
 
     /**
      *
      * @param baseVer Base version.
      */
-    public void salvageRemote(GridCacheVersion baseVer) {
+    void salvageRemote(GridCacheVersion baseVer) {
         mvcc.salvageRemote(baseVer);
     }
 
@@ -210,17 +197,16 @@ public void salvageRemote(GridCacheVersion baseVer) {
      * @param baseVer Base version.
      * @param committedVers Committed versions relative to base.
      * @param rolledbackVers Rolled back versions relative to base.
-     * @return Lock owner.
      */
-    @Nullable public GridCacheMvccCandidate orderCompleted(GridCacheVersion baseVer,
+    void orderCompleted(GridCacheVersion baseVer,
         Collection<GridCacheVersion> committedVers, Collection<GridCacheVersion> rolledbackVers) {
-        return mvcc.orderCompleted(baseVer, committedVers, rolledbackVers);
+        mvcc.orderCompleted(baseVer, committedVers, rolledbackVers);
     }
 
     /**
      * @param ver Version.
      */
-    public void doneRemote(GridCacheVersion ver) {
+    void doneRemote(GridCacheVersion ver) {
         mvcc.doneRemote(ver, Collections.<GridCacheVersion>emptyList(),
             Collections.<GridCacheVersion>emptyList(), Collections.<GridCacheVersion>emptyList());
     }
@@ -229,16 +215,15 @@ public void doneRemote(GridCacheVersion ver) {
      * @param baseVer Base version.
      * @param owned Owned.
      */
-    public void orderOwned(GridCacheVersion baseVer, GridCacheVersion owned) {
+    void orderOwned(GridCacheVersion baseVer, GridCacheVersion owned) {
         mvcc.markOwned(baseVer, owned);
     }
 
     /**
      * @param ver Lock version to acquire or set to ready.
-     * @return Current owner.
      */
-    @Nullable public GridCacheMvccCandidate readyLocal(GridCacheVersion ver) {
-        return mvcc.readyLocal(ver);
+    void readyLocal(GridCacheVersion ver) {
+        mvcc.readyLocal(ver);
     }
 
     /**
@@ -247,44 +232,33 @@ public void orderOwned(GridCacheVersion baseVer, GridCacheVersion owned) {
      * @param committedVers Committed versions.
      * @param rolledbackVers Rolled back versions.
      * @param pending Pending versions.
-     * @return Lock owner.
      */
-    @Nullable public GridCacheMvccCandidate readyNearLocal(GridCacheVersion ver, GridCacheVersion mapped,
+    void readyNearLocal(GridCacheVersion ver, GridCacheVersion mapped,
         Collection<GridCacheVersion> committedVers, Collection<GridCacheVersion> rolledbackVers,
         Collection<GridCacheVersion> pending) {
-        return mvcc.readyNearLocal(ver, mapped, committedVers, rolledbackVers, pending);
+        mvcc.readyNearLocal(ver, mapped, committedVers, rolledbackVers, pending);
     }
 
     /**
      * @param cand Candidate to set to ready.
-     * @return Current owner.
      */
-    @Nullable public GridCacheMvccCandidate readyLocal(GridCacheMvccCandidate cand) {
-        return mvcc.readyLocal(cand);
-    }
-
-    /**
-     * Local local release.
-     * @return Removed lock candidate or <tt>null</tt> if candidate was not removed.
-     */
-    @Nullable public GridCacheMvccCandidate releaseLocal() {
-        return releaseLocal(Thread.currentThread().getId());
+    void readyLocal(GridCacheMvccCandidate cand) {
+        mvcc.readyLocal(cand);
     }
 
     /**
      * Local release.
      *
      * @param threadId ID of the thread.
-     * @return Current owner.
      */
-    @Nullable public GridCacheMvccCandidate releaseLocal(long threadId) {
-        return mvcc.releaseLocal(threadId);
+    void releaseLocal(long threadId) {
+        mvcc.releaseLocal(threadId);
     }
 
     /**
      *
      */
-    public void recheckLock() {
+    void recheckLock() {
         mvcc.recheck();
     }
 
@@ -594,8 +568,9 @@ public void recheckLock() {
         long timeout,
         @Nullable GridCacheVersion serOrder,
         GridCacheVersion serReadVer,
-        boolean keepBinary) {
-        assert false; return false;
+        boolean read) {
+        assert false;
+        return false;
     }
 
     /** @inheritDoc */
@@ -765,7 +740,7 @@ public void recheckLock() {
     }
 
     /** @inheritDoc */
-    public Collection<GridCacheMvccCandidate> localCandidates(boolean reentries, GridCacheVersion... exclude) {
+    Collection<GridCacheMvccCandidate> localCandidates(boolean reentries, GridCacheVersion... exclude) {
         return mvcc.localCandidates(reentries, exclude);
     }
 
@@ -793,7 +768,7 @@ public Collection<GridCacheMvccCandidate> localCandidates(boolean reentries, Gri
     /**
      * @return Any MVCC owner.
      */
-    public GridCacheMvccCandidate anyOwner() {
+    GridCacheMvccCandidate anyOwner() {
         return mvcc.anyOwner();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridHashMapLoadTest.java b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridHashMapLoadTest.java
index b459dde735b0d..5c12f8496ee99 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridHashMapLoadTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridHashMapLoadTest.java
@@ -21,6 +21,7 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
+import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.testframework.junits.GridTestKernalContext;
@@ -86,10 +87,14 @@ public void testMapEntry() throws Exception {
                     long timeout,
                     @Nullable GridCacheVersion serOrder,
                     GridCacheVersion serReadVer,
-                    boolean keepBinary) {
+                    boolean read) {
                     return false;
                 }
 
+                @Override protected void checkThreadChain(GridCacheMvccCandidate owner) {
+                    // No-op.
+                }
+
                 @Override public void txUnlock(IgniteInternalTx tx) {
                     // No-op.
                 }

From cc13d9d155f70e22e08ef203ac64e5cc0aa0a50f Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 28 Nov 2016 11:30:14 +0300
Subject: [PATCH 014/446] IGNITE-4026: Fixed BinaryObjectBuilder.build() can
 fail if one of the fields is Externalizable, enum from binary object. This
 closes #1281. This closes #1289.

(cherry picked from commit 0b7c62d)
---
 .../internal/binary/GridBinaryMarshaller.java |   2 +-
 .../binary/builder/BinaryBuilderReader.java   |  11 ++
 .../builder/BinaryBuilderSerializer.java      |   4 +
 .../builder/BinaryObjectBuilderImpl.java      |  10 ++
 ...BinaryObjectBuilderAdditionalSelfTest.java | 157 +++++++++++++++++-
 5 files changed, 181 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java
index ad635214898d0..624fa33fd0f5d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java
@@ -43,7 +43,7 @@ public class GridBinaryMarshaller {
     private static final ThreadLocal<BinaryContext> BINARY_CTX = new ThreadLocal<>();
 
     /** */
-    static final byte OPTM_MARSH = -2;
+    public static final byte OPTM_MARSH = -2;
 
     /** */
     public static final byte BYTE = 1;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
index 347fb2bb9d99e..baaabd6a1a0bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
@@ -33,6 +33,7 @@
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
@@ -757,6 +758,16 @@ else if (flag == GridBinaryMarshaller.DECIMAL) {
                 return new BinaryPlainBinaryObject(binaryObj);
             }
 
+            case GridBinaryMarshaller.OPTM_MARSH: {
+                final BinaryHeapInputStream bin = BinaryHeapInputStream.create(arr, pos);
+
+                final Object obj = BinaryUtils.doReadOptimized(bin, ctx, U.resolveClassLoader(ctx.configuration()));
+
+                pos = bin.position();
+
+                return obj;
+            }
+
             default:
                 throw new BinaryObjectException("Invalid flag value: " + type);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
index b296437226426..6974176f90053 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
@@ -116,8 +116,12 @@ public void writeValue(BinaryWriterExImpl writer, Object val, boolean forceCol,
             String typeName = writer.context().userTypeName(clsName);
 
             BinaryMetadata meta = new BinaryMetadata(typeId, typeName, null, null, null, true);
+
             writer.context().updateMetadata(typeId, meta);
 
+            // Need register class for marshaller to be able to deserialize enum value.
+            writer.context().descriptorForClass(val.getClass(), false);
+
             writer.writeByte(GridBinaryMarshaller.ENUM);
             writer.writeInt(typeId);
             writer.writeInt(((Enum)val).ordinal());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
index f0bc8745bf266..6ea9e7ec3a728 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.binary.BinaryObjectBuilder;
 import org.apache.ignite.binary.BinaryObjectException;
 import org.apache.ignite.binary.BinaryType;
+import org.apache.ignite.internal.binary.BinaryEnumObjectImpl;
 import org.apache.ignite.internal.binary.BinaryMetadata;
 import org.apache.ignite.internal.binary.BinaryObjectImpl;
 import org.apache.ignite.internal.binary.BinaryWriterExImpl;
@@ -385,6 +386,15 @@ private Map<String, Integer> checkMetadata(BinaryType meta, Map<String, Integer>
             if (((BinaryValueWithType)newVal).value() == null)
                 nullFieldVal = true;
         }
+        // Detect Enum and Enum array type.
+        else if (newVal instanceof BinaryEnumObjectImpl)
+            newFldTypeId = GridBinaryMarshaller.ENUM;
+        else if (newVal.getClass().isArray() && newVal.getClass().getComponentType() == BinaryObject.class) {
+            BinaryObject[] arr = (BinaryObject[])newVal;
+
+            newFldTypeId = arr.length > 0 && arr[0] instanceof BinaryEnumObjectImpl ?
+                GridBinaryMarshaller.ENUM_ARR : GridBinaryMarshaller.OBJ_ARR;
+        }
         else
             newFldTypeId = BinaryUtils.typeByClass(newVal.getClass());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderAdditionalSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderAdditionalSelfTest.java
index 24806cbf2bf01..507aa6bf239ea 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderAdditionalSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderAdditionalSelfTest.java
@@ -21,6 +21,10 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
 import java.lang.reflect.Field;
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -1423,11 +1427,19 @@ private BinaryObjectBuilderImpl wrap(Object obj) {
      * @return Wrapper.
      */
     private BinaryObjectBuilderImpl newWrapper(Class<?> aCls) {
+        return newWrapper(aCls.getName());
+    }
+
+    /**
+     * @param typeName Type name.
+     * @return Wrapper.
+     */
+    private BinaryObjectBuilderImpl newWrapper(String typeName) {
         CacheObjectBinaryProcessorImpl processor = (CacheObjectBinaryProcessorImpl)(
             (IgniteBinaryImpl)binaries()).processor();
 
-        return new BinaryObjectBuilderImpl(processor.binaryContext(), processor.typeId(aCls.getName()),
-            processor.binaryContext().userTypeName(aCls.getName()));
+        return new BinaryObjectBuilderImpl(processor.binaryContext(), processor.typeId(typeName),
+            processor.binaryContext().userTypeName(typeName));
     }
 
     /**
@@ -1508,4 +1520,145 @@ public void testCollectionsSerialization() {
         assert OBJ.equals(binaryObj.type().fieldTypeName("asSetHint"));
         assert OBJ.equals(binaryObj.type().fieldTypeName("asMapHint"));
     }
+
+    /**
+     * Checks that externalizable value is correctly serialized/deserialized.
+     *
+     * @throws Exception If failed.
+     */
+    public void testBuilderExternalizable() throws Exception {
+        BinaryObjectBuilder builder = newWrapper("TestType");
+
+        final TestObjectExternalizable exp = new TestObjectExternalizable("test");
+        final TestObjectExternalizable[] expArr = new TestObjectExternalizable[]{
+            new TestObjectExternalizable("test1"), new TestObjectExternalizable("test2")};
+
+        BinaryObject extObj = builder.setField("extVal", exp).setField("extArr", expArr).build();
+
+        assertEquals(exp, extObj.field("extVal"));
+        Assert.assertArrayEquals(expArr, (Object[])extObj.field("extArr"));
+
+        builder = extObj.toBuilder();
+
+        extObj = builder.setField("intVal", 10).build();
+
+        assertEquals(exp, extObj.field("extVal"));
+        Assert.assertArrayEquals(expArr, (Object[])extObj.field("extArr"));
+        assertEquals(Integer.valueOf(10), extObj.field("intVal"));
+
+        builder = extObj.toBuilder();
+
+        extObj = builder.setField("strVal", "some string").build();
+
+        assertEquals(exp, extObj.field("extVal"));
+        Assert.assertArrayEquals(expArr, (Object[])extObj.field("extArr"));
+        assertEquals(Integer.valueOf(10), extObj.field("intVal"));
+        assertEquals("some string", extObj.field("strVal"));
+    }
+
+    /**
+     * Checks correct serialization/deserialization of enums in builder.
+     *
+     * @throws Exception If failed.
+     */
+    public void testEnum() throws Exception {
+        BinaryObjectBuilder builder = newWrapper("TestType");
+
+        final TestEnum exp = TestEnum.A;
+        final TestEnum[] expArr = {TestEnum.A, TestEnum.B};
+
+        BinaryObject enumObj = builder.setField("testEnum", exp).setField("testEnumArr", expArr).build();
+
+        assertEquals(exp, ((BinaryObject)enumObj.field("testEnum")).deserialize());
+        Assert.assertArrayEquals(expArr, (Object[])deserializeEnumBinaryArray(enumObj.field("testEnumArr")));
+
+        builder = newWrapper(enumObj.type().typeName());
+
+        enumObj = builder.setField("testEnum", (Object)enumObj.field("testEnum"))
+            .setField("testEnumArr", (Object)enumObj.field("testEnumArr")).build();
+
+        assertEquals(exp, ((BinaryObject)enumObj.field("testEnum")).deserialize());
+        Assert.assertArrayEquals(expArr, (Object[])deserializeEnumBinaryArray(enumObj.field("testEnumArr")));
+    }
+
+    /**
+     * @param obj BinaryObject array.
+     * @return Deserialized enums.
+     */
+    private TestEnum[] deserializeEnumBinaryArray(Object obj) {
+        Object[] arr = (Object[])obj;
+
+        final TestEnum[] res = new TestEnum[arr.length];
+
+        for (int i = 0; i < arr.length; i++)
+            res[i] = ((BinaryObject)arr[i]).deserialize();
+
+        return res;
+    }
+
+    /**
+     *
+     */
+    private static class TestObjectExternalizable implements Externalizable {
+        /** */
+        private String val;
+
+        /**
+         *
+         */
+        public TestObjectExternalizable() {
+        }
+
+        /**
+         * @param val Value.
+         */
+        public TestObjectExternalizable(final String val) {
+            this.val = val;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(final ObjectOutput out) throws IOException {
+            out.writeUTF(val);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException {
+            val = in.readUTF();
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(final Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            final TestObjectExternalizable that = (TestObjectExternalizable)o;
+
+            return val != null ? val.equals(that.val) : that.val == null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return val != null ? val.hashCode() : 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "TestObjectExternalizable{" +
+                "val='" + val + '\'' +
+                '}';
+        }
+    }
+
+    /**
+     *
+     */
+    private enum TestEnum {
+        /** */
+        A,
+
+        /** */
+        B
+    }
 }

From b4aedfd5350b4a318f1608596a171789513835a4 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Fri, 2 Dec 2016 20:10:09 +0300
Subject: [PATCH 015/446] IGNITE-4347: Fixed NPE.

---
 .../internal/processors/odbc/OdbcRequestHandler.java   | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
index eef99454db212..0a70a68856672 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
@@ -184,11 +184,17 @@ private OdbcResponse executeQuery(long reqId, OdbcQueryExecuteRequest req) {
             qry.setDistributedJoins(distributedJoins);
             qry.setEnforceJoinOrder(enforceJoinOrder);
 
-            IgniteCache<Object, Object> cache = ctx.grid().cache(req.cacheName());
+            IgniteCache<Object, Object> cache0 = ctx.grid().cache(req.cacheName());
+
+            if (cache0 == null)
+                return new OdbcResponse(OdbcResponse.STATUS_FAILED,
+                        "Cache doesn't exist (did you configure it?): " + req.cacheName());
+
+            IgniteCache<Object, Object> cache = cache0.withKeepBinary();
 
             if (cache == null)
                 return new OdbcResponse(OdbcResponse.STATUS_FAILED,
-                    "Cache doesn't exist (did you configure it?): " + req.cacheName());
+                    "Can not get cache with keep binary: " + req.cacheName());
 
             QueryCursor qryCur = cache.query(qry);
 

From acf20b32d8fb68e42b904b091fb2b943f4558cef Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Mon, 5 Dec 2016 14:01:28 +0300
Subject: [PATCH 016/446] ignite-4296 Optimize GridDhtPartitionsSingleMessage
 processing - optimized processing of GridDhtPartitionsSingleMessage - minor
 optimizations for RendezvousAffinityFunction  - fixed heartbeats sending in
 tcp discovery

---
 .../RendezvousAffinityFunction.java           |  80 +++--
 .../discovery/GridDiscoveryManager.java       | 118 +-------
 .../GridCachePartitionExchangeManager.java    |   6 +-
 .../processors/cache/GridCacheUtils.java      |  17 --
 .../CacheObjectBinaryProcessorImpl.java       |   3 +-
 .../dht/GridClientPartitionTopology.java      | 120 ++++----
 .../dht/GridDhtLocalPartition.java            |   1 -
 .../dht/GridDhtPartitionTopology.java         |  28 +-
 .../dht/GridDhtPartitionTopologyImpl.java     | 284 +++++++++++-------
 .../preloader/GridDhtPartitionFullMap.java    |  18 +-
 .../GridDhtPartitionsExchangeFuture.java      |  56 +++-
 .../cache/transactions/IgniteTxHandler.java   |   4 +-
 .../service/GridServiceProcessor.java         |   4 +-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  40 ++-
 .../tcp/internal/TcpDiscoveryStatistics.java  |   4 +
 .../AbstractAffinityFunctionSelfTest.java     |   2 +-
 ...ridDiscoveryManagerAliveCacheSelfTest.java |   2 +-
 ...ridCachePartitionedAffinitySpreadTest.java |   7 +-
 .../dht/GridCacheDhtTestUtils.java            | 232 --------------
 .../h2/twostep/GridReduceQueryExecutor.java   |  14 +-
 20 files changed, 437 insertions(+), 603 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtTestUtils.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
index ec12973ceefd7..75e7c92b4f24c 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.cache.affinity.rendezvous;
 
-import java.io.ByteArrayOutputStream;
 import java.io.Externalizable;
 import java.io.IOException;
 import java.io.ObjectInput;
@@ -354,46 +353,69 @@ public Object resolveNodeHash(ClusterNode node) {
 
     /**
      * Returns collection of nodes (primary first) for specified partition.
+     *
+     * @param d Message digest.
+     * @param part Partition.
+     * @param nodes Nodes.
+     * @param nodesHash Serialized nodes hashes.
+     * @param backups Number of backups.
+     * @param neighborhoodCache Neighborhood.
+     * @return Assignment.
      */
-    public List<ClusterNode> assignPartition(int part, List<ClusterNode> nodes, int backups,
+    public List<ClusterNode> assignPartition(MessageDigest d,
+        int part,
+        List<ClusterNode> nodes,
+        Map<ClusterNode, byte[]> nodesHash,
+        int backups,
         @Nullable Map<UUID, Collection<ClusterNode>> neighborhoodCache) {
         if (nodes.size() <= 1)
             return nodes;
 
-        List<IgniteBiTuple<Long, ClusterNode>> lst = new ArrayList<>();
+        if (d == null)
+            d = digest.get();
 
-        MessageDigest d = digest.get();
+        List<IgniteBiTuple<Long, ClusterNode>> lst = new ArrayList<>(nodes.size());
 
-        for (ClusterNode node : nodes) {
-            Object nodeHash = resolveNodeHash(node);
+        try {
+            for (int i = 0; i < nodes.size(); i++) {
+                ClusterNode node = nodes.get(i);
 
-            try {
-                ByteArrayOutputStream out = new ByteArrayOutputStream();
+                byte[] nodeHashBytes = nodesHash.get(node);
 
-                byte[] nodeHashBytes = U.marshal(ignite.configuration().getMarshaller(), nodeHash);
+                if (nodeHashBytes == null) {
+                    Object nodeHash = resolveNodeHash(node);
 
-                out.write(U.intToBytes(part), 0, 4); // Avoid IOException.
-                out.write(nodeHashBytes, 0, nodeHashBytes.length); // Avoid IOException.
+                    byte[] nodeHashBytes0 = U.marshal(ignite.configuration().getMarshaller(), nodeHash);
+
+                    // Add 4 bytes for partition bytes.
+                    nodeHashBytes = new byte[nodeHashBytes0.length + 4];
+
+                    System.arraycopy(nodeHashBytes0, 0, nodeHashBytes, 4, nodeHashBytes0.length);
+
+                    nodesHash.put(node, nodeHashBytes);
+                }
+
+                U.intToBytes(part, nodeHashBytes, 0);
 
                 d.reset();
 
-                byte[] bytes = d.digest(out.toByteArray());
+                byte[] bytes = d.digest(nodeHashBytes);
 
                 long hash =
-                      (bytes[0] & 0xFFL)
-                    | ((bytes[1] & 0xFFL) << 8)
-                    | ((bytes[2] & 0xFFL) << 16)
-                    | ((bytes[3] & 0xFFL) << 24)
-                    | ((bytes[4] & 0xFFL) << 32)
-                    | ((bytes[5] & 0xFFL) << 40)
-                    | ((bytes[6] & 0xFFL) << 48)
-                    | ((bytes[7] & 0xFFL) << 56);
+                    (bytes[0] & 0xFFL)
+                        | ((bytes[1] & 0xFFL) << 8)
+                        | ((bytes[2] & 0xFFL) << 16)
+                        | ((bytes[3] & 0xFFL) << 24)
+                        | ((bytes[4] & 0xFFL) << 32)
+                        | ((bytes[5] & 0xFFL) << 40)
+                        | ((bytes[6] & 0xFFL) << 48)
+                        | ((bytes[7] & 0xFFL) << 56);
 
                 lst.add(F.t(hash, node));
             }
-            catch (IgniteCheckedException e) {
-                throw new IgniteException(e);
-            }
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException(e);
         }
 
         Collections.sort(lst, COMPARATOR);
@@ -474,8 +496,18 @@ else if (affinityBackupFilter == null && backupFilter == null)
         Map<UUID, Collection<ClusterNode>> neighborhoodCache = exclNeighbors ?
             GridCacheUtils.neighbors(affCtx.currentTopologySnapshot()) : null;
 
+        MessageDigest d = digest.get();
+
+        List<ClusterNode> nodes = affCtx.currentTopologySnapshot();
+
+        Map<ClusterNode, byte[]> nodesHash = U.newHashMap(nodes.size());
+
         for (int i = 0; i < parts; i++) {
-            List<ClusterNode> partAssignment = assignPartition(i, affCtx.currentTopologySnapshot(), affCtx.backups(),
+            List<ClusterNode> partAssignment = assignPartition(d,
+                i,
+                nodes,
+                nodesHash,
+                affCtx.backups(),
                 neighborhoodCache);
 
             assignments.add(partAssignment);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index d24f900023a43..ddd4ee3e07b4c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -42,7 +42,7 @@
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -1620,17 +1620,6 @@ public Collection<ClusterNode> cacheNodes(AffinityTopologyVersion topVer) {
         return resolveDiscoCache(null, topVer).allNodesWithCaches(topVer.topologyVersion());
     }
 
-    /**
-     * Gets cache remote nodes for cache with given name.
-     *
-     * @param cacheName Cache name.
-     * @param topVer Topology version.
-     * @return Collection of cache nodes.
-     */
-    public Collection<ClusterNode> remoteCacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).remoteCacheNodes(cacheName, topVer.topologyVersion());
-    }
-
     /**
      * Gets cache remote nodes for cache with given name.
      *
@@ -1648,7 +1637,7 @@ public Collection<ClusterNode> remoteCacheNodes(AffinityTopologyVersion topVer)
      * @param topVer Topology version.
      * @return Collection of cache nodes.
      */
-    public Collection<ClusterNode> aliveCacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
+    Collection<ClusterNode> aliveCacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
         return resolveDiscoCache(cacheName, topVer).aliveCacheNodes(cacheName, topVer.topologyVersion());
     }
 
@@ -1659,38 +1648,20 @@ public Collection<ClusterNode> aliveCacheNodes(@Nullable String cacheName, Affin
      * @param topVer Topology version.
      * @return Collection of cache nodes.
      */
-    public Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
+    Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
         return resolveDiscoCache(cacheName, topVer).aliveRemoteCacheNodes(cacheName, topVer.topologyVersion());
     }
 
     /**
-     * Gets alive remote server nodes with at least one cache configured.
-     *
      * @param topVer Topology version (maximum allowed node order).
-     * @return Collection of alive cache nodes.
+     * @return Oldest alive server nodes with at least one cache configured.
      */
-    public Collection<ClusterNode> aliveRemoteServerNodesWithCaches(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).aliveRemoteServerNodesWithCaches(topVer.topologyVersion());
-    }
+    @Nullable public ClusterNode oldestAliveCacheServerNode(AffinityTopologyVersion topVer) {
+        DiscoCache cache = resolveDiscoCache(null, topVer);
 
-    /**
-     * Gets alive server nodes with at least one cache configured.
-     *
-     * @param topVer Topology version (maximum allowed node order).
-     * @return Collection of alive cache nodes.
-     */
-    public Collection<ClusterNode> aliveServerNodesWithCaches(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).aliveServerNodesWithCaches(topVer.topologyVersion());
-    }
+        Map.Entry<ClusterNode, Boolean> e = cache.aliveSrvNodesWithCaches.firstEntry();
 
-    /**
-     * Gets alive nodes with at least one cache configured.
-     *
-     * @param topVer Topology version (maximum allowed node order).
-     * @return Collection of alive cache nodes.
-     */
-    public Collection<ClusterNode> aliveNodesWithCaches(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).aliveNodesWithCaches(topVer.topologyVersion());
+        return e != null ? e.getKey() : null;
     }
 
     /**
@@ -2579,20 +2550,10 @@ private class DiscoCache {
          */
         private final ConcurrentMap<String, Collection<ClusterNode>> aliveRmtCacheNodes;
 
-        /**
-         * Cached alive remote nodes with caches.
-         */
-        private final Collection<ClusterNode> aliveNodesWithCaches;
-
         /**
          * Cached alive server remote nodes with caches.
          */
-        private final Collection<ClusterNode> aliveSrvNodesWithCaches;
-
-        /**
-         * Cached alive remote server nodes with caches.
-         */
-        private final Collection<ClusterNode> aliveRmtSrvNodesWithCaches;
+        private final ConcurrentSkipListMap<ClusterNode, Boolean> aliveSrvNodesWithCaches;
 
         /**
          * @param loc Local node.
@@ -2625,9 +2586,7 @@ private DiscoCache(ClusterNode loc, Collection<ClusterNode> rmts) {
 
             aliveCacheNodes = new ConcurrentHashMap8<>(allNodes.size(), 1.0f);
             aliveRmtCacheNodes = new ConcurrentHashMap8<>(allNodes.size(), 1.0f);
-            aliveNodesWithCaches = new ConcurrentSkipListSet<>();
-            aliveSrvNodesWithCaches = new ConcurrentSkipListSet<>();
-            aliveRmtSrvNodesWithCaches = new ConcurrentSkipListSet<>();
+            aliveSrvNodesWithCaches = new ConcurrentSkipListMap<>(GridNodeOrderComparator.INSTANCE);
             nodesByVer = new TreeMap<>();
 
             long maxOrder0 = 0;
@@ -2681,18 +2640,8 @@ private DiscoCache(ClusterNode loc, Collection<ClusterNode> rmts) {
                     }
                 }
 
-                if (hasCaches) {
-                    if (alive(node.id())) {
-                        aliveNodesWithCaches.add(node);
-
-                        if (!CU.clientNode(node)) {
-                            aliveSrvNodesWithCaches.add(node);
-
-                            if (!loc.id().equals(node.id()))
-                                aliveRmtSrvNodesWithCaches.add(node);
-                        }
-                    }
-                }
+                if (hasCaches && alive(node.id()) && !CU.clientNode(node))
+                    aliveSrvNodesWithCaches.put(node, Boolean.TRUE);
 
                 IgniteProductVersion nodeVer = U.productVersion(node);
 
@@ -2820,17 +2769,6 @@ Collection<ClusterNode> cacheNodes(@Nullable String cacheName, final long topVer
             return filter(topVer, allCacheNodes.get(cacheName));
         }
 
-        /**
-         * Gets all remote nodes that have cache with given name.
-         *
-         * @param cacheName Cache name.
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> remoteCacheNodes(@Nullable String cacheName, final long topVer) {
-            return filter(topVer, rmtCacheNodes.get(cacheName));
-        }
-
         /**
          * Gets all remote nodes that have at least one cache configured.
          *
@@ -2875,36 +2813,6 @@ Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, final
             return filter(topVer, aliveRmtCacheNodes.get(maskNull(cacheName)));
         }
 
-        /**
-         * Gets all alive remote server nodes with at least one cache configured.
-         *
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> aliveRemoteServerNodesWithCaches(final long topVer) {
-            return filter(topVer, aliveRmtSrvNodesWithCaches);
-        }
-
-        /**
-         * Gets all alive server nodes with at least one cache configured.
-         *
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> aliveServerNodesWithCaches(final long topVer) {
-            return filter(topVer, aliveSrvNodesWithCaches);
-        }
-
-        /**
-         * Gets all alive remote nodes with at least one cache configured.
-         *
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> aliveNodesWithCaches(final long topVer) {
-            return filter(topVer, aliveNodesWithCaches);
-        }
-
         /**
          * Checks if cache with given name has at least one node with near cache enabled.
          *
@@ -2928,9 +2836,7 @@ void updateAlives(ClusterNode leftNode) {
 
             filterNodeMap(aliveRmtCacheNodes, leftNode);
 
-            aliveNodesWithCaches.remove(leftNode);
             aliveSrvNodesWithCaches.remove(leftNode);
-            aliveRmtSrvNodesWithCaches.remove(leftNode);
         }
 
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 503b3348e51b4..7a24aa14ded91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -740,7 +740,7 @@ public void scheduleResendPartitions() {
      * Partition refresh callback.
      */
     private void refreshPartitions() {
-        ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, AffinityTopologyVersion.NONE);
+        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(AffinityTopologyVersion.NONE);
 
         if (oldest == null) {
             if (log.isDebugEnabled())
@@ -1224,7 +1224,7 @@ else if (!cacheCtx.isLocal())
                         top = cacheCtx.topology();
 
                     if (top != null)
-                        updated |= top.update(null, entry.getValue(), null) != null;
+                        updated |= top.update(null, entry.getValue(), null);
                 }
 
                 if (!cctx.kernalContext().clientNode() && updated)
@@ -1273,7 +1273,7 @@ else if (!cacheCtx.isLocal())
                         top = cacheCtx.topology();
 
                     if (top != null) {
-                        updated |= top.update(null, entry.getValue(), null) != null;
+                        updated |= top.update(null, entry.getValue(), null, true);
 
                         cctx.affinity().checkRebalanceState(top, cacheId);
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 90e428ca10b80..d32f4c1867bcf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -489,23 +489,6 @@ public static boolean isNearEnabled(CacheConfiguration cfg) {
         return cfg.getNearConfiguration() != null;
     }
 
-    /**
-     * Gets oldest alive server node with at least one cache configured for specified topology version.
-     *
-     * @param ctx Context.
-     * @param topVer Maximum allowed topology version.
-     * @return Oldest alive cache server node.
-     */
-    @Nullable public static ClusterNode oldestAliveCacheServerNode(GridCacheSharedContext ctx,
-        AffinityTopologyVersion topVer) {
-        Collection<ClusterNode> nodes = ctx.discovery().aliveServerNodesWithCaches(topVer);
-
-        if (nodes.isEmpty())
-            return null;
-
-        return oldest(nodes);
-    }
-
     /**
      * @param nodes Nodes.
      * @return Oldest node for the given topology version.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 568a4dae683cc..1d60c422a6417 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -304,8 +304,7 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
             assert !metaDataCache.context().affinityNode();
 
             while (true) {
-                ClusterNode oldestSrvNode =
-                    CU.oldestAliveCacheServerNode(ctx.cache().context(), AffinityTopologyVersion.NONE);
+                ClusterNode oldestSrvNode = ctx.discovery().oldestAliveCacheServerNode(AffinityTopologyVersion.NONE);
 
                 if (oldestSrvNode == null)
                     break;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 5efb31710a8a5..816132d351139 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -271,7 +271,7 @@ private void beforeExchange0(ClusterNode loc, GridDhtPartitionsExchangeFuture ex
             removeNode(exchId.nodeId());
 
         // In case if node joins, get topology at the time of joining node.
-        ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, topVer);
+        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
 
         assert oldest != null;
 
@@ -536,7 +536,8 @@ public long lastUpdateSequence() {
 
         try {
             assert node2part != null && node2part.valid() : "Invalid node2part [node2part: " + node2part +
-                ", locNodeId=" + cctx.localNodeId() + ", gridName=" + cctx.gridName() + ']';
+                ", locNodeId=" + cctx.localNodeId() +
+                ", gridName=" + cctx.gridName() + ']';
 
             GridDhtPartitionFullMap m = node2part;
 
@@ -549,7 +550,7 @@ public long lastUpdateSequence() {
 
     /** {@inheritDoc} */
     @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-    @Nullable @Override public GridDhtPartitionMap2 update(@Nullable GridDhtPartitionExchangeId exchId,
+    @Nullable @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         GridDhtPartitionFullMap partMap,
         Map<Integer, Long> cntrMap) {
         if (log.isDebugEnabled())
@@ -563,7 +564,7 @@ public long lastUpdateSequence() {
                     log.debug("Stale exchange id for full partition map update (will ignore) [lastExchId=" +
                         lastExchangeId + ", exchId=" + exchId + ']');
 
-                return null;
+                return false;
             }
 
             if (node2part != null && node2part.compareTo(partMap) >= 0) {
@@ -571,7 +572,7 @@ public long lastUpdateSequence() {
                     log.debug("Stale partition map for full partition map update (will ignore) [lastExchId=" +
                         lastExchangeId + ", exchId=" + exchId + ", curMap=" + node2part + ", newMap=" + partMap + ']');
 
-                return null;
+                return false;
             }
 
             updateSeq.incrementAndGet();
@@ -634,7 +635,7 @@ public long lastUpdateSequence() {
             if (log.isDebugEnabled())
                 log.debug("Partition map after full update: " + fullMapString());
 
-            return null;
+            return false;
         }
         finally {
             lock.writeLock().unlock();
@@ -642,10 +643,10 @@ public long lastUpdateSequence() {
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-    @Nullable @Override public GridDhtPartitionMap2 update(@Nullable GridDhtPartitionExchangeId exchId,
+    @Nullable @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         GridDhtPartitionMap2 parts,
-        Map<Integer, Long> cntrMap) {
+        Map<Integer, Long> cntrMap,
+        boolean checkEvictions) {
         if (log.isDebugEnabled())
             log.debug("Updating single partition map [exchId=" + exchId + ", parts=" + mapString(parts) + ']');
 
@@ -654,29 +655,27 @@ public long lastUpdateSequence() {
                 log.debug("Received partition update for non-existing node (will ignore) [exchId=" + exchId +
                     ", parts=" + parts + ']');
 
-            return null;
+            return false;
         }
 
         lock.writeLock().lock();
 
         try {
             if (stopping)
-                return null;
+                return false;
 
             if (lastExchangeId != null && exchId != null && lastExchangeId.compareTo(exchId) > 0) {
                 if (log.isDebugEnabled())
                     log.debug("Stale exchange id for single partition map update (will ignore) [lastExchId=" +
                         lastExchangeId + ", exchId=" + exchId + ']');
 
-                return null;
+                return false;
             }
 
             if (exchId != null)
                 lastExchangeId = exchId;
 
             if (node2part == null) {
-                U.dumpStack(log, "Created invalid: " + node2part);
-
                 // Create invalid partition map.
                 node2part = new GridDhtPartitionFullMap();
             }
@@ -688,43 +687,45 @@ public long lastUpdateSequence() {
                     log.debug("Stale update sequence for single partition map update (will ignore) [exchId=" + exchId +
                         ", curSeq=" + cur.updateSequence() + ", newSeq=" + parts.updateSequence() + ']');
 
-                return null;
+                return false;
             }
 
             long updateSeq = this.updateSeq.incrementAndGet();
 
-            node2part = new GridDhtPartitionFullMap(node2part, updateSeq);
-
-            boolean changed = false;
+            node2part.updateSequence(updateSeq);
 
-            if (cur == null || !cur.equals(parts))
-                changed = true;
+            boolean changed = cur == null || !cur.equals(parts);
 
-            node2part.put(parts.nodeId(), parts);
+            if (changed) {
+                node2part.put(parts.nodeId(), parts);
 
-            part2node = new HashMap<>(part2node);
+                // Add new mappings.
+                for (Integer p : parts.keySet()) {
+                    Set<UUID> ids = part2node.get(p);
 
-            // Add new mappings.
-            for (Integer p : parts.keySet()) {
-                Set<UUID> ids = part2node.get(p);
+                    if (ids == null)
+                        // Initialize HashSet to size 3 in anticipation that there won't be
+                        // more than 3 nodes per partition.
+                        part2node.put(p, ids = U.newHashSet(3));
 
-                if (ids == null)
-                    // Initialize HashSet to size 3 in anticipation that there won't be
-                    // more than 3 nodes per partition.
-                    part2node.put(p, ids = U.newHashSet(3));
+                    ids.add(parts.nodeId());
+                }
 
-                changed |= ids.add(parts.nodeId());
-            }
+                // Remove obsolete mappings.
+                if (cur != null) {
+                    for (Integer p : cur.keySet()) {
+                        if (parts.containsKey(p))
+                            continue;
 
-            // Remove obsolete mappings.
-            if (cur != null) {
-                for (Integer p : F.view(cur.keySet(), F0.notIn(parts.keySet()))) {
-                    Set<UUID> ids = part2node.get(p);
+                        Set<UUID> ids = part2node.get(p);
 
-                    if (ids != null)
-                        changed |= ids.remove(parts.nodeId());
+                        if (ids != null)
+                            ids.remove(parts.nodeId());
+                    }
                 }
             }
+            else
+                cur.updateSequence(parts.updateSequence(), parts.topologyVersion());
 
             if (cntrMap != null) {
                 for (Map.Entry<Integer, Long> e : cntrMap.entrySet()) {
@@ -740,13 +741,18 @@ public long lastUpdateSequence() {
             if (log.isDebugEnabled())
                 log.debug("Partition map after single update: " + fullMapString());
 
-            return changed ? localPartitionMap() : null;
+            return changed;
         }
         finally {
             lock.writeLock().unlock();
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void checkEvictions() {
+        // No-op.
+    }
+
     /**
      * Updates value for single partition.
      *
@@ -755,13 +761,12 @@ public long lastUpdateSequence() {
      * @param state State.
      * @param updateSeq Update sequence.
      */
-    @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
     private void updateLocal(int p, UUID nodeId, GridDhtPartitionState state, long updateSeq) {
         assert lock.isWriteLockedByCurrentThread();
         assert nodeId.equals(cctx.localNodeId());
 
         // In case if node joins, get topology at the time of joining node.
-        ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, topVer);
+        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
 
         // If this node became the oldest node.
         if (oldest.id().equals(cctx.localNodeId())) {
@@ -811,7 +816,7 @@ private void removeNode(UUID nodeId) {
         assert nodeId != null;
         assert lock.writeLock().isHeldByCurrentThread();
 
-        ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, topVer);
+        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
 
         ClusterNode loc = cctx.localNode();
 
@@ -876,18 +881,6 @@ private void removeNode(UUID nodeId) {
         }
     }
 
-    /** {@inheritDoc} */
-    @Nullable @Override public GridDhtPartitionMap2 partitions(UUID nodeId) {
-        lock.readLock().lock();
-
-        try {
-            return node2part.get(nodeId);
-        }
-        finally {
-            lock.readLock().unlock();
-        }
-    }
-
     /** {@inheritDoc} */
     @Override public Map<Integer, Long> updateCounters(boolean skipZeros) {
         lock.readLock().lock();
@@ -918,6 +911,27 @@ private void removeNode(UUID nodeId) {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean hasMovingPartitions() {
+        lock.readLock().lock();
+
+        try {
+            assert node2part != null && node2part.valid() : "Invalid node2part [node2part: " + node2part +
+                ", locNodeId=" + cctx.localNodeId() +
+                ", gridName=" + cctx.gridName() + ']';
+
+            for (GridDhtPartitionMap2 map : node2part.values()) {
+                if (map.hasMovingPartitions())
+                    return true;
+            }
+
+            return false;
+        }
+        finally {
+            lock.readLock().unlock();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void printMemoryStats(int threshold) {
         X.println(">>>  Cache partition topology stats [grid=" + cctx.gridName() + ", cacheId=" + cacheId + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 39a3e08cc5c51..668a1cdc006d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -479,7 +479,6 @@ IgniteInternalFuture<?> rent(boolean updateSeq) {
 
         if ((reservations & 0xFFFF) == 0 && casState(reservations, RENTING)) {
                 shouldBeRenting = false;
-
             if (log.isDebugEnabled())
                 log.debug("Moved partition to RENTING state: " + this);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 0f75a5db47f3a..14ce1f9589ff1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -194,6 +194,11 @@ public void beforeExchange(GridDhtPartitionsExchangeFuture exchFut, boolean affR
      */
     public GridDhtPartitionFullMap partitionMap(boolean onlyActive);
 
+    /**
+     * @return {@code True} If one of cache nodes has partitions in {@link GridDhtPartitionState#MOVING} state.
+     */
+    public boolean hasMovingPartitions();
+
     /**
      * @param e Entry removed from cache.
      */
@@ -203,9 +208,9 @@ public void beforeExchange(GridDhtPartitionsExchangeFuture exchFut, boolean affR
      * @param exchId Exchange ID.
      * @param partMap Update partition map.
      * @param cntrMap Partition update counters.
-     * @return Local partition map if there were evictions or {@code null} otherwise.
+     * @return {@code True} if topology state changed.
      */
-    public GridDhtPartitionMap2 update(@Nullable GridDhtPartitionExchangeId exchId,
+    public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         GridDhtPartitionFullMap partMap,
         @Nullable Map<Integer, Long> cntrMap);
 
@@ -213,11 +218,18 @@ public GridDhtPartitionMap2 update(@Nullable GridDhtPartitionExchangeId exchId,
      * @param exchId Exchange ID.
      * @param parts Partitions.
      * @param cntrMap Partition update counters.
-     * @return Local partition map if there were evictions or {@code null} otherwise.
+     * @param checkEvictions Check evictions flag.
+     * @return {@code True} if topology state changed.
      */
-    @Nullable public GridDhtPartitionMap2 update(@Nullable GridDhtPartitionExchangeId exchId,
+    @Nullable public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         GridDhtPartitionMap2 parts,
-        @Nullable Map<Integer, Long> cntrMap);
+        @Nullable Map<Integer, Long> cntrMap,
+        boolean checkEvictions);
+
+    /**
+     *
+     */
+    public void checkEvictions();
 
     /**
      * @param skipZeros If {@code true} then filters out zero counters.
@@ -237,12 +249,6 @@ public GridDhtPartitionMap2 update(@Nullable GridDhtPartitionExchangeId exchId,
      */
     public void onEvicted(GridDhtLocalPartition part, boolean updateSeq);
 
-    /**
-     * @param nodeId Node to get partitions for.
-     * @return Partitions for node.
-     */
-    @Nullable public GridDhtPartitionMap2 partitions(UUID nodeId);
-
     /**
      * Prints memory stats.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index ab573bd23c152..1b4dcc9293dee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -44,7 +44,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap2;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
-import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.GridAtomicLong;
 import org.apache.ignite.internal.util.StripedCompositeReadWriteLock;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -330,8 +329,9 @@ private boolean waitForRent() throws IgniteCheckedException {
     }
 
     /** {@inheritDoc} */
-    @Override public void initPartitions(
-        GridDhtPartitionsExchangeFuture exchFut) throws IgniteInterruptedCheckedException {
+    @Override public void initPartitions(GridDhtPartitionsExchangeFuture exchFut)
+        throws IgniteInterruptedCheckedException
+    {
         U.writeLock(lock);
 
         try {
@@ -356,9 +356,7 @@ private boolean waitForRent() throws IgniteCheckedException {
     private void initPartitions0(GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
         ClusterNode loc = cctx.localNode();
 
-        ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx.shared(), topVer);
-
-        assert oldest != null || cctx.kernalContext().clientNode();
+        ClusterNode oldest = currentCoordinator();
 
         GridDhtPartitionExchangeId exchId = exchFut.exchangeId();
 
@@ -397,7 +395,7 @@ private void initPartitions0(GridDhtPartitionsExchangeFuture exchFut, long updat
                         if (log.isDebugEnabled())
                             log.debug("Owned partition for oldest node: " + locPart);
 
-                        updateLocal(p, loc.id(), locPart.state(), updateSeq);
+                        updateSeq = updateLocal(p, locPart.state(), updateSeq);
                     }
                 }
             }
@@ -419,7 +417,7 @@ private void initPartitions0(GridDhtPartitionsExchangeFuture exchFut, long updat
                         if (state.active()) {
                             locPart.rent(false);
 
-                            updateLocal(p, loc.id(), locPart.state(), updateSeq);
+                            updateSeq = updateLocal(p, locPart.state(), updateSeq);
 
                             if (log.isDebugEnabled())
                                 log.debug("Evicting partition with rebalancing disabled " +
@@ -443,8 +441,6 @@ else if (belongs)
      * @param updateSeq Update sequence.
      */
     private void createPartitions(List<List<ClusterNode>> aff, long updateSeq) {
-        ClusterNode loc = cctx.localNode();
-
         int num = cctx.affinity().partitions();
 
         for (int p = 0; p < num; p++) {
@@ -454,7 +450,7 @@ private void createPartitions(List<List<ClusterNode>> aff, long updateSeq) {
                     // will be created in MOVING state.
                     GridDhtLocalPartition locPart = createPartition(p);
 
-                    updateLocal(p, loc.id(), locPart.state(), updateSeq);
+                    updateSeq = updateLocal(p, locPart.state(), updateSeq);
                 }
             }
             // If this node's map is empty, we pre-create local partitions,
@@ -485,10 +481,7 @@ else if (localNode(p, aff))
             if (exchId.isLeft())
                 removeNode(exchId.nodeId());
 
-            // In case if node joins, get topology at the time of joining node.
-            ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx.shared(), topVer);
-
-            assert oldest != null || cctx.kernalContext().clientNode();
+            ClusterNode oldest = currentCoordinator();
 
             if (log.isDebugEnabled())
                 log.debug("Partition map beforeExchange [exchId=" + exchId + ", fullMap=" + fullMapString() + ']');
@@ -548,8 +541,6 @@ else if (!node2part.nodeId().equals(loc.id())) {
     @Override public boolean afterExchange(GridDhtPartitionsExchangeFuture exchFut) throws IgniteCheckedException {
         boolean changed = waitForRent();
 
-        ClusterNode loc = cctx.localNode();
-
         int num = cctx.affinity().partitions();
 
         AffinityTopologyVersion topVer = exchFut.topologyVersion();
@@ -600,7 +591,7 @@ else if (!node2part.nodeId().equals(loc.id())) {
                                 assert owned : "Failed to own partition [cacheName" + cctx.name() + ", locPart=" +
                                     locPart + ']';
 
-                                updateLocal(p, loc.id(), locPart.state(), updateSeq);
+                                updateSeq = updateLocal(p, locPart.state(), updateSeq);
 
                                 changed = true;
 
@@ -620,7 +611,7 @@ else if (log.isDebugEnabled())
                                     locPart + ", owners = " + owners + ']');
                         }
                         else
-                            updateLocal(p, loc.id(), locPart.state(), updateSeq);
+                            updateSeq = updateLocal(p, locPart.state(), updateSeq);
                     }
                 }
                 else {
@@ -630,7 +621,7 @@ else if (log.isDebugEnabled())
                         if (state == MOVING) {
                             locPart.rent(false);
 
-                            updateLocal(p, loc.id(), locPart.state(), updateSeq);
+                            updateSeq = updateLocal(p, locPart.state(), updateSeq);
 
                             changed = true;
 
@@ -803,8 +794,11 @@ private GridDhtLocalPartition localPartition(int p,
                 map.put(i, part.state());
             }
 
-            return new GridDhtPartitionMap2(cctx.nodeId(), updateSeq.get(), topVer,
-                Collections.unmodifiableMap(map), true);
+            return new GridDhtPartitionMap2(cctx.nodeId(),
+                updateSeq.get(),
+                topVer,
+                Collections.unmodifiableMap(map),
+                true);
         }
         finally {
             lock.readLock().unlock();
@@ -985,7 +979,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
 
     /** {@inheritDoc} */
     @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-    @Nullable @Override public GridDhtPartitionMap2 update(@Nullable GridDhtPartitionExchangeId exchId,
+    @Nullable @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         GridDhtPartitionFullMap partMap,
         @Nullable Map<Integer, Long> cntrMap) {
         if (log.isDebugEnabled())
@@ -997,7 +991,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
 
         try {
             if (stopping)
-                return null;
+                return false;
 
             if (cntrMap != null) {
                 for (Map.Entry<Integer, Long> e : cntrMap.entrySet()) {
@@ -1025,7 +1019,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
                     log.debug("Stale exchange id for full partition map update (will ignore) [lastExchId=" +
                         lastExchangeId + ", exchId=" + exchId + ']');
 
-                return null;
+                return false;
             }
 
             if (node2part != null && node2part.compareTo(partMap) >= 0) {
@@ -1033,7 +1027,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
                     log.debug("Stale partition map for full partition map update (will ignore) [lastExchId=" +
                         lastExchangeId + ", exchId=" + exchId + ", curMap=" + node2part + ", newMap=" + partMap + ']');
 
-                return null;
+                return false;
             }
 
             long updateSeq = this.updateSeq.incrementAndGet();
@@ -1076,7 +1070,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
 
             node2part = partMap;
 
-            Map<Integer, Set<UUID>> p2n = new HashMap<>(cctx.affinity().partitions(), 1.0f);
+            Map<Integer, Set<UUID>> p2n = U.newHashMap(cctx.affinity().partitions());
 
             for (Map.Entry<UUID, GridDhtPartitionMap2> e : partMap.entrySet()) {
                 for (Integer p : e.getValue().keySet()) {
@@ -1110,7 +1104,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
             if (log.isDebugEnabled())
                 log.debug("Partition map after full update: " + fullMapString());
 
-            return changed ? localPartitionMap() : null;
+            return changed;
         }
         finally {
             lock.writeLock().unlock();
@@ -1118,10 +1112,10 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-    @Nullable @Override public GridDhtPartitionMap2 update(@Nullable GridDhtPartitionExchangeId exchId,
+    @Nullable @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         GridDhtPartitionMap2 parts,
-        @Nullable Map<Integer, Long> cntrMap) {
+        @Nullable Map<Integer, Long> cntrMap,
+        boolean checkEvictions) {
         if (log.isDebugEnabled())
             log.debug("Updating single partition map [exchId=" + exchId + ", parts=" + mapString(parts) + ']');
 
@@ -1130,33 +1124,28 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
                 log.debug("Received partition update for non-existing node (will ignore) [exchId=" + exchId +
                     ", parts=" + parts + ']');
 
-            return null;
+            return false;
         }
 
         lock.writeLock().lock();
 
         try {
             if (stopping)
-                return null;
+                return false;
 
             if (cntrMap != null) {
                 for (Map.Entry<Integer, Long> e : cntrMap.entrySet()) {
-                    Long cntr = this.cntrMap.get(e.getKey());
+                    Integer p = e.getKey();
 
-                    if (cntr == null || cntr < e.getValue())
-                        this.cntrMap.put(e.getKey(), e.getValue());
-                }
-
-                for (int i = 0; i < locParts.length(); i++) {
-                    GridDhtLocalPartition part = locParts.get(i);
+                    Long cntr = this.cntrMap.get(p);
 
-                    if (part == null)
-                        continue;
+                    if (cntr == null || cntr < e.getValue())
+                        this.cntrMap.put(p, e.getValue());
 
-                    Long cntr = cntrMap.get(part.id());
+                    GridDhtLocalPartition part = locParts.get(p);
 
-                    if (cntr != null)
-                        part.updateCounter(cntr);
+                    if (part != null)
+                        part.updateCounter(e.getValue());
                 }
             }
 
@@ -1165,7 +1154,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
                     log.debug("Stale exchange id for single partition map update (will ignore) [lastExchId=" +
                         lastExchangeId + ", exchId=" + exchId + ']');
 
-                return null;
+                return false;
             }
 
             if (exchId != null)
@@ -1182,60 +1171,91 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
                     log.debug("Stale update sequence for single partition map update (will ignore) [exchId=" + exchId +
                         ", curSeq=" + cur.updateSequence() + ", newSeq=" + parts.updateSequence() + ']');
 
-                return null;
+                return false;
             }
 
             long updateSeq = this.updateSeq.incrementAndGet();
 
-            node2part = new GridDhtPartitionFullMap(node2part, updateSeq);
-
-            boolean changed = false;
+            node2part.newUpdateSequence(updateSeq);
 
-            if (cur == null || !cur.equals(parts))
-                changed = true;
+            boolean changed = cur == null || !cur.equals(parts);
 
-            node2part.put(parts.nodeId(), parts);
+            if (changed) {
+                node2part.put(parts.nodeId(), parts);
 
-            part2node = new HashMap<>(part2node);
+                // Add new mappings.
+                for (Integer p : parts.keySet()) {
+                    Set<UUID> ids = part2node.get(p);
 
-            // Add new mappings.
-            for (Integer p : parts.keySet()) {
-                Set<UUID> ids = part2node.get(p);
+                    if (ids == null)
+                        // Initialize HashSet to size 3 in anticipation that there won't be
+                        // more than 3 nodes per partition.
+                        part2node.put(p, ids = U.newHashSet(3));
 
-                if (ids == null)
-                    // Initialize HashSet to size 3 in anticipation that there won't be
-                    // more than 3 nodes per partition.
-                    part2node.put(p, ids = U.newHashSet(3));
+                    ids.add(parts.nodeId());
+                }
 
-                changed |= ids.add(parts.nodeId());
-            }
+                // Remove obsolete mappings.
+                if (cur != null) {
+                    for (Integer p : cur.keySet()) {
+                        if (parts.containsKey(p))
+                            continue;
 
-            // Remove obsolete mappings.
-            if (cur != null) {
-                for (Integer p : F.view(cur.keySet(), F0.notIn(parts.keySet()))) {
-                    Set<UUID> ids = part2node.get(p);
+                        Set<UUID> ids = part2node.get(p);
 
-                    if (ids != null)
-                        changed |= ids.remove(parts.nodeId());
+                        if (ids != null)
+                            ids.remove(parts.nodeId());
+                    }
                 }
             }
+            else
+                cur.updateSequence(parts.updateSequence(), parts.topologyVersion());
 
-            AffinityTopologyVersion affVer = cctx.affinity().affinityTopologyVersion();
-
-            if (!affVer.equals(AffinityTopologyVersion.NONE) && affVer.compareTo(topVer) >= 0) {
-                List<List<ClusterNode>> aff = cctx.affinity().assignments(topVer);
-
-                changed |= checkEvictions(updateSeq, aff);
-
-                updateRebalanceVersion(aff);
-            }
+            if (checkEvictions)
+                changed |= checkEvictions(updateSeq);
 
             consistencyCheck();
 
             if (log.isDebugEnabled())
                 log.debug("Partition map after single update: " + fullMapString());
 
-            return changed ? localPartitionMap() : null;
+            return changed;
+        }
+        finally {
+            lock.writeLock().unlock();
+        }
+    }
+
+    /**
+     * @param updateSeq Update sequence.
+     * @return {@code True} if state changed.
+     */
+    private boolean checkEvictions(long updateSeq) {
+        AffinityTopologyVersion affVer = cctx.affinity().affinityTopologyVersion();
+
+        boolean changed = false;
+
+        if (!affVer.equals(AffinityTopologyVersion.NONE) && affVer.compareTo(topVer) >= 0) {
+            List<List<ClusterNode>> aff = cctx.affinity().assignments(topVer);
+
+            changed = checkEvictions(updateSeq, aff);
+
+            updateRebalanceVersion(aff);
+        }
+
+        return changed;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void checkEvictions() {
+        lock.writeLock().lock();
+
+        try {
+            long updateSeq = this.updateSeq.incrementAndGet();
+
+            node2part.newUpdateSequence(updateSeq);
+
+            checkEvictions(updateSeq);
         }
         finally {
             lock.writeLock().unlock();
@@ -1270,7 +1290,7 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
                     if (nodeIds.containsAll(F.nodeIds(affNodes))) {
                         part.rent(false);
 
-                        updateLocal(part.id(), locId, part.state(), updateSeq);
+                        updateSeq = updateLocal(part.id(), part.state(), updateSeq);
 
                         changed = true;
 
@@ -1295,7 +1315,7 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
                                 if (locId.equals(n.id())) {
                                     part.rent(false);
 
-                                    updateLocal(part.id(), locId, part.state(), updateSeq);
+                                    updateSeq = updateLocal(part.id(), part.state(), updateSeq);
 
                                     changed = true;
 
@@ -1315,20 +1335,28 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
         return changed;
     }
 
+    /**
+     * @return Current coordinator node.
+     */
+    @Nullable private ClusterNode currentCoordinator() {
+        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
+
+        assert oldest != null || cctx.kernalContext().clientNode();
+
+        return oldest;
+    }
+
     /**
      * Updates value for single partition.
      *
      * @param p Partition.
-     * @param nodeId Node ID.
      * @param state State.
      * @param updateSeq Update sequence.
+     * @return Update sequence.
      */
     @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-    private void updateLocal(int p, UUID nodeId, GridDhtPartitionState state, long updateSeq) {
-        assert nodeId.equals(cctx.nodeId());
-
-        // In case if node joins, get topology at the time of joining node.
-        ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx.shared(), topVer);
+    private long updateLocal(int p, GridDhtPartitionState state, long updateSeq) {
+        ClusterNode oldest = currentCoordinator();
 
         assert oldest != null || cctx.kernalContext().clientNode();
 
@@ -1338,12 +1366,16 @@ private void updateLocal(int p, UUID nodeId, GridDhtPartitionState state, long u
 
             if (seq != updateSeq) {
                 if (seq > updateSeq) {
-                    if (this.updateSeq.get() < seq) {
+                    long seq0 = this.updateSeq.get();
+
+                    if (seq0 < seq) {
                         // Update global counter if necessary.
-                        boolean b = this.updateSeq.compareAndSet(this.updateSeq.get(), seq + 1);
+                        boolean b = this.updateSeq.compareAndSet(seq0, seq + 1);
 
-                        assert b : "Invalid update sequence [updateSeq=" + updateSeq + ", seq=" + seq +
-                            ", curUpdateSeq=" + this.updateSeq.get() + ", node2part=" + node2part.toFullString() + ']';
+                        assert b : "Invalid update sequence [updateSeq=" + updateSeq +
+                            ", seq=" + seq +
+                            ", curUpdateSeq=" + this.updateSeq.get() +
+                            ", node2part=" + node2part.toFullString() + ']';
 
                         updateSeq = seq + 1;
                     }
@@ -1355,11 +1387,19 @@ private void updateLocal(int p, UUID nodeId, GridDhtPartitionState state, long u
             }
         }
 
-        GridDhtPartitionMap2 map = node2part.get(nodeId);
+        UUID locNodeId = cctx.localNodeId();
+
+        GridDhtPartitionMap2 map = node2part.get(locNodeId);
 
-        if (map == null)
-            node2part.put(nodeId, map = new GridDhtPartitionMap2(nodeId, updateSeq, topVer,
-                Collections.<Integer, GridDhtPartitionState>emptyMap(), false));
+        if (map == null) {
+            map = new GridDhtPartitionMap2(locNodeId,
+                updateSeq,
+                topVer,
+                Collections.<Integer, GridDhtPartitionState>emptyMap(),
+                false);
+
+            node2part.put(locNodeId, map);
+        }
 
         map.updateSequence(updateSeq, topVer);
 
@@ -1370,7 +1410,9 @@ private void updateLocal(int p, UUID nodeId, GridDhtPartitionState state, long u
         if (ids == null)
             part2node.put(p, ids = U.newHashSet(3));
 
-        ids.add(nodeId);
+        ids.add(locNodeId);
+
+        return updateSeq;
     }
 
     /**
@@ -1395,8 +1437,6 @@ private void removeNode(UUID nodeId) {
             else
                 node2part = new GridDhtPartitionFullMap(node2part, node2part.updateSequence());
 
-            part2node = new HashMap<>(part2node);
-
             GridDhtPartitionMap2 parts = node2part.remove(nodeId);
 
             if (parts != null) {
@@ -1418,13 +1458,11 @@ private void removeNode(UUID nodeId) {
 
     /** {@inheritDoc} */
     @Override public boolean own(GridDhtLocalPartition part) {
-        ClusterNode loc = cctx.localNode();
-
         lock.writeLock().lock();
 
         try {
             if (part.own()) {
-                updateLocal(part.id(), loc.id(), part.state(), updateSeq.incrementAndGet());
+                updateLocal(part.id(), part.state(), updateSeq.incrementAndGet());
 
                 consistencyCheck();
 
@@ -1452,7 +1490,7 @@ private void removeNode(UUID nodeId) {
 
             long seq = updateSeq ? this.updateSeq.incrementAndGet() : this.updateSeq.get();
 
-            updateLocal(part.id(), cctx.localNodeId(), part.state(), seq);
+            updateLocal(part.id(), part.state(), seq);
 
             consistencyCheck();
         }
@@ -1461,18 +1499,6 @@ private void removeNode(UUID nodeId) {
         }
     }
 
-    /** {@inheritDoc} */
-    @Nullable @Override public GridDhtPartitionMap2 partitions(UUID nodeId) {
-        lock.readLock().lock();
-
-        try {
-            return node2part.get(nodeId);
-        }
-        finally {
-            lock.readLock().unlock();
-        }
-    }
-
     /** {@inheritDoc} */
     @Override public Map<Integer, Long> updateCounters(boolean skipZeros) {
         lock.readLock().lock();
@@ -1525,6 +1551,30 @@ private void removeNode(UUID nodeId) {
         return curTopVer.equals(topVer) && curTopVer.equals(rebalancedTopVer);
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean hasMovingPartitions() {
+        lock.readLock().lock();
+
+        try {
+            assert node2part != null && node2part.valid() : "Invalid node2part [node2part: " + node2part +
+                ", cache=" + cctx.name() +
+                ", started=" + cctx.started() +
+                ", stopping=" + stopping +
+                ", locNodeId=" + cctx.localNode().id() +
+                ", locName=" + cctx.gridName() + ']';
+
+            for (GridDhtPartitionMap2 map : node2part.values()) {
+                if (map.hasMovingPartitions())
+                    return true;
+            }
+
+            return false;
+        }
+        finally {
+            lock.readLock().unlock();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void printMemoryStats(int threshold) {
         X.println(">>>  Cache partition topology stats [grid=" + cctx.gridName() + ", cache=" + cctx.name() + ']');
@@ -1607,10 +1657,12 @@ private boolean hasState(final int p, @Nullable UUID nodeId, final GridDhtPartit
             if (state == match)
                 return true;
 
-            if (matches != null && matches.length > 0)
-                for (GridDhtPartitionState s : matches)
+            if (matches != null && matches.length > 0) {
+                for (GridDhtPartitionState s : matches) {
                     if (state == s)
                         return true;
+                }
+            }
         }
 
         return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java
index 8f5ad1714225c..e8860f25b2ab8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java
@@ -103,10 +103,13 @@ public GridDhtPartitionFullMap(UUID nodeId, long nodeOrder, long updateSeq, Map<
         for (Map.Entry<UUID, GridDhtPartitionMap2> e : m.entrySet()) {
             GridDhtPartitionMap2 part = e.getValue();
 
-            if (onlyActive)
-                put(e.getKey(), new GridDhtPartitionMap2(part.nodeId(), part.updateSequence(), part.topologyVersion(), part.map(), true));
-            else
-                put(e.getKey(), part);
+            GridDhtPartitionMap2 cpy = new GridDhtPartitionMap2(part.nodeId(),
+                part.updateSequence(),
+                part.topologyVersion(),
+                part.map(),
+                onlyActive);
+
+            put(e.getKey(), cpy);
         }
     }
 
@@ -175,6 +178,13 @@ public boolean partitionStateEquals(GridDhtPartitionFullMap fullMap) {
         return true;
     }
 
+    /**
+     * @param updateSeq New update sequence value.
+     */
+    public void newUpdateSequence(long updateSeq) {
+        this.updateSeq = updateSeq;
+    }
+
     /**
      * @param updateSeq New update sequence value.
      * @return Old update sequence value.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index f391265039d99..e945de958a339 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -110,6 +110,10 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     @GridToStringExclude
     private final Set<UUID> remaining = new HashSet<>();
 
+    /** */
+    @GridToStringExclude
+    private int pendingSingleUpdates;
+
     /** */
     @GridToStringExclude
     private List<ClusterNode> srvNodes;
@@ -1162,13 +1166,16 @@ public void onReceive(final ClusterNode node, final GridDhtPartitionsSingleMessa
      */
     private void processMessage(ClusterNode node, GridDhtPartitionsSingleMessage msg) {
         boolean allReceived = false;
+        boolean updateSingleMap = false;
 
         synchronized (mux) {
             assert crd != null;
 
             if (crd.isLocal()) {
                 if (remaining.remove(node.id())) {
-                    updatePartitionSingleMap(msg);
+                    updateSingleMap = true;
+
+                    pendingSingleUpdates++;
 
                     allReceived = remaining.isEmpty();
                 }
@@ -1177,8 +1184,42 @@ private void processMessage(ClusterNode node, GridDhtPartitionsSingleMessage msg
                 singleMsgs.put(node, msg);
         }
 
-        if (allReceived)
+        if (updateSingleMap) {
+            try {
+                updatePartitionSingleMap(msg);
+            }
+            finally {
+                synchronized (mux) {
+                    assert pendingSingleUpdates > 0;
+
+                    pendingSingleUpdates--;
+
+                    if (pendingSingleUpdates == 0)
+                        mux.notifyAll();
+                }
+            }
+        }
+
+        if (allReceived) {
+            awaitSingleMapUpdates();
+
             onAllReceived(false);
+        }
+    }
+
+    /**
+     *
+     */
+    private void awaitSingleMapUpdates() {
+        synchronized (mux) {
+            try {
+                while (pendingSingleUpdates > 0)
+                    U.wait(mux);
+            }
+            catch (IgniteInterruptedCheckedException e) {
+                U.warn(log, "Failed to wait for partition map updates, thread was interrupted: " + e);
+            }
+        }
     }
 
     /**
@@ -1218,6 +1259,11 @@ private void onAllReceived(boolean discoThread) {
                 }
             }
 
+            for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
+                if (!cacheCtx.isLocal())
+                    cacheCtx.topology().checkEvictions();
+            }
+
             updateLastVersion(cctx.versions().last());
 
             cctx.versions().onExchange(lastVer.get().order());
@@ -1374,7 +1420,7 @@ private void updatePartitionFullMap(GridDhtPartitionsFullMessage msg) {
             if (cacheCtx != null)
                 cacheCtx.topology().update(exchId, entry.getValue(), cntrMap);
             else {
-                ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, AffinityTopologyVersion.NONE);
+                ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(AffinityTopologyVersion.NONE);
 
                 if (oldest != null && oldest.isLocal())
                     cctx.exchange().clientTopology(cacheId, this).update(exchId, entry.getValue(), cntrMap);
@@ -1395,7 +1441,7 @@ private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
             GridDhtPartitionTopology top = cacheCtx != null ? cacheCtx.topology() :
                 cctx.exchange().clientTopology(cacheId, this);
 
-            top.update(exchId, entry.getValue(), msg.partitionUpdateCounters(cacheId));
+            top.update(exchId, entry.getValue(), msg.partitionUpdateCounters(cacheId), false);
         }
     }
 
@@ -1557,6 +1603,8 @@ public void onNodeLeft(final ClusterNode node) {
 
                         if (crd0.isLocal()) {
                             if (allReceived) {
+                                awaitSingleMapUpdates();
+
                                 onAllReceived(true);
 
                                 return;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index fbd8ce50d034f..cf692640c2365 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -679,14 +679,14 @@ else if (txFinishMsgLog.isDebugEnabled()) {
      * @param req Request.
      * @return Future.
      */
-    @Nullable public IgniteInternalFuture<IgniteInternalTx> processNearTxFinishRequest(UUID nodeId,
+    @Nullable private IgniteInternalFuture<IgniteInternalTx> processNearTxFinishRequest(UUID nodeId,
         GridNearTxFinishRequest req) {
         if (txFinishMsgLog.isDebugEnabled())
             txFinishMsgLog.debug("Received near finish request [txId=" + req.version() + ", node=" + nodeId + ']');
 
         IgniteInternalFuture<IgniteInternalTx> fut = finish(nodeId, null, req);
 
-        assert req.txState() != null || fut.error() != null ||
+        assert req.txState() != null || (fut != null && fut.error() != null) ||
             (ctx.tm().tx(req.version()) == null && ctx.tm().nearTx(req.version()) == null);
 
         return fut;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6c263638de7c5..b9b92b856b6c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1265,7 +1265,7 @@ private Iterator<Cache.Entry<Object, Object>> serviceEntries(IgniteBiPredicate<O
         try {
             if (!cache.context().affinityNode()) {
                 ClusterNode oldestSrvNode =
-                    CU.oldestAliveCacheServerNode(cache.context().shared(), AffinityTopologyVersion.NONE);
+                    ctx.discovery().oldestAliveCacheServerNode(AffinityTopologyVersion.NONE);
 
                 if (oldestSrvNode == null)
                     return new GridEmptyIterator<>();
@@ -1589,7 +1589,7 @@ private class TopologyListener implements GridLocalEventListener {
 
                 depExe.submit(new BusyRunnable() {
                     @Override public void run0() {
-                        ClusterNode oldest = CU.oldestAliveCacheServerNode(cache.context().shared(), topVer);
+                        ClusterNode oldest = ctx.discovery().oldestAliveCacheServerNode(topVer);
 
                         if (oldest != null && oldest.isLocal()) {
                             final Collection<GridServiceDeployment> retries = new ConcurrentLinkedQueue<>();
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 881474547740f..a660ec8bb09ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -129,6 +129,7 @@
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingResponse;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryRedirectToClient;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryStatusCheckMessage;
+import org.apache.ignite.thread.IgniteThreadPoolExecutor;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
@@ -174,8 +175,7 @@ class ServerImpl extends TcpDiscoveryImpl {
         IgniteProductVersion.fromString("1.5.0");
 
     /** */
-    private final ThreadPoolExecutor utilityPool = new ThreadPoolExecutor(0, 1, 2000, TimeUnit.MILLISECONDS,
-        new LinkedBlockingQueue<Runnable>());
+    private IgniteThreadPoolExecutor utilityPool;
 
     /** Nodes ring. */
     @GridToStringExclude
@@ -297,6 +297,13 @@ class ServerImpl extends TcpDiscoveryImpl {
             spiState = DISCONNECTED;
         }
 
+        utilityPool = new IgniteThreadPoolExecutor("disco-pool",
+            spi.ignite().name(),
+            0,
+            1,
+            2000,
+            new LinkedBlockingQueue<Runnable>());
+
         if (debugMode) {
             if (!log.isInfoEnabled())
                 throw new IgniteSpiException("Info log level should be enabled for TCP discovery to work " +
@@ -2403,9 +2410,12 @@ private class RingMessageWorker extends MessageWorkerAdapter<TcpDiscoveryAbstrac
         /** Connection check threshold. */
         private long connCheckThreshold;
 
+        /** */
+        private long lastRingMsgTime;
+
         /**
          */
-        protected RingMessageWorker() {
+        RingMessageWorker() {
             super("tcp-disco-msg-worker", 10);
 
             initConnectionCheckFrequency();
@@ -2500,6 +2510,8 @@ private void initConnectionCheckFrequency() {
          * @param msg Message to process.
          */
         @Override protected void processMessage(TcpDiscoveryAbstractMessage msg) {
+            sendHeartbeatMessage();
+
             DebugLogger log = messageLogger(msg);
 
             if (log.isDebugEnabled())
@@ -2508,6 +2520,11 @@ private void initConnectionCheckFrequency() {
             if (debugMode)
                 debugLog(msg, "Processing message [cls=" + msg.getClass().getSimpleName() + ", id=" + msg.id() + ']');
 
+            boolean ensured = spi.ensured(msg);
+
+            if (!locNode.id().equals(msg.senderNodeId()) && ensured)
+                lastRingMsgTime = U.currentTimeMillis();
+
             if (locNode.internalOrder() == 0) {
                 boolean proc = false;
 
@@ -2564,7 +2581,7 @@ else if (msg instanceof TcpDiscoveryClientPingRequest)
             else
                 assert false : "Unknown message type: " + msg.getClass().getSimpleName();
 
-            if (spi.ensured(msg) && redirectToClients(msg))
+            if (ensured && redirectToClients(msg))
                 msgHist.add(msg);
 
             if (msg.senderNodeId() != null && !msg.senderNodeId().equals(getLocalNodeId())) {
@@ -5336,12 +5353,9 @@ private void notifyDiscoveryListener(TcpDiscoveryCustomEventMessage msg) {
          * Sends heartbeat message if needed.
          */
         private void sendHeartbeatMessage() {
-            if (!isLocalNodeCoordinator())
-                return;
-
             long elapsed = (lastTimeHbMsgSent + spi.hbFreq) - U.currentTimeMillis();
 
-            if (elapsed > 0)
+            if (elapsed > 0 || !isLocalNodeCoordinator())
                 return;
 
             TcpDiscoveryHeartbeatMessage msg = new TcpDiscoveryHeartbeatMessage(getConfiguredNodeId());
@@ -5361,7 +5375,9 @@ private void checkHeartbeatsReceiving() {
             if (lastTimeStatusMsgSent < locNode.lastUpdateTime())
                 lastTimeStatusMsgSent = locNode.lastUpdateTime();
 
-            long elapsed = (lastTimeStatusMsgSent + hbCheckFreq) - U.currentTimeMillis();
+            long updateTime = Math.max(lastTimeStatusMsgSent, lastRingMsgTime);
+
+            long elapsed = (updateTime + hbCheckFreq) - U.currentTimeMillis();
 
             if (elapsed > 0)
                 return;
@@ -6062,11 +6078,11 @@ private boolean processJoinRequestMessage(TcpDiscoveryJoinRequestMessage msg,
 
             TcpDiscoverySpiState state = spiStateCopy();
 
-            long socketTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
+            long sockTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
                 spi.getSocketTimeout();
 
             if (state == CONNECTED) {
-                spi.writeToSocket(msg, sock, RES_OK, socketTimeout);
+                spi.writeToSocket(msg, sock, RES_OK, sockTimeout);
 
                 if (log.isDebugEnabled())
                     log.debug("Responded to join request message [msg=" + msg + ", res=" + RES_OK + ']');
@@ -6103,7 +6119,7 @@ private boolean processJoinRequestMessage(TcpDiscoveryJoinRequestMessage msg,
                     // Local node is stopping. Remote node should try next one.
                     res = RES_CONTINUE_JOIN;
 
-                spi.writeToSocket(msg, sock, res, socketTimeout);
+                spi.writeToSocket(msg, sock, res, sockTimeout);
 
                 if (log.isDebugEnabled())
                     log.debug("Responded to join request message [msg=" + msg + ", res=" + res + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
index 9e73632feae6b..c79064491bb43 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
@@ -28,6 +28,8 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryCustomEventMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddedMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeLeftMessage;
@@ -317,7 +319,9 @@ public synchronized void onMessageSent(TcpDiscoveryAbstractMessage msg, long tim
         assert time >= 0 : time;
 
         if (crdSinceTs.get() > 0 &&
+            (msg instanceof TcpDiscoveryCustomEventMessage) ||
             (msg instanceof TcpDiscoveryNodeAddedMessage) ||
+            (msg instanceof TcpDiscoveryNodeAddFinishedMessage) ||
             (msg instanceof TcpDiscoveryNodeLeftMessage) ||
             (msg instanceof TcpDiscoveryNodeFailedMessage)) {
             ringMsgsSndTs.put(msg.id(), U.currentTimeMillis());
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionSelfTest.java
index 878d7d1a6a6a2..43017db856389 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionSelfTest.java
@@ -104,6 +104,7 @@ public void testRandomReassignmentThreeBackups() throws Exception {
     }
 
     /**
+     * @param backups Number of backups.
      * @throws Exception If failed.
      */
     protected void checkNodeRemoved(int backups) throws Exception {
@@ -247,7 +248,6 @@ else if (nodes.size() == maxNodes) {
         }
     }
 
-
     /**
      * @param assignment Assignment to verify.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
index 390c83e46bba1..31b4bc7ca6682 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
@@ -239,7 +239,7 @@ private void validateAlives() {
                 GridCacheSharedContext<?, ?> ctx = k.context().cache().context();
 
                 ClusterNode oldest =
-                    GridCacheUtils.oldestAliveCacheServerNode(ctx, new AffinityTopologyVersion(currVer));
+                    ctx.discovery().oldestAliveCacheServerNode(new AffinityTopologyVersion(currVer));
 
                 assertNotNull(oldest);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedAffinitySpreadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedAffinitySpreadTest.java
index a59ca8b4b30db..2d46cf4bc3e91 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedAffinitySpreadTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedAffinitySpreadTest.java
@@ -76,7 +76,12 @@ private void checkDistribution(RendezvousAffinityFunction aff, Collection<Cluste
         Map<ClusterNode, Integer> parts = new HashMap<>(nodes.size());
 
         for (int part = 0; part < aff.getPartitions(); part++) {
-            Collection<ClusterNode> affNodes = aff.assignPartition(part, new ArrayList(nodes), 0, null);
+            Collection<ClusterNode> affNodes = aff.assignPartition(null,
+                part,
+                new ArrayList<>(nodes),
+                new HashMap<ClusterNode, byte[]>(),
+                0,
+                null);
 
             assertEquals(1, affNodes.size());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtTestUtils.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtTestUtils.java
deleted file mode 100644
index 377a55f57d269..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtTestUtils.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.distributed.dht;
-
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.affinity.AffinityFunction;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.processors.cache.GridCacheConcurrentMap;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap2;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.internal.CU;
-
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
-
-/**
- * Utility methods for dht preloader testing.
- */
-public class GridCacheDhtTestUtils {
-    /**
-     * Ensure singleton.
-     */
-    private GridCacheDhtTestUtils() {
-        // No-op.
-    }
-
-    /**
-     * @param dht Cache.
-     * @param keyCnt Number of test keys to put into cache.
-     * @throws IgniteCheckedException If failed to prepare.
-     */
-    @SuppressWarnings({"UnusedAssignment", "unchecked"})
-    static void prepareKeys(GridDhtCache<Integer, String> dht, int keyCnt) throws IgniteCheckedException {
-        AffinityFunction aff = dht.context().config().getAffinity();
-
-        GridCacheConcurrentMap cacheMap;
-
-        try {
-            Field field = GridCacheAdapter.class.getDeclaredField("map");
-
-            field.setAccessible(true);
-
-            cacheMap = (GridCacheConcurrentMap)field.get(dht);
-        }
-        catch (Exception e) {
-            throw new IgniteCheckedException("Failed to get cache map.", e);
-        }
-
-        GridDhtPartitionTopology top = dht.topology();
-
-        GridCacheContext ctx = dht.context();
-
-        for (int i = 0; i < keyCnt; i++) {
-            KeyCacheObject cacheKey = ctx.toCacheKeyObject(i);
-
-            cacheMap.putEntryIfObsoleteOrAbsent(
-                AffinityTopologyVersion.NONE,
-                cacheKey,
-                ctx.toCacheKeyObject("value" + i),
-                false,
-                false);
-
-            dht.preloader().request(Collections.singleton(cacheKey), AffinityTopologyVersion.NONE);
-
-            GridDhtLocalPartition part = top.localPartition(aff.partition(i), false);
-
-            assert part != null;
-
-            part.own();
-        }
-    }
-
-    /**
-     * @param dht Dht cache.
-     * @param idx Cache index
-     */
-    static void printDhtTopology(GridDhtCache<Integer, String> dht, int idx) {
-        final Affinity<Integer> aff = dht.affinity();
-
-        Ignite ignite = dht.context().grid();
-        ClusterNode locNode = ignite.cluster().localNode();
-
-        GridDhtPartitionTopology top = dht.topology();
-
-        System.out.println("\nTopology of cache #" + idx + " (" + locNode.id() + ")" + ":");
-        System.out.println("----------------------------------");
-
-        List<Integer> affParts = new LinkedList<>();
-
-        GridDhtPartitionMap2 map = dht.topology().partitions(locNode.id());
-
-        if (map != null)
-            for (int p : map.keySet())
-                affParts.add(p);
-
-        Collections.sort(affParts);
-
-        System.out.println("Affinity partitions: " + affParts + "\n");
-
-        List<GridDhtLocalPartition> locals = new ArrayList<GridDhtLocalPartition>(top.localPartitions());
-
-        Collections.sort(locals);
-
-        for (final GridDhtLocalPartition part : locals) {
-            Collection<ClusterNode> partNodes = aff.mapKeyToPrimaryAndBackups(part.id());
-
-            String ownStr = !partNodes.contains(dht.context().localNode()) ? "NOT AN OWNER" :
-                F.eqNodes(CU.primary(partNodes), locNode) ? "PRIMARY" : "BACKUP";
-
-            Collection<Integer> keys = F.viewReadOnly(dht.keySet(), F.<Integer>identity(), new P1<Integer>() {
-                @Override public boolean apply(Integer k) {
-                    return aff.partition(k) == part.id();
-                }
-            });
-
-            System.out.println("Local partition: [" + part + "], [owning=" + ownStr + ", keyCnt=" + keys.size() +
-                ", keys=" + keys + "]");
-        }
-
-        System.out.println("\nNode map:");
-
-        for (Map.Entry<UUID, GridDhtPartitionMap2> e : top.partitionMap(false).entrySet()) {
-            List<Integer> list = new ArrayList<>(e.getValue().keySet());
-
-            Collections.sort(list);
-
-            System.out.println("[node=" + e.getKey() + ", parts=" + list + "]");
-        }
-
-        System.out.println("");
-    }
-
-    /**
-     * Checks consistency of partitioned cache.
-     * Any preload processes must be finished before this method call().
-     *
-     * @param dht Dht cache.
-     * @param idx Cache index.
-     * @param log Logger.
-     */
-    @SuppressWarnings("unchecked")
-    static void checkDhtTopology(GridDhtCache<Integer, String> dht, int idx, IgniteLogger log) {
-        assert dht != null;
-        assert idx >= 0;
-        assert log != null;
-
-        log.info("Checking balanced state of cache #" + idx);
-
-        Affinity<Object> aff = (Affinity)dht.affinity();
-
-        Ignite ignite = dht.context().grid();
-        ClusterNode locNode = ignite.cluster().localNode();
-
-        GridDhtPartitionTopology top = dht.topology();
-
-        // Expected partitions calculated with affinity function.
-        // They should be in topology in OWNING state.
-        Collection<Integer> affParts = new HashSet<>();
-
-        GridDhtPartitionMap2 map = dht.topology().partitions(locNode.id());
-
-        if (map != null)
-            for (int p : map.keySet())
-                affParts.add(p);
-
-        if (F.isEmpty(affParts))
-            return;
-
-        for (int p : affParts)
-            assert top.localPartition(p, false) != null :
-                "Partition does not exist in topology: [cache=" + idx + ", part=" + p + "]";
-
-        for (GridDhtLocalPartition p : top.localPartitions()) {
-            assert affParts.contains(p.id()) :
-                "Invalid local partition: [cache=" + idx + ", part=" + p + ", node partitions=" + affParts + "]";
-
-            assert p.state() == OWNING : "Invalid partition state [cache=" + idx + ", part=" + p + "]";
-
-            Collection<ClusterNode> partNodes = aff.mapPartitionToPrimaryAndBackups(p.id());
-
-            assert partNodes.contains(locNode) :
-                "Partition affinity nodes does not contain local node: [cache=" + idx + "]";
-        }
-
-        // Check keys.
-        for (GridCacheEntryEx e : dht.entries()) {
-            GridDhtCacheEntry entry = (GridDhtCacheEntry)e;
-
-            if (!affParts.contains(entry.partition()))
-                log.warning("Partition of stored entry is obsolete for node: [cache=" + idx + ", entry=" + entry +
-                    ", node partitions=" + affParts + "]");
-
-            int p = aff.partition(entry.key());
-
-            if (!affParts.contains(p))
-                log.warning("Calculated entry partition is not in node partitions: [cache=" + idx + ", part=" + p +
-                    ", entry=" + entry + ", node partitions=" + affParts + "]");
-        }
-    }
-}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 48567dabb018e..40b1197317acd 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -58,8 +58,7 @@
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap2;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
 import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
@@ -383,17 +382,10 @@ private boolean isPreloadingActive(final GridCacheContext<?, ?> cctx, List<Integ
 
     /**
      * @param cctx Cache context.
-     * @return {@code true} If cache context
+     * @return {@code True} If cache has partitions in {@link GridDhtPartitionState#MOVING} state.
      */
     private boolean hasMovingPartitions(GridCacheContext<?, ?> cctx) {
-        GridDhtPartitionFullMap fullMap = cctx.topology().partitionMap(false);
-
-        for (GridDhtPartitionMap2 map : fullMap.values()) {
-            if (map.hasMovingPartitions())
-                return true;
-        }
-
-        return false;
+        return cctx.topology().hasMovingPartitions();
     }
 
     /**

From 6ba1711a1fa10d8276974227491136070c3ed43a Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Tue, 6 Dec 2016 12:55:41 +0300
Subject: [PATCH 017/446] IGNITE-4242 ExchangeManager should wait for cache
 rebalancing in async way

---
 .../GridCachePartitionExchangeManager.java    | 128 +++-------
 .../processors/cache/GridCachePreloader.java  |  11 +-
 .../cache/GridCachePreloaderAdapter.java      |   5 +-
 .../preloader/GridDhtPartitionDemander.java   | 230 +++++++++++-------
 .../dht/preloader/GridDhtPreloader.java       |   9 +-
 .../GridCacheRebalancingSyncSelfTest.java     |   2 +
 6 files changed, 183 insertions(+), 202 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7a24aa14ded91..f04a6ce2fa025 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -21,18 +21,18 @@
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
-import java.util.Queue;
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.LinkedBlockingDeque;
@@ -87,7 +87,6 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.internal.util.typedef.internal.GPC;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
@@ -97,13 +96,11 @@
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
-import org.jsr166.ConcurrentLinkedDeque8;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PRELOAD_RESEND_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_THREAD_DUMP_ON_EXCHANGE_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.getLong;
-import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STARTED;
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
@@ -156,9 +153,6 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     /** */
     private GridFutureAdapter<?> reconnectExchangeFut;
 
-    /** */
-    private final Queue<Callable<Boolean>> rebalanceQ = new ConcurrentLinkedDeque8<>();
-
     /**
      * Partition map futures.
      * This set also contains already completed exchange futures to address race conditions when coordinator
@@ -1596,12 +1590,8 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
         @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
             long timeout = cctx.gridConfig().getNetworkTimeout();
 
-            boolean startEvtFired = false;
-
             int cnt = 0;
 
-            IgniteInternalFuture asyncStartFut = null;
-
             while (!isCancelled()) {
                 GridDhtPartitionsExchangeFuture exchFut = null;
 
@@ -1703,20 +1693,8 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                     continue;
 
                                 changed |= cacheCtx.topology().afterExchange(exchFut);
-
-                                // Preload event notification.
-                                if (!exchFut.skipPreload() && cacheCtx.events().isRecordable(EVT_CACHE_REBALANCE_STARTED)) {
-                                    if (!cacheCtx.isReplicated() || !startEvtFired) {
-                                        DiscoveryEvent discoEvt = exchFut.discoveryEvent();
-
-                                        cacheCtx.events().addPreloadEvent(-1, EVT_CACHE_REBALANCE_STARTED,
-                                            discoEvt.eventNode(), discoEvt.type(), discoEvt.timestamp());
-                                    }
-                                }
                             }
 
-                            startEvtFired = true;
-
                             if (!cctx.kernalContext().clientNode() && changed && futQ.isEmpty())
                                 refreshPartitions();
                         }
@@ -1755,8 +1733,6 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                     if (assignsMap != null) {
                         int size = assignsMap.size();
 
-                        rebalanceQ.clear();
-
                         NavigableMap<Integer, List<Integer>> orderMap = new TreeMap<>();
 
                         for (Map.Entry<Integer, GridDhtPreloaderAssignments> e : assignsMap.entrySet()) {
@@ -1772,101 +1748,65 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                             orderMap.get(order).add(cacheId);
                         }
 
-                        Callable<Boolean> marshR = null;
-                        List<Callable<Boolean>> orderedRs = new ArrayList<>(size);
+                        Runnable r = null;
+
+                        List<String> rebList = new LinkedList<>();
+
+                        boolean assignsCancelled = false;
 
-                        //Ordered rebalance scheduling.
-                        for (Integer order : orderMap.keySet()) {
+                        for (Integer order : orderMap.descendingKeySet()) {
                             for (Integer cacheId : orderMap.get(order)) {
                                 GridCacheContext<K, V> cacheCtx = cctx.cacheContext(cacheId);
 
-                                List<String> waitList = new ArrayList<>(size - 1);
+                                GridDhtPreloaderAssignments assigns = assignsMap.get(cacheId);
 
-                                for (List<Integer> cIds : orderMap.headMap(order).values()) {
-                                    for (Integer cId : cIds)
-                                        waitList.add(cctx.cacheContext(cId).name());
-                                }
+                                if (assigns != null)
+                                    assignsCancelled |= assigns.cancelled();
 
-                                Callable<Boolean> r = cacheCtx.preloader().addAssignments(assignsMap.get(cacheId),
+                                // Cancels previous rebalance future (in case it's not done yet).
+                                // Sends previous rebalance stopped event (if necessary).
+                                // Creates new rebalance future.
+                                // Sends current rebalance started event (if necessary).
+                                // Finishes cache sync future (on empty assignments).
+                                Runnable cur = cacheCtx.preloader().addAssignments(assigns,
                                     forcePreload,
-                                    waitList,
-                                    cnt);
+                                    cnt,
+                                    r);
 
-                                if (r != null) {
-                                    U.log(log, "Cache rebalancing scheduled: [cache=" + cacheCtx.name() +
-                                        ", waitList=" + waitList.toString() + "]");
+                                if (cur != null) {
+                                    rebList.add(U.maskName(cacheCtx.name()));
 
-                                    if (cacheId == CU.cacheId(GridCacheUtils.MARSH_CACHE_NAME))
-                                        marshR = r;
-                                    else
-                                        orderedRs.add(r);
+                                    r = cur;
                                 }
                             }
                         }
 
-                        if (asyncStartFut != null)
-                            asyncStartFut.get(); // Wait for thread stop.
+                        if (assignsCancelled) { // Pending exchange.
+                            U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
+                                "[top=" + exchFut.topologyVersion() + ", evt=" + exchFut.discoveryEvent().name() +
+                                ", node=" + exchFut.discoveryEvent().eventNode().id() + ']');
+                        }
+                        else if (r != null) {
+                            Collections.reverse(rebList);
 
-                        rebalanceQ.addAll(orderedRs);
+                            U.log(log, "Rebalancing scheduled [order=" + rebList + "]");
 
-                        if (marshR != null || !rebalanceQ.isEmpty()) {
                             if (futQ.isEmpty()) {
-                                U.log(log, "Rebalancing required " +
+                                U.log(log, "Rebalancing started " +
                                     "[top=" + exchFut.topologyVersion() + ", evt=" + exchFut.discoveryEvent().name() +
                                     ", node=" + exchFut.discoveryEvent().eventNode().id() + ']');
 
-                                if (marshR != null) {
-                                    try {
-                                        marshR.call(); //Marshaller cache rebalancing launches in sync way.
-                                    }
-                                    catch (Exception ex) {
-                                        if (log.isDebugEnabled())
-                                            log.debug("Failed to send initial demand request to node");
-
-                                        continue;
-                                    }
-                                }
-
-                                final GridFutureAdapter fut = new GridFutureAdapter();
-
-                                asyncStartFut = fut;
-
-                                cctx.kernalContext().closure().callLocalSafe(new GPC<Boolean>() {
-                                    @Override public Boolean call() {
-                                        try {
-                                            while (true) {
-                                                Callable<Boolean> r = rebalanceQ.poll();
-
-                                                if (r == null)
-                                                    return false;
-
-                                                if (!r.call())
-                                                    return false;
-                                            }
-                                        }
-                                        catch (Exception ex) {
-                                            if (log.isDebugEnabled())
-                                                log.debug("Failed to send initial demand request to node");
-
-                                            return false;
-                                        }
-                                        finally {
-                                            fut.onDone();
-                                        }
-                                    }
-                                }, /*system pool*/true);
+                                r.run(); // Starts rebalancing routine.
                             }
-                            else {
+                            else
                                 U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
                                     "[top=" + exchFut.topologyVersion() + ", evt=" + exchFut.discoveryEvent().name() +
                                     ", node=" + exchFut.discoveryEvent().eventNode().id() + ']');
-                            }
                         }
-                        else {
+                        else
                             U.log(log, "Skipping rebalancing (nothing scheduled) " +
                                 "[top=" + exchFut.topologyVersion() + ", evt=" + exchFut.discoveryEvent().name() +
                                 ", node=" + exchFut.discoveryEvent().eventNode().id() + ']');
-                        }
                     }
                 }
                 catch (IgniteInterruptedCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index 1d1cfab04ce63..3c4456d5984d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -19,7 +19,6 @@
 
 import java.util.Collection;
 import java.util.UUID;
-import java.util.concurrent.Callable;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -84,14 +83,14 @@ public interface GridCachePreloader {
      *
      * @param assignments Assignments to add.
      * @param forcePreload Force preload flag.
-     * @param caches Rebalancing of these caches will be finished before this started.
      * @param cnt Counter.
-     * @return Rebalancing closure.
+     * @param next Runnable responsible for cache rebalancing start.
+     * @return Rebalancing runnable.
      */
-    public Callable<Boolean> addAssignments(GridDhtPreloaderAssignments assignments,
+    public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
         boolean forcePreload,
-        Collection<String> caches,
-        int cnt);
+        int cnt,
+        Runnable next);
 
     /**
      * @param p Preload predicate.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index b15ebc5029b12..656a960b56b9f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -19,7 +19,6 @@
 
 import java.util.Collection;
 import java.util.UUID;
-import java.util.concurrent.Callable;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.affinity.AffinityFunction;
@@ -166,8 +165,8 @@ public GridCachePreloaderAdapter(GridCacheContext<?, ?> cctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public Callable<Boolean> addAssignments(GridDhtPreloaderAssignments assignments, boolean forcePreload,
-        Collection<String> caches, int cnt) {
+    @Override public Runnable addAssignments(GridDhtPreloaderAssignments assignments, boolean forcePreload,
+        int cnt, Runnable next) {
         return null;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 57d522922e801..a6808c73577e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -28,8 +28,8 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
 import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
@@ -72,6 +72,7 @@
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_OBJECT_LOADED;
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_LOADED;
+import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STARTED;
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
 import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
@@ -121,6 +122,18 @@ public class GridDhtPartitionDemander {
     /** Cached rebalance topics. */
     private final Map<Integer, Object> rebalanceTopics;
 
+    /**
+     * Started event sent.
+     * Make sense for replicated cache only.
+     */
+    private final AtomicBoolean startedEvtSent = new AtomicBoolean();
+
+    /**
+     * Stopped event sent.
+     * Make sense for replicated cache only.
+     */
+    private final AtomicBoolean stoppedEvtSent = new AtomicBoolean();
+
     /**
      * @param cctx Cctx.
      * @param demandLock Demand lock.
@@ -249,45 +262,25 @@ private void preloadEvent(int part, int type, DiscoveryEvent discoEvt) {
     }
 
     /**
-     * @param name Cache name.
-     * @param fut Future.
-     * @throws IgniteCheckedException If failed.
+     * Sets last exchange future.
+     *
+     * @param lastFut Last future to set.
      */
-    private boolean waitForCacheRebalancing(String name, RebalanceFuture fut) throws IgniteCheckedException {
-        if (log.isDebugEnabled())
-            log.debug("Waiting for another cache to start rebalancing [cacheName=" + cctx.name() +
-                ", waitCache=" + name + ']');
-
-        RebalanceFuture wFut = (RebalanceFuture)cctx.kernalContext().cache().internalCache(name)
-            .preloader().rebalanceFuture();
-
-        if (!topologyChanged(fut) && wFut.updateSeq == fut.updateSeq) {
-            if (!wFut.get()) {
-                U.log(log, "Skipping waiting of " + name + " cache [top=" + fut.topologyVersion() +
-                    "] (cache rebalanced with missed partitions)");
-
-                return false;
-            }
-
-            return true;
-        }
-        else {
-            U.log(log, "Skipping waiting of " + name + " cache [top=" + fut.topologyVersion() +
-                "] (topology already changed)");
-
-            return false;
-        }
+    void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
+        lastExchangeFut = lastFut;
     }
 
     /**
      * @param assigns Assignments.
      * @param force {@code True} if dummy reassign.
-     * @param caches Rebalancing of these caches will be finished before this started.
      * @param cnt Counter.
-     * @return Rebalancing closure.
+     * @param next Runnable responsible for cache rebalancing start.
+     * @return Rebalancing runnable.
      */
-    Callable<Boolean> addAssignments(final GridDhtPreloaderAssignments assigns, boolean force,
-        final Collection<String> caches, int cnt) {
+    Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
+        boolean force,
+        int cnt,
+        final Runnable next) {
         if (log.isDebugEnabled())
             log.debug("Adding partition assignments: " + assigns);
 
@@ -296,7 +289,7 @@ Callable<Boolean> addAssignments(final GridDhtPreloaderAssignments assigns, bool
         if (delay == 0 || force) {
             final RebalanceFuture oldFut = rebalanceFut;
 
-            final RebalanceFuture fut = new RebalanceFuture(assigns, cctx, log, oldFut.isInitial(), cnt);
+            final RebalanceFuture fut = new RebalanceFuture(assigns, cctx, log, startedEvtSent, stoppedEvtSent, cnt);
 
             if (!oldFut.isInitial())
                 oldFut.cancel();
@@ -310,20 +303,69 @@ Callable<Boolean> addAssignments(final GridDhtPreloaderAssignments assigns, bool
 
             rebalanceFut = fut;
 
-            if (assigns.isEmpty()) {
-                fut.doneIfEmpty(assigns.cancelled());
+            fut.sendRebalanceStartedEvent();
+
+            if (assigns.cancelled()) { // Pending exchange.
+                if (log.isDebugEnabled())
+                    log.debug("Rebalancing skipped due to cancelled assignments.");
+
+                fut.onDone(false);
+
+                fut.sendRebalanceFinishedEvent();
+
+                return null;
+            }
+
+            if (assigns.isEmpty()) { // Nothing to rebalance.
+                if (log.isDebugEnabled())
+                    log.debug("Rebalancing skipped due to empty assignments.");
+
+                fut.onDone(true);
+
+                ((GridFutureAdapter)cctx.preloader().syncFuture()).onDone();
+
+                fut.sendRebalanceFinishedEvent();
 
                 return null;
             }
 
-            return new Callable<Boolean>() {
-                @Override public Boolean call() throws Exception {
-                    for (String c : caches) {
-                        if (!waitForCacheRebalancing(c, fut))
-                            return false;
+            return new Runnable() {
+                @Override public void run() {
+                    try {
+                        if (next != null)
+                            fut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
+                                @Override public void apply(IgniteInternalFuture<Boolean> f) {
+                                    try {
+                                        if (f.get()) // Not cancelled.
+                                            next.run(); // Starts next cache rebalancing (according to the order).
+                                    }
+                                    catch (IgniteCheckedException ignored) {
+                                        if (log.isDebugEnabled())
+                                            log.debug(ignored.getMessage());
+                                    }
+                                }
+                            });
+
+                        requestPartitions(fut, assigns);
                     }
+                    catch (IgniteCheckedException e) {
+                        ClusterTopologyCheckedException cause = e.getCause(ClusterTopologyCheckedException.class);
 
-                    return requestPartitions(fut, assigns);
+                        if (cause != null)
+                            log.warning("Failed to send initial demand request to node. " + e.getMessage());
+                        else
+                            log.error("Failed to send initial demand request to node.", e);
+
+                        fut.cancel();
+                    }
+                    catch (Throwable th) {
+                        log.error("Runtime error caught during initial demand request sending.", th);
+
+                        fut.cancel();
+
+                        if (th instanceof Error)
+                            throw th;
+                    }
                 }
             };
         }
@@ -361,14 +403,17 @@ else if (delay > 0) {
      * @throws IgniteCheckedException If failed.
      * @return Partitions were requested.
      */
-    private boolean requestPartitions(
+    private void requestPartitions(
         RebalanceFuture fut,
         GridDhtPreloaderAssignments assigns
     ) throws IgniteCheckedException {
-        for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assigns.entrySet()) {
-            if (topologyChanged(fut))
-                return false;
+        if (topologyChanged(fut)) {
+            fut.cancel();
+
+            return;
+        }
 
+        for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assigns.entrySet()) {
             final ClusterNode node = e.getKey();
 
             GridDhtPartitionDemandMessage d = e.getValue();
@@ -387,7 +432,7 @@ private boolean requestPartitions(
 
             //Check remote node rebalancing API version.
             if (node.version().compareTo(GridDhtPreloader.REBALANCING_VER_2_SINCE) >= 0) {
-                U.log(log, "Starting rebalancing [cache=" + cctx.name() + ", mode=" + cfg.getRebalanceMode() +
+                U.log(log, "Starting rebalancing [mode=" + cfg.getRebalanceMode() +
                     ", fromNode=" + node.id() + ", partitionsCount=" + parts.size() +
                     ", topology=" + fut.topologyVersion() + ", updateSeq=" + fut.updateSeq + "]");
 
@@ -446,8 +491,6 @@ private boolean requestPartitions(
                 worker.run(node, d);
             }
         }
-
-        return true;
     }
 
     /**
@@ -738,15 +781,6 @@ else if (log.isDebugEnabled())
         return S.toString(GridDhtPartitionDemander.class, this);
     }
 
-    /**
-     * Sets last exchange future.
-     *
-     * @param lastFut Last future to set.
-     */
-    void updateLastExchangeFuture(GridDhtPartitionsExchangeFuture lastFut) {
-        lastExchangeFut = lastFut;
-    }
-
     /**
      *
      */
@@ -754,8 +788,11 @@ public static class RebalanceFuture extends GridFutureAdapter<Boolean> {
         /** */
         private static final long serialVersionUID = 1L;
 
-        /** Should EVT_CACHE_REBALANCE_STOPPED event be sent of not. */
-        private final boolean sndStoppedEvnt;
+        /** Should EVT_CACHE_REBALANCE_STARTED event be sent or not. */
+        private final AtomicBoolean startedEvtSent;
+
+        /** Should EVT_CACHE_REBALANCE_STOPPED event be sent or not. */
+        private final AtomicBoolean stoppedEvtSent;
 
         /** */
         private final GridCacheContext<?, ?> cctx;
@@ -783,13 +820,15 @@ public static class RebalanceFuture extends GridFutureAdapter<Boolean> {
          * @param assigns Assigns.
          * @param cctx Context.
          * @param log Logger.
-         * @param sentStopEvnt Stop event flag.
+         * @param startedEvtSent Start event sent flag.
+         * @param stoppedEvtSent Stop event sent flag.
          * @param updateSeq Update sequence.
          */
         RebalanceFuture(GridDhtPreloaderAssignments assigns,
             GridCacheContext<?, ?> cctx,
             IgniteLogger log,
-            boolean sentStopEvnt,
+            AtomicBoolean startedEvtSent,
+            AtomicBoolean stoppedEvtSent,
             long updateSeq) {
             assert assigns != null;
 
@@ -797,7 +836,8 @@ public static class RebalanceFuture extends GridFutureAdapter<Boolean> {
             this.topVer = assigns.topologyVersion();
             this.cctx = cctx;
             this.log = log;
-            this.sndStoppedEvnt = sentStopEvnt;
+            this.startedEvtSent = startedEvtSent;
+            this.stoppedEvtSent = stoppedEvtSent;
             this.updateSeq = updateSeq;
         }
 
@@ -809,7 +849,8 @@ public RebalanceFuture() {
             this.topVer = null;
             this.cctx = null;
             this.log = null;
-            this.sndStoppedEvnt = false;
+            this.startedEvtSent = null;
+            this.stoppedEvtSent = null;
             this.updateSeq = -1;
         }
 
@@ -847,24 +888,6 @@ private void appendPartitions(UUID nodeId, Collection<Integer> parts) {
             }
         }
 
-        /**
-         * @param cancelled Is cancelled.
-         */
-        private void doneIfEmpty(boolean cancelled) {
-            synchronized (this) {
-                if (isDone())
-                    return;
-
-                assert remaining.isEmpty();
-
-                if (log.isDebugEnabled())
-                    log.debug("Rebalancing is not required [cache=" + cctx.name() +
-                        ", topology=" + topVer + "]");
-
-                checkIsDone(cancelled, true);
-            }
-        }
-
         /**
          * Cancels this future.
          *
@@ -875,8 +898,7 @@ private void doneIfEmpty(boolean cancelled) {
                 if (isDone())
                     return true;
 
-                U.log(log, "Cancelled rebalancing from all nodes [cache=" + cctx.name()
-                    + ", topology=" + topologyVersion() + ']');
+                U.log(log, "Cancelled rebalancing from all nodes [topology=" + topologyVersion() + ']');
 
                 if (!cctx.kernalContext().isStopping()) {
                     for (UUID nodeId : remaining.keySet())
@@ -885,7 +907,7 @@ private void doneIfEmpty(boolean cancelled) {
 
                 remaining.clear();
 
-                checkIsDone(true /* cancelled */, false);
+                checkIsDone(true /* cancelled */);
             }
 
             return true;
@@ -907,7 +929,7 @@ private void cancel(UUID nodeId) {
 
                 remaining.remove(nodeId);
 
-                onDone(false); // Finishing rebalance future a non completed.
+                onDone(false); // Finishing rebalance future as non completed.
 
                 checkIsDone(); // But will finish syncFuture only when other nodes are preloaded or rebalancing cancelled.
             }
@@ -988,8 +1010,7 @@ private void partitionDone(UUID nodeId, int p) {
 
                 if (parts.isEmpty()) {
                     U.log(log, "Completed " + ((remaining.size() == 1 ? "(final) " : "") +
-                        "rebalancing [cache=" + cctx.name() +
-                        ", fromNode=" + nodeId + ", topology=" + topologyVersion() +
+                        "rebalancing [fromNode=" + nodeId + ", topology=" + topologyVersion() +
                         ", time=" + (U.currentTimeMillis() - t.get1()) + " ms]"));
 
                     remaining.remove(nodeId);
@@ -1022,23 +1043,20 @@ private void preloadEvent(int type, DiscoveryEvent discoEvt) {
          *
          */
         private void checkIsDone() {
-            checkIsDone(false, false);
+            checkIsDone(false);
         }
 
         /**
          * @param cancelled Is cancelled.
-         * @param wasEmpty {@code True} if future was created without assignments.
          */
-        private void checkIsDone(boolean cancelled, boolean wasEmpty) {
+        private void checkIsDone(boolean cancelled) {
             if (remaining.isEmpty()) {
-                if (cctx.events().isRecordable(EVT_CACHE_REBALANCE_STOPPED) && (!cctx.isReplicated() || sndStoppedEvnt))
-                    preloadEvent(EVT_CACHE_REBALANCE_STOPPED, exchFut.discoveryEvent());
+                sendRebalanceFinishedEvent();
 
                 if (log.isDebugEnabled())
                     log.debug("Completed rebalance future: " + this);
 
-                if (!wasEmpty)
-                    cctx.shared().exchange().scheduleResendPartitions();
+                cctx.shared().exchange().scheduleResendPartitions();
 
                 Collection<Integer> m = new HashSet<>();
 
@@ -1064,6 +1082,30 @@ private void checkIsDone(boolean cancelled, boolean wasEmpty) {
             }
         }
 
+        /**
+         *
+         */
+        private void sendRebalanceStartedEvent() {
+            if (cctx.events().isRecordable(EVT_CACHE_REBALANCE_STARTED) &&
+                (!cctx.isReplicated() || !startedEvtSent.get())) {
+                preloadEvent(EVT_CACHE_REBALANCE_STARTED, exchFut.discoveryEvent());
+
+                startedEvtSent.set(true);
+            }
+        }
+
+        /**
+         *
+         */
+        private void sendRebalanceFinishedEvent() {
+            if (cctx.events().isRecordable(EVT_CACHE_REBALANCE_STOPPED) &&
+                (!cctx.isReplicated() || !stoppedEvtSent.get())) {
+                preloadEvent(EVT_CACHE_REBALANCE_STOPPED, exchFut.discoveryEvent());
+
+                stoppedEvtSent.set(true);
+            }
+        }
+
         /** {@inheritDoc} */
         public String toString() {
             return S.toString(RebalanceFuture.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 0865d9f709d16..692e7c0de45a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -22,7 +22,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReadWriteLock;
@@ -255,7 +254,7 @@ private IgniteCheckedException stopError() {
     @Override public void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
         supplier.onTopologyChanged(lastFut.topologyVersion());
 
-        demander.updateLastExchangeFuture(lastFut);
+        demander.onTopologyChanged(lastFut);
     }
 
     /** {@inheritDoc} */
@@ -413,9 +412,9 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
     }
 
     /** {@inheritDoc} */
-    @Override public Callable<Boolean> addAssignments(GridDhtPreloaderAssignments assignments,
-        boolean forcePreload, Collection<String> caches, int cnt) {
-        return demander.addAssignments(assignments, forcePreload, caches, cnt);
+    @Override public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
+        boolean forcePreload, int cnt, Runnable next) {
+        return demander.addAssignments(assignments, forcePreload, cnt, next);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
index de38952da7fd9..3dfcd85ea46b9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
@@ -501,6 +501,8 @@ protected void checkPartitionMapMessagesAbsent() throws Exception {
 
         record = true;
 
+        log.info("Checking GridDhtPartitions*Message absent (it will take 30 SECONDS) ... ");
+
         U.sleep(30_000);
 
         record = false;

From 3df412e7f25aac8227b68cffe1577593a05d1431 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 7 Dec 2016 12:25:32 +0300
Subject: [PATCH 018/446] ignite-2545 Optimization for GridCompoundFuture's
 futures iteration

---
 .../GridCacheTxRecoveryFuture.java            |  4 +-
 .../distributed/dht/GridDhtGetFuture.java     |  2 +-
 .../distributed/dht/GridDhtLockFuture.java    |  4 +-
 .../dht/GridDhtTxPrepareFuture.java           |  4 +-
 .../colocated/GridDhtColocatedLockFuture.java |  4 +-
 .../distributed/near/GridNearLockFuture.java  |  4 +-
 ...OptimisticSerializableTxPrepareFuture.java |  4 +-
 .../GridNearOptimisticTxPrepareFuture.java    | 15 +++--
 .../GridNearPessimisticTxPrepareFuture.java   |  4 +-
 .../near/GridNearTxFinishFuture.java          |  4 +-
 .../util/future/GridCompoundFuture.java       | 56 +++++++++++++------
 11 files changed, 74 insertions(+), 31 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java
index c07a817001c75..e27f777bd145d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java
@@ -427,8 +427,10 @@ public void onResult(UUID nodeId, GridCacheTxRecoveryResponse res) {
     private MiniFuture miniFuture(IgniteUuid miniId) {
         // We iterate directly over the futs collection here to avoid copy.
         synchronized (sync) {
+            int size = futuresCountNoLock();
+
             // Avoid iterator creation.
-            for (int i = 0; i < futuresCount(); i++) {
+            for (int i = 0; i < size; i++) {
                 IgniteInternalFuture<Boolean> fut = future(i);
 
                 if (!isMini(fut))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
index d2a3b3c022ddc..d2eab5fe66437 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
@@ -275,7 +275,7 @@ else if (mappedKeys != null)
         // Optimization to avoid going through compound future,
         // if getAsync() has been completed and no other futures added to this
         // compound future.
-        if (fut.isDone() && futuresCount() == 0) {
+        if (fut.isDone() && !hasFutures()) {
             if (fut.error() != null)
                 onDone(fut.error());
             else
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
index f2b5f4925d948..f6159a8501e8a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
@@ -543,8 +543,10 @@ void onResult(UUID nodeId, GridDhtLockResponse res) {
     private MiniFuture miniFuture(IgniteUuid miniId) {
         // We iterate directly over the futs collection here to avoid copy.
         synchronized (sync) {
+            int size = futuresCountNoLock();
+
             // Avoid iterator creation.
-            for (int i = 0; i < futuresCount(); i++) {
+            for (int i = 0; i < size; i++) {
                 MiniFuture mini = (MiniFuture) future(i);
 
                 if (mini.futureId().equals(miniId)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index b2b443019be3c..bfabdb6863c92 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -564,8 +564,10 @@ public void onResult(UUID nodeId, GridDhtTxPrepareResponse res) {
     private MiniFuture miniFuture(IgniteUuid miniId) {
         // We iterate directly over the futs collection here to avoid copy.
         synchronized (sync) {
+            int size = futuresCountNoLock();
+
             // Avoid iterator creation.
-            for (int i = 0; i < futuresCount(); i++) {
+            for (int i = 0; i < size; i++) {
                 IgniteInternalFuture<IgniteInternalTx> fut = future(i);
 
                 if (!isMini(fut))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index ddb6500118113..11dd517588dd2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -482,8 +482,10 @@ private Set<IgniteTxKey> requestedKeys0() {
     private MiniFuture miniFuture(IgniteUuid miniId) {
         // We iterate directly over the futs collection here to avoid copy.
         synchronized (sync) {
+            int size = futuresCountNoLock();
+
             // Avoid iterator creation.
-            for (int i = 0; i < futuresCount(); i++) {
+            for (int i = 0; i < size; i++) {
                 IgniteInternalFuture<Boolean> fut = future(i);
 
                 if (!isMini(fut))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index 02f6cce502706..7502f78726980 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -528,8 +528,10 @@ private Set<IgniteTxKey> requestedKeys0() {
     private MiniFuture miniFuture(IgniteUuid miniId) {
         // We iterate directly over the futs collection here to avoid copy.
         synchronized (sync) {
+            int size = futuresCountNoLock();
+
             // Avoid iterator creation.
-            for (int i = 0; i < futuresCount(); i++) {
+            for (int i = 0; i < size; i++) {
                 IgniteInternalFuture<Boolean> fut = future(i);
 
                 if (!isMini(fut))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index 3676a3ce04070..c464b36217fd8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -230,8 +230,10 @@ private void onError(@Nullable GridDistributedTxMapping m, Throwable e) {
     private MiniFuture miniFuture(IgniteUuid miniId) {
         // We iterate directly over the futs collection here to avoid copy.
         synchronized (sync) {
+            int size = futuresCountNoLock();
+
             // Avoid iterator creation.
-            for (int i = 0; i < futuresCount(); i++) {
+            for (int i = 0; i < size; i++) {
                 IgniteInternalFuture<GridNearTxPrepareResponse> fut = future(i);
 
                 if (!isMini(fut))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 87c9e1d35f451..b314b81bf7b50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -202,7 +202,9 @@ void onError(Throwable e, boolean discoThread) {
     @SuppressWarnings("ForLoopReplaceableByForEach")
     public Set<IgniteTxKey> requestedKeys() {
         synchronized (sync) {
-            for (int i = 0; i < futuresCount(); i++) {
+            int size = futuresCountNoLock();
+
+            for (int i = 0; i < size; i++) {
                 IgniteInternalFuture<GridNearTxPrepareResponse> fut = future(i);
 
                 if (isMini(fut) && !fut.isDone()) {
@@ -233,8 +235,10 @@ public Set<IgniteTxKey> requestedKeys() {
     private MiniFuture miniFuture(IgniteUuid miniId) {
         // We iterate directly over the futs collection here to avoid copy.
         synchronized (sync) {
+            int size = futuresCountNoLock();
+
             // Avoid iterator creation.
-            for (int i = 0; i < futuresCount(); i++) {
+            for (int i = size - 1; i >= 0; i--) {
                 IgniteInternalFuture<GridNearTxPrepareResponse> fut = future(i);
 
                 if (!isMini(fut))
@@ -687,7 +691,9 @@ private void onTimeout() {
                 keys = new HashSet<>(keyLockFut.lockKeys);
             else {
                 synchronized (sync) {
-                    for (int i = 0; i < futuresCount(); i++) {
+                    int size = futuresCountNoLock();
+
+                    for (int i = 0; i < size; i++) {
                         IgniteInternalFuture<GridNearTxPrepareResponse> fut = future(i);
 
                         if (isMini(fut) && !fut.isDone()) {
@@ -895,7 +901,8 @@ void onResult(final GridNearTxPrepareResponse res) {
                         }
                         else
                             remap();
-                    } else {
+                    }
+                    else {
                         parent.onPrepareResponse(m, res);
 
                         // Proceed prepare before finishing mini future.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index 01fb5fd653449..f9a2f90787b3c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -133,8 +133,10 @@ public GridNearPessimisticTxPrepareFuture(GridCacheSharedContext cctx, GridNearT
     private MiniFuture miniFuture(IgniteUuid miniId) {
         // We iterate directly over the futs collection here to avoid copy.
         synchronized (sync) {
+            int size = futuresCountNoLock();
+
             // Avoid iterator creation.
-            for (int i = 0; i < futuresCount(); i++) {
+            for (int i = 0; i < size; i++) {
                 MiniFuture mini = (MiniFuture) future(i);
 
                 if (mini.futureId().equals(miniId)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index f14d747ba50e6..54bd5436c636b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -201,7 +201,9 @@ public void onResult(UUID nodeId, GridNearTxFinishResponse res) {
             FinishMiniFuture finishFut = null;
 
             synchronized (sync) {
-                for (int i = 0; i < futuresCount(); i++) {
+                int size = futuresCountNoLock();
+
+                for (int i = 0; i < size; i++) {
                     IgniteInternalFuture<IgniteInternalTx> fut = future(i);
 
                     if (fut.getClass() == FinishMiniFuture.class) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
index 0f7e02097979d..7abd3673df9c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
@@ -88,7 +88,7 @@ public GridCompoundFuture(@Nullable IgniteReducer<T, R> rdc) {
     }
 
     /** {@inheritDoc} */
-    @Override public void apply(IgniteInternalFuture<T> fut) {
+    @Override public final void apply(IgniteInternalFuture<T> fut) {
         try {
             T t = fut.get();
 
@@ -160,9 +160,9 @@ public GridCompoundFuture(@Nullable IgniteReducer<T, R> rdc) {
      * @return Collection of futures.
      */
     @SuppressWarnings("unchecked")
-    public Collection<IgniteInternalFuture<T>> futures() {
+    public final Collection<IgniteInternalFuture<T>> futures() {
         synchronized (sync) {
-            if(futs == null)
+            if (futs == null)
                 return Collections.emptyList();
 
             if (futs instanceof IgniteInternalFuture)
@@ -190,10 +190,12 @@ protected boolean ignoreFailure(Throwable err) {
      * @return {@code True} if there are pending futures.
      */
     @SuppressWarnings("ForLoopReplaceableByForEach")
-    public boolean hasPending() {
+    protected final boolean hasPending() {
         synchronized (sync) {
+            int size = futuresCountNoLock();
+
             // Avoid iterator creation and collection copy.
-            for (int i = 0; i < futuresCount(); i++) {
+            for (int i = 0; i < size; i++) {
                 IgniteInternalFuture<T> fut = future(i);
 
                 if (!fut.isDone())
@@ -210,7 +212,7 @@ public boolean hasPending() {
      * @param fut Future to add.
      */
     @SuppressWarnings("unchecked")
-    public void add(IgniteInternalFuture<T> fut) {
+    public final void add(IgniteInternalFuture<T> fut) {
         assert fut != null;
 
         synchronized (sync) {
@@ -243,7 +245,7 @@ else if (futs instanceof IgniteInternalFuture) {
     /**
      * Clear futures.
      */
-    protected void clear() {
+    protected final void clear() {
         synchronized (sync) {
             futs = null;
         }
@@ -253,14 +255,14 @@ protected void clear() {
      * @return {@code True} if this future was initialized. Initialization happens when {@link #markInitialized()}
      * method is called on future.
      */
-    public boolean initialized() {
+    public final boolean initialized() {
         return initFlag == INIT_FLAG;
     }
 
     /**
      * Mark this future as initialized.
      */
-    public void markInitialized() {
+    public final void markInitialized() {
         if (FLAGS_UPD.compareAndSet(this, 0, INIT_FLAG))
             checkComplete();
     }
@@ -295,9 +297,9 @@ private void checkComplete() {
      * @return Future.
      */
     @SuppressWarnings("unchecked")
-    protected IgniteInternalFuture<T> future(int idx) {
+    protected final IgniteInternalFuture<T> future(int idx) {
         assert Thread.holdsLock(sync);
-        assert futs != null && idx >= 0 && idx < futuresCount();
+        assert futs != null && idx >= 0 && idx < futuresCountNoLock();
 
         if (futs instanceof IgniteInternalFuture) {
             assert idx == 0;
@@ -312,15 +314,33 @@ protected IgniteInternalFuture<T> future(int idx) {
      * @return Futures size.
      */
     @SuppressWarnings("unchecked")
-    protected int futuresCount() {
-        synchronized (sync) {
-            if (futs == null)
-                return 0;
+    protected final int futuresCountNoLock() {
+        assert Thread.holdsLock(sync);
 
-            if (futs instanceof IgniteInternalFuture)
-                return 1;
+        if (futs == null)
+            return 0;
+
+        if (futs instanceof IgniteInternalFuture)
+            return 1;
 
-            return ((Collection<IgniteInternalFuture>)futs).size();
+        return ((Collection<IgniteInternalFuture>)futs).size();
+    }
+
+    /**
+     * @return Futures size.
+     */
+    private int futuresCount() {
+        synchronized (sync) {
+            return futuresCountNoLock();
+        }
+    }
+
+    /**
+     * @return {@code True} if has at least one future.
+     */
+    protected final boolean hasFutures() {
+        synchronized (sync) {
+            return futs != null;
         }
     }
 

From f3db74f782c68c7f73ef3ef4390e010976493634 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Wed, 7 Dec 2016 13:15:37 +0300
Subject: [PATCH 019/446] IGNITE-4238: Added geospatial queries example (nolgpl
 examples hotfix)

---
 examples/pom-standalone.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/examples/pom-standalone.xml b/examples/pom-standalone.xml
index 6247ced4ef758..08c27fe675d53 100644
--- a/examples/pom-standalone.xml
+++ b/examples/pom-standalone.xml
@@ -166,6 +166,12 @@
                     <artifactId>ignite-schedule</artifactId>
                     <version>to_be_replaced_by_ignite_version</version>
                 </dependency>
+
+                <dependency>
+                    <groupId>org.apache.ignite</groupId>
+                    <artifactId>ignite-geospatial</artifactId>
+                    <version>to_be_replaced_by_ignite_version</version>
+                </dependency>
             </dependencies>
         </profile>
     </profiles>

From 0d4a1b7381fece47ee480f3a06bff7c51a7fead4 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 7 Dec 2016 18:02:49 +0700
Subject: [PATCH 020/446] Improved exception handling for failed queries.

---
 .../org/apache/ignite/internal/visor/query/VisorQueryJob.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
index 6d1de6adb1a13..c66b2dda8d22c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
@@ -197,7 +197,7 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 }
             }
         }
-        catch (Exception e) {
+        catch (Throwable e) {
             return new IgniteBiTuple<>(new VisorExceptionWrapper(e), null);
         }
     }

From 56efb10c40fd4481d6a9dc00928e7beee1f164c5 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Wed, 7 Dec 2016 14:25:53 +0300
Subject: [PATCH 021/446] IGNITE-4353 Parent Cassandra module deployed on maven
 repository (hotfix: deploy to custom maven repository)

---
 modules/cassandra/pom.xml | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/modules/cassandra/pom.xml b/modules/cassandra/pom.xml
index 124c9b2de75da..a665538fc5567 100644
--- a/modules/cassandra/pom.xml
+++ b/modules/cassandra/pom.xml
@@ -51,18 +51,16 @@
     </modules>
 
     <build>
-        <pluginManagement>
-            <plugins>
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-deploy-plugin</artifactId>
-                    <version>2.8.2</version>
-                    <inherited>false</inherited>
-                    <configuration>
-                        <skip>true</skip>
-                    </configuration>
-                </plugin>
-            </plugins>
-        </pluginManagement>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <version>2.8.2</version>
+                <inherited>false</inherited>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
+        </plugins>
     </build>
 </project>

From ac8602dbdf2bbf5b16a611eaf6d520a0a7b0010b Mon Sep 17 00:00:00 2001
From: Sergi Vladykin <sergi.vladykin@gmail.com>
Date: Mon, 15 Aug 2016 16:46:54 +0300
Subject: [PATCH 022/446] ignite-3685 - fixed

---
 .../processors/query/h2/opt/GridH2Row.java       |  2 +-
 .../IgniteCacheAbstractFieldsQuerySelfTest.java  |  2 +-
 .../IgniteCacheLocalFieldsQuerySelfTest.java     | 16 ++++++++++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Row.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Row.java
index 9486a2ec6d432..8e7b1612c1ed3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Row.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Row.java
@@ -98,7 +98,7 @@ public abstract class GridH2Row extends Row implements GridSearchRowPointer {
 
     /** {@inheritDoc} */
     @Override public void setKey(long key) {
-        throw new UnsupportedOperationException();
+        // No-op, may be set in H2 INFORMATION_SCHEMA.
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
index 926d294ec1d25..d5f02ebdf980d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
@@ -71,7 +71,7 @@ public abstract class IgniteCacheAbstractFieldsQuerySelfTest extends GridCommonA
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
     /** Cache name. */
-    private static final String CACHE = "cache";
+    protected static final String CACHE = "cache";
 
     /** Empty cache name. */
     private static final String EMPTY_CACHE = "emptyCache";
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalFieldsQuerySelfTest.java
index be1f196ab5075..462118f1f8b9a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalFieldsQuerySelfTest.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.internal.processors.cache.local;
 
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractFieldsQuerySelfTest;
 
 import static org.apache.ignite.cache.CacheMode.LOCAL;
@@ -26,6 +28,10 @@
  * Tests for fields queries.
  */
 public class IgniteCacheLocalFieldsQuerySelfTest extends IgniteCacheAbstractFieldsQuerySelfTest {
+//    static {
+//        System.setProperty(IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE, "1");
+//    }
+
     /** {@inheritDoc} */
     @Override protected CacheMode cacheMode() {
         return LOCAL;
@@ -35,4 +41,14 @@ public class IgniteCacheLocalFieldsQuerySelfTest extends IgniteCacheAbstractFiel
     @Override protected int gridCount() {
         return 1;
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testInformationSchema() throws Exception {
+        IgniteEx ignite = grid(0);
+
+        ignite.cache(CACHE).query(
+            new SqlFieldsQuery("SELECT VALUE FROM INFORMATION_SCHEMA.SETTINGS").setLocal(true)).getAll();
+    }
 }
\ No newline at end of file

From bbaa79af8ef526b5d2684db0e3d71d60a8f1ebe7 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 7 Dec 2016 19:36:11 +0300
Subject: [PATCH 023/446] IGNITE-3770 GridLogThrottle.warn ignores the
 exception

---
 .../affinity/fair/FairAffinityFunction.java   |  2 +-
 .../RendezvousAffinityFunction.java           |  2 +-
 .../apache/ignite/internal/IgniteKernal.java  |  2 +-
 .../GridDeploymentPerVersionStore.java        |  2 +-
 .../discovery/GridDiscoveryManager.java       |  2 +-
 .../eventstorage/GridEventStorageManager.java |  2 +-
 .../cache/GridCacheDeploymentManager.java     |  4 +-
 .../cache/GridCacheEventManager.java          | 10 ++---
 .../store/GridCacheStoreManagerAdapter.java   |  2 +-
 .../store/GridCacheWriteBehindStore.java      |  2 +-
 .../clock/GridClockSyncProcessor.java         |  2 +-
 .../igfs/IgfsFragmentizerManager.java         | 29 +++++++------
 .../internal/processors/igfs/IgfsImpl.java    | 35 ++++++++--------
 .../OsDiscoveryNodeValidationProcessor.java   |  2 +-
 .../processors/task/GridTaskWorker.java       |  2 +-
 .../ignite/internal/util/GridLogThrottle.java | 35 ++++++++--------
 .../ignite/internal/util/IgniteUtils.java     |  6 +--
 .../shmem/IpcSharedMemoryNativeLoader.java    |  2 +-
 .../shmem/IpcSharedMemoryServerEndpoint.java  |  4 +-
 .../nio/GridConnectionBytesVerifyFilter.java  |  2 +-
 .../internal/util/nio/GridNioCodecFilter.java |  2 +-
 .../internal/util/nio/GridNioFilterChain.java |  2 +-
 .../internal/util/nio/GridNioServer.java      |  2 +-
 .../util/nio/GridSelectorNioSessionImpl.java  |  2 +-
 .../tcp/TcpCommunicationSpi.java              | 12 +++---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  7 ++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 41 ++++++++-----------
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |  2 +-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |  8 ++--
 .../TcpDiscoveryMulticastIpFinder.java        |  4 +-
 .../ignite/testframework/GridTestUtils.java   |  2 +-
 .../junits/common/GridCommonAbstractTest.java |  6 +--
 .../ignite/util/GridLogThrottleTest.java      | 27 ++++--------
 .../HadoopExternalCommunication.java          |  6 +--
 .../processors/query/h2/IgniteH2Indexing.java |  2 +-
 .../IgniteCacheOffheapEvictQueryTest.java     |  2 +-
 .../http/UriDeploymentHttpScanner.java        |  8 ++--
 37 files changed, 133 insertions(+), 151 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
index 105efabe529fc..cffcf108c18df 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
@@ -331,7 +331,7 @@ public void setExcludeNeighbors(boolean exclNeighbors) {
                 balance(tier, pendingParts, fullMap, topSnapshot, true);
 
                 if (!exclNeighborsWarn) {
-                    LT.warn(log, null, "Affinity function excludeNeighbors property is ignored " +
+                    LT.warn(log, "Affinity function excludeNeighbors property is ignored " +
                         "because topology has no enough nodes to assign backups.");
 
                     exclNeighborsWarn = true;
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
index 75e7c92b4f24c..cbd0136576b4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
@@ -462,7 +462,7 @@ else if (affinityBackupFilter == null && backupFilter == null)
             }
 
             if (!exclNeighborsWarn) {
-                LT.warn(log, null, "Affinity function excludeNeighbors property is ignored " +
+                LT.warn(log, "Affinity function excludeNeighbors property is ignored " +
                     "because topology has no enough nodes to assign backups.");
 
                 exclNeighborsWarn = true;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index c5365069b809b..8fda72fc18419 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1001,7 +1001,7 @@ else if (e instanceof IgniteCheckedException)
                     // at least one waiting request, then it is possible starvation.
                     if (exec.getPoolSize() == exec.getActiveCount() && completedCnt == lastCompletedCnt &&
                         !exec.getQueue().isEmpty())
-                        LT.warn(log, null, "Possible thread pool starvation detected (no task completed in last " +
+                        LT.warn(log, "Possible thread pool starvation detected (no task completed in last " +
                             interval + "ms, is executorService pool size large enough?)");
 
                     lastCompletedCnt = completedCnt;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentPerVersionStore.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentPerVersionStore.java
index 5e30bf6457707..0bf8328472237 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentPerVersionStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentPerVersionStore.java
@@ -317,7 +317,7 @@ else if (log.isDebugEnabled())
                         if (ctx.localNodeId().equals(e.getKey())) {
                             // Warn only if mode is not CONTINUOUS.
                             if (meta.deploymentMode() != CONTINUOUS)
-                                LT.warn(log, null, "Local node is in participants (most probably, " +
+                                LT.warn(log, "Local node is in participants (most probably, " +
                                     "IgniteConfiguration.getPeerClassLoadingLocalClassPathExclude() " +
                                     "is not used properly " +
                                     "[locNodeId=" + ctx.localNodeId() + ", meta=" + meta + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index ddd4ee3e07b4c..9aa4db1e0042f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -995,7 +995,7 @@ private void checkSegmentOnStart() throws IgniteCheckedException {
                 break;
 
             if (ctx.config().isWaitForSegmentOnStart()) {
-                LT.warn(log, null, "Failed to check network segment (retrying every 2000 ms).");
+                LT.warn(log, "Failed to check network segment (retrying every 2000 ms).");
 
                 // Wait and check again.
                 U.sleep(2000);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
index 5b451a17a0c3f..607bb9688aaf1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
@@ -278,7 +278,7 @@ public void record(Event evt) {
             int type = evt.type();
 
             if (!isRecordable(type)) {
-                LT.warn(log, null, "Trying to record event without checking if it is recordable: " +
+                LT.warn(log, "Trying to record event without checking if it is recordable: " +
                     U.gridEventName(type));
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java
index 8e662333f4ec5..ad4892bcea09e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java
@@ -399,7 +399,7 @@ public void p2pContext(UUID sndId, IgniteUuid ldrId, String userVer, DeploymentM
                     ", daemon=" + daemon + ']');
 
             if (!daemon) {
-                LT.warn(log, null, "Ignoring deployment in PRIVATE or ISOLATED mode " +
+                LT.warn(log, "Ignoring deployment in PRIVATE or ISOLATED mode " +
                     "[sndId=" + sndId + ", ldrId=" + ldrId + ", userVer=" + userVer + ", mode=" + mode +
                     ", participants=" + participants + ", daemon=" + daemon + ']');
             }
@@ -408,7 +408,7 @@ public void p2pContext(UUID sndId, IgniteUuid ldrId, String userVer, DeploymentM
         }
 
         if (mode != cctx.gridConfig().getDeploymentMode()) {
-            LT.warn(log, null, "Local and remote deployment mode mismatch (please fix configuration and restart) " +
+            LT.warn(log, "Local and remote deployment mode mismatch (please fix configuration and restart) " +
                 "[locDepMode=" + cctx.gridConfig().getDeploymentMode() + ", rmtDepMode=" + mode + ", rmtNodeId=" +
                 sndId + ']');
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java
index ec8b8cc453e7b..1c1873814e675 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java
@@ -251,7 +251,7 @@ public void addEvent(
         assert key != null || type == EVT_CACHE_STARTED || type == EVT_CACHE_STOPPED;
 
         if (!cctx.events().isRecordable(type))
-            LT.warn(log, null, "Added event without checking if event is recordable: " + U.gridEventName(type));
+            LT.warn(log, "Added event without checking if event is recordable: " + U.gridEventName(type));
 
         // Events are not fired for internal entry.
         if (key == null || !key.internal()) {
@@ -261,7 +261,7 @@ public void addEvent(
                 evtNode = findNodeInHistory(evtNodeId);
 
             if (evtNode == null)
-                LT.warn(log, null, "Failed to find event node in grid topology history " +
+                LT.warn(log, "Failed to find event node in grid topology history " +
                     "(try to increase topology history size configuration property of configured " +
                     "discovery SPI): " + evtNodeId);
 
@@ -284,7 +284,7 @@ public void addEvent(
                     log.debug("Failed to unmarshall cache object value for the event notification: " + e);
 
                 if (!forceKeepBinary)
-                    LT.warn(log, null, "Failed to unmarshall cache object value for the event notification " +
+                    LT.warn(log, "Failed to unmarshall cache object value for the event notification " +
                         "(all further notifications will keep binary object format).");
 
                 forceKeepBinary = true;
@@ -351,7 +351,7 @@ public void addPreloadEvent(int part, int type, ClusterNode discoNode, int disco
         assert discoTs > 0;
 
         if (!cctx.events().isRecordable(type))
-            LT.warn(log, null, "Added event without checking if event is recordable: " + U.gridEventName(type));
+            LT.warn(log, "Added event without checking if event is recordable: " + U.gridEventName(type));
 
         cctx.gridEvents().record(new CacheRebalancingEvent(cctx.name(), cctx.localNode(),
             "Cache rebalancing event.", type, part, discoNode, discoType, discoTs));
@@ -364,7 +364,7 @@ public void addPreloadEvent(int part, int type, ClusterNode discoNode, int disco
      */
     public void addUnloadEvent(int part) {
         if (!cctx.events().isRecordable(EVT_CACHE_REBALANCE_PART_UNLOADED))
-            LT.warn(log, null, "Added event without checking if event is recordable: " +
+            LT.warn(log, "Added event without checking if event is recordable: " +
                 U.gridEventName(EVT_CACHE_REBALANCE_PART_UNLOADED));
 
         cctx.gridEvents().record(new CacheRebalancingEvent(cctx.name(), cctx.localNode(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index cd0c50f57a1f8..024375e75c477 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -547,7 +547,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
             return true;
         }
 
-        LT.warn(log, null, "Calling Cache.loadCache() method will have no effect, " +
+        LT.warn(log, "Calling Cache.loadCache() method will have no effect, " +
             "CacheConfiguration.getStore() is not defined for cache: " + cctx.namexx());
 
         return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
index 468945b97ec6c..858d9a7498510 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
@@ -705,7 +705,7 @@ private boolean updateStore(StoreOperation operation,
             }
         }
         catch (Exception e) {
-            LT.warn(log, e, "Unable to update underlying store: " + store);
+            LT.error(log, e, "Unable to update underlying store: " + store);
 
             if (writeCache.sizex() > cacheCriticalSize || stopping.get()) {
                 for (Map.Entry<K, Entry<? extends K, ? extends  V>> entry : vals.entrySet()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
index b5c89cf5834f2..07643164368ce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
@@ -458,7 +458,7 @@ private void requestTime(UUID rmtNodeId) {
                     srv.sendPacket(req, addr, port);
                 }
                 catch (IgniteCheckedException e) {
-                    LT.warn(log, e, "Failed to send time request to remote node [rmtNodeId=" + rmtNodeId +
+                    LT.error(log, e, "Failed to send time request to remote node [rmtNodeId=" + rmtNodeId +
                         ", addr=" + addr + ", port=" + port + ']');
                 }
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
index d64c64ad1e02f..2e82f33024b57 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
@@ -17,6 +17,19 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
@@ -41,20 +54,6 @@
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
@@ -383,7 +382,7 @@ protected FragmentizerCoordinator() {
                 }
                 catch (IgniteCheckedException | IgniteException e) {
                     if (!X.hasCause(e, InterruptedException.class) && !X.hasCause(e, IgniteInterruptedCheckedException.class))
-                        LT.warn(log, e, "Failed to get fragmentizer file info (will retry).");
+                        LT.error(log, e, "Failed to get fragmentizer file info (will retry).");
                     else {
                         if (log.isDebugEnabled())
                             log.debug("Got interrupted exception in fragmentizer coordinator (grid is stopping).");
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 1c985c0d6ed5c..ab4ee8533a051 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -17,6 +17,21 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -81,22 +96,6 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
-import java.io.OutputStream;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
 import static org.apache.ignite.events.EventType.EVT_IGFS_DIR_DELETED;
 import static org.apache.ignite.events.EventType.EVT_IGFS_FILE_DELETED;
 import static org.apache.ignite.events.EventType.EVT_IGFS_FILE_OPENED_READ;
@@ -1308,7 +1307,7 @@ private IgfsOutputStream create0(
                         secondarySpaceSize = secondaryFs.usedSpaceSize();
                     }
                     catch (IgniteException e) {
-                        LT.warn(log, e, "Failed to get secondary file system consumed space size.");
+                        LT.error(log, e, "Failed to get secondary file system consumed space size.");
 
                         secondarySpaceSize = -1;
                     }
@@ -1841,4 +1840,4 @@ public FormatRunnable(GridFutureAdapter<?> fut) {
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/nodevalidation/OsDiscoveryNodeValidationProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/nodevalidation/OsDiscoveryNodeValidationProcessor.java
index a7e06e9beb918..37e59bc20262a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/nodevalidation/OsDiscoveryNodeValidationProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/nodevalidation/OsDiscoveryNodeValidationProcessor.java
@@ -58,7 +58,7 @@ public OsDiscoveryNodeValidationProcessor(GridKernalContext ctx) {
                     ", rmtNodeAddrs=" + U.addressesAsString(node) +
                     ", locNodeId=" + locNode.id() + ", rmtNodeId=" + node.id() + ']';
 
-                LT.warn(log, null, errMsg);
+                LT.warn(log, errMsg);
 
                 // Always output in debug.
                 if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
index 0be69d126bb36..3478c70a86e11 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
@@ -626,7 +626,7 @@ private void processMappedJobs(Map<? extends ComputeJob, ClusterNode> jobs) thro
                 res.setOccupied(true);
 
                 if (resCache && jobRes.size() > ctx.discovery().size() && jobRes.size() % SPLIT_WARN_THRESHOLD == 0)
-                    LT.warn(log, null, "Number of jobs in task is too large for task: " + ses.getTaskName() +
+                    LT.warn(log, "Number of jobs in task is too large for task: " + ses.getTaskName() +
                         ". Consider reducing number of jobs or disabling job result cache with " +
                         "@ComputeTaskNoResultCache annotation.");
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java
index c8ba86502746b..7f30dd71e0f31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java
@@ -93,13 +93,12 @@ public static void error(@Nullable IgniteLogger log, @Nullable Throwable e, Stri
      * Logs warning if needed.
      *
      * @param log Logger.
-     * @param e Error (optional).
      * @param msg Message.
      */
-    public static void warn(@Nullable IgniteLogger log, @Nullable Throwable e, String msg) {
+    public static void warn(@Nullable IgniteLogger log, String msg) {
         assert !F.isEmpty(msg);
 
-        log(log, e, msg, null, LogLevel.WARN, false, false);
+        log(log, null, msg, null, LogLevel.WARN, false, false);
     }
 
     /**
@@ -122,28 +121,26 @@ public static void warn(@Nullable IgniteLogger log, @Nullable Throwable e, Strin
      * Logs warning if needed.
      *
      * @param log Logger.
-     * @param e Error (optional).
      * @param msg Message.
-     * @param quite Print warning anyway.
+     * @param quiet Print warning anyway.
      */
-    public static void warn(@Nullable IgniteLogger log, @Nullable Throwable e, String msg, boolean quite) {
+    public static void warn(@Nullable IgniteLogger log, String msg, boolean quiet) {
         assert !F.isEmpty(msg);
 
-        log(log, e, msg, null, LogLevel.WARN, quite, false);
+        log(log, null, msg, null, LogLevel.WARN, quiet, false);
     }
 
     /**
      * Logs warning if needed.
      *
      * @param log Logger.
-     * @param e Error (optional).
      * @param longMsg Long message (or just message).
-     * @param shortMsg Short message for quite logging.
+     * @param shortMsg Short message for quiet logging.
      */
-    public static void warn(@Nullable IgniteLogger log, @Nullable Throwable e, String longMsg, @Nullable String shortMsg) {
+    public static void warn(@Nullable IgniteLogger log, String longMsg, @Nullable String shortMsg) {
         assert !F.isEmpty(longMsg);
 
-        log(log, e, longMsg, shortMsg, LogLevel.WARN, false, false);
+        log(log, null, longMsg, shortMsg, LogLevel.WARN, false, false);
     }
 
     /**
@@ -151,12 +148,12 @@ public static void warn(@Nullable IgniteLogger log, @Nullable Throwable e, Strin
      *
      * @param log Logger.
      * @param msg Message.
-     * @param quite Print info anyway.
+     * @param quiet Print info anyway.
      */
-    public static void info(@Nullable IgniteLogger log, String msg, boolean quite) {
+    public static void info(@Nullable IgniteLogger log, String msg, boolean quiet) {
         assert !F.isEmpty(msg);
 
-        log(log, null, msg, null, LogLevel.INFO, quite, false);
+        log(log, null, msg, null, LogLevel.INFO, quiet, false);
     }
 
     /**
@@ -166,6 +163,8 @@ public static void info(@Nullable IgniteLogger log, String msg, boolean quite) {
      * @param msg Message.
      */
     public static void info(@Nullable IgniteLogger log, String msg) {
+        assert !F.isEmpty(msg);
+
         info(log, msg, false);
     }
 
@@ -182,13 +181,13 @@ public static void clear() {
      * @param log Logger.
      * @param e Error (optional).
      * @param longMsg Long message (or just message).
-     * @param shortMsg Short message for quite logging.
+     * @param shortMsg Short message for quiet logging.
      * @param level Level where messages should appear.
      * @param byMsg Errors group by message, not by tuple(error, msg).
      */
     @SuppressWarnings({"RedundantTypeArguments"})
-    private static void log(@Nullable IgniteLogger log, @Nullable Throwable e, String longMsg, @Nullable String shortMsg,
-        LogLevel level, boolean quiet, boolean byMsg) {
+    private static void log(@Nullable IgniteLogger log, @Nullable Throwable e, String longMsg,
+        @Nullable String shortMsg, LogLevel level, boolean quiet, boolean byMsg) {
         assert !F.isEmpty(longMsg);
 
         IgniteBiTuple<Class<? extends Throwable>, String> tup =
@@ -283,4 +282,4 @@ private enum LogLevel {
          */
         public abstract void doLog(IgniteLogger log, String longMsg, String shortMsg, Throwable e, boolean quiet);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 914b3ec32f3e9..3fa3f7b92eccb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -4105,7 +4105,7 @@ public static void warn(@Nullable IgniteLogger log, Object msg) {
     }
 
     /**
-     * Logs warning message in both verbose and quite modes.
+     * Logs warning message in both verbose and quiet modes.
      *
      * @param log Logger to use.
      * @param msg Message to log.
@@ -4115,7 +4115,7 @@ public static void quietAndWarn(IgniteLogger log, Object msg) {
     }
 
     /**
-     * Logs warning message in both verbose and quite modes.
+     * Logs warning message in both verbose and quiet modes.
      *
      * @param log Logger to use.
      * @param shortMsg Short message.
@@ -4285,7 +4285,7 @@ public static void quiet(boolean err, Object... objs) {
     }
 
     /**
-     * Prints out the message in quite and info modes.
+     * Prints out the message in quiet and info modes.
      *
      * @param log Logger.
      * @param msg Message to print.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNativeLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNativeLoader.java
index 2771d28fc5421..02c4de58719e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNativeLoader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNativeLoader.java
@@ -150,7 +150,7 @@ private static void doLoad(IgniteLogger log) throws IgniteCheckedException {
 
             try {
                 if (log != null)
-                    LT.warn(log, null, "Failed to load 'igniteshmem' library from classpath. Will try to load it from IGNITE_HOME.");
+                    LT.warn(log, "Failed to load 'igniteshmem' library from classpath. Will try to load it from IGNITE_HOME.");
 
                 String igniteHome = X.resolveIgniteHome();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryServerEndpoint.java b/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryServerEndpoint.java
index 94c3820a9004b..178e6081d9403 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryServerEndpoint.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryServerEndpoint.java
@@ -304,13 +304,13 @@ public void omitOutOfResourcesWarning(boolean omitOutOfResourcesWarn) {
                     String msg = "Failed to process incoming connection (most probably, shared memory " +
                         "rest endpoint has been configured by mistake).";
 
-                    LT.warn(log, null, msg);
+                    LT.warn(log, msg);
 
                     sendErrorResponse(out, e);
                 }
                 catch (IpcOutOfSystemResourcesException e) {
                     if (!omitOutOfResourcesWarn)
-                        LT.warn(log, null, OUT_OF_RESOURCES_MSG);
+                        LT.warn(log, OUT_OF_RESOURCES_MSG);
 
                     sendErrorResponse(out, e);
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java
index 13d7ca7cc2e96..213fd8dd040a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java
@@ -115,7 +115,7 @@ else if (U.bytesEqual(magicBuf, 0, U.IGNITE_HEADER, 0, U.IGNITE_HEADER.length))
             else {
                 ses.close();
 
-                LT.warn(log, null, "Unknown connection detected (is some other software connecting to this " +
+                LT.warn(log, "Unknown connection detected (is some other software connecting to this " +
                     "Ignite port?) [rmtAddr=" + ses.remoteAddress() + ", locAddr=" + ses.localAddress() + ']');
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java
index a2f543d5338a6..7083ccf492f17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java
@@ -110,7 +110,7 @@ public GridNioCodecFilter(GridNioParser parser, IgniteLogger log, boolean direct
                         if (directMode)
                             return;
 
-                        LT.warn(log, null, "Parser returned null but there are still unread data in input buffer (bug in " +
+                        LT.warn(log, "Parser returned null but there are still unread data in input buffer (bug in " +
                             "parser code?) [parser=" + parser + ", ses=" + ses + ']');
 
                         input.position(input.limit());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java
index 8a43e2938832f..a3a74e3c235a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java
@@ -158,7 +158,7 @@ public String toString() {
             head.onExceptionCaught(ses, e);
         }
         catch (Exception ex) {
-            LT.warn(log, ex, "Failed to forward GridNioException to filter chain [ses=" + ses + ", e=" + e + ']');
+            LT.error(log, ex, "Failed to forward GridNioException to filter chain [ses=" + ses + ", e=" + e + ']');
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 24b8fad0c8d7f..c8e2e0be067a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -768,7 +768,7 @@ else if (cnt == 0)
                 filterChain.onMessageReceived(ses, readBuf);
 
                 if (readBuf.remaining() > 0) {
-                    LT.warn(log, null, "Read buffer contains data after filter chain processing (will discard " +
+                    LT.warn(log, "Read buffer contains data after filter chain processing (will discard " +
                         "remaining bytes) [ses=" + ses + ", remainingCnt=" + readBuf.remaining() + ']');
 
                     readBuf.clear();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
index 0ba6af2bd1c58..63c9845ca3fc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
@@ -227,7 +227,7 @@ void resend(Collection<GridNioFuture<?>> futs) {
 
             if (recovery != null) {
                 if (!recovery.add(last)) {
-                    LT.warn(log, null, "Unacknowledged messages queue size overflow, will attempt to reconnect " +
+                    LT.warn(log, "Unacknowledged messages queue size overflow, will attempt to reconnect " +
                         "[remoteAddr=" + remoteAddress() +
                         ", queueLimit=" + recovery.queueLimit() + ']');
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 767490fb7e90f..1fe437cc710c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -331,7 +331,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     private final GridNioServerListener<Message> srvLsnr =
         new GridNioServerListenerAdapter<Message>() {
             @Override public void onSessionWriteTimeout(GridNioSession ses) {
-                LT.warn(log, null, "Communication SPI Session write timed out (consider increasing " +
+                LT.warn(log, "Communication SPI Session write timed out (consider increasing " +
                     "'socketWriteTimeout' " + "configuration property) [remoteAddr=" + ses.remoteAddress() +
                     ", writeTimeout=" + sockWriteTimeout + ']');
 
@@ -2146,9 +2146,9 @@ private GridCommunicationClient reserveClient(ClusterNode node) throws IgniteChe
             catch (IgniteCheckedException e) {
                 if (e.hasCause(IpcOutOfSystemResourcesException.class))
                     // Has cause or is itself the IpcOutOfSystemResourcesException.
-                    LT.warn(log, null, OUT_OF_RESOURCES_TCP_MSG);
+                    LT.warn(log, OUT_OF_RESOURCES_TCP_MSG);
                 else if (getSpiContext().node(node.id()) != null)
-                    LT.warn(log, null, e.getMessage());
+                    LT.warn(log, e.getMessage());
                 else if (log.isDebugEnabled())
                     log.debug("Failed to establish shared memory connection with local node (node has left): " +
                         node.id());
@@ -2510,11 +2510,11 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                     boolean failureDetThrReached = timeoutHelper.checkFailureTimeoutReached(e);
 
                     if (failureDetThrReached)
-                        LT.warn(log, null, "Connect timed out (consider increasing 'failureDetectionTimeout' " +
+                        LT.warn(log, "Connect timed out (consider increasing 'failureDetectionTimeout' " +
                             "configuration property) [addr=" + addr + ", failureDetectionTimeout=" +
                             failureDetectionTimeout() + ']');
                     else if (X.hasCause(e, SocketTimeoutException.class))
-                        LT.warn(log, null, "Connect timed out (consider increasing 'connTimeout' " +
+                        LT.warn(log, "Connect timed out (consider increasing 'connTimeout' " +
                             "configuration property) [addr=" + addr + ", connTimeout=" + connTimeout + ']');
 
                     if (errs == null)
@@ -2545,7 +2545,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
             assert errs != null;
 
             if (X.hasCause(errs, ConnectException.class))
-                LT.warn(log, null, "Failed to connect to a remote node " +
+                LT.warn(log, "Failed to connect to a remote node " +
                     "(make sure that destination node is alive and " +
                     "operating system firewall is disabled on local and remote hosts) " +
                     "[addrs=" + addrs + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 7b8c029f03d38..8928f28d4fd27 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -64,7 +64,6 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
-import org.apache.ignite.marshaller.MarshallerUtils;
 import org.apache.ignite.spi.IgniteSpiContext;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
@@ -483,7 +482,7 @@ else if (state == DISCONNECTED) {
                     if (timeout > 0 && (U.currentTimeMillis() - startTime) > timeout)
                         return null;
 
-                    LT.warn(log, null, "IP finder returned empty addresses list. " +
+                    LT.warn(log, "IP finder returned empty addresses list. " +
                             "Please check IP finder configuration" +
                             (spi.ipFinder instanceof TcpDiscoveryMulticastIpFinder ?
                                 " and make sure multicast works on your network. " : ". ") +
@@ -553,7 +552,7 @@ else if (addrs.isEmpty()) {
                 if (timeout > 0 && (U.currentTimeMillis() - startTime) > timeout)
                     return null;
 
-                LT.warn(log, null, "Failed to connect to any address from IP finder (will retry to join topology " +
+                LT.warn(log, "Failed to connect to any address from IP finder (will retry to join topology " +
                     "every 2 secs): " + toOrderedList(addrs0), true);
 
                 Thread.sleep(2000);
@@ -917,7 +916,7 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
                             ClassNotFoundException clsNotFoundEx = X.cause(e, ClassNotFoundException.class);
 
                             if (clsNotFoundEx != null)
-                                LT.warn(log, null, "Failed to read message due to ClassNotFoundException " +
+                                LT.warn(log, "Failed to read message due to ClassNotFoundException " +
                                     "(make sure same versions of all classes are available on all nodes) " +
                                     "[rmtNodeId=" + rmtNodeId + ", err=" + clsNotFoundEx.getMessage() + ']');
                             else
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index a660ec8bb09ae..204b685984be3 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -54,7 +54,6 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.net.ssl.SSLException;
@@ -529,7 +528,7 @@ else if (log.isInfoEnabled()) {
         boolean res = pingNode(node);
 
         if (!res && !node.isClient() && nodeAlive(nodeId)) {
-            LT.warn(log, null, "Failed to ping node (status check will be initiated): " + nodeId);
+            LT.warn(log, "Failed to ping node (status check will be initiated): " + nodeId);
 
             msgWorker.addMessage(new TcpDiscoveryStatusCheckMessage(locNode, node.id()));
         }
@@ -910,7 +909,7 @@ else if (spiState == LOOPBACK_PROBLEM) {
                         U.addressesAsString(msg.addresses(), msg.hostNames()) + ']');
                 }
                 else
-                    LT.warn(log, null, "Node has not been connected to topology and will repeat join process. " +
+                    LT.warn(log, "Node has not been connected to topology and will repeat join process. " +
                         "Check remote nodes logs for possible error messages. " +
                         "Note that large topology may require significant time to start. " +
                         "Increase 'TcpDiscoverySpi.networkTimeout' configuration property " +
@@ -1059,7 +1058,7 @@ else if (!spi.ipFinder.isShared() && !ipFinderHasLocAddr) {
                 }
 
                 if (e != null && X.hasCause(e, ConnectException.class)) {
-                    LT.warn(log, null, "Failed to connect to any address from IP finder " +
+                    LT.warn(log, "Failed to connect to any address from IP finder " +
                         "(make sure IP finder addresses are correct and firewalls are disabled on all host machines): " +
                         toOrderedList(addrs), true);
                 }
@@ -3148,7 +3147,7 @@ else if (log.isDebugEnabled())
                     }
                 }
 
-                LT.warn(log, null, "Local node has detected failed nodes and started cluster-wide procedure. " +
+                LT.warn(log, "Local node has detected failed nodes and started cluster-wide procedure. " +
                         "To speed up failure detection please see 'Failure Detection' section under javadoc" +
                         " for 'TcpDiscoverySpi'");
             }
@@ -3233,7 +3232,7 @@ private void processJoinRequestMessage(final TcpDiscoveryJoinRequestMessage msg)
                         "[locNodeAddrs=" + U.addressesAsString(locNode) +
                         ", rmtNodeAddrs=" + U.addressesAsString(node) + ']';
 
-                    LT.warn(log, null, errMsg);
+                    LT.warn(log, errMsg);
 
                     // Always output in debug.
                     if (log.isDebugEnabled())
@@ -3286,7 +3285,7 @@ private void processJoinRequestMessage(final TcpDiscoveryJoinRequestMessage msg)
                         }
 
                         // Output warning.
-                        LT.warn(log, null, "Ignoring join request from node (duplicate ID) [node=" + node +
+                        LT.warn(log, "Ignoring join request from node (duplicate ID) [node=" + node +
                             ", existingNode=" + existingNode + ']');
 
                         // Ignore join request.
@@ -3341,8 +3340,7 @@ else if (log.isDebugEnabled())
 
                         if (subj == null) {
                             // Node has not pass authentication.
-                            LT.warn(log, null,
-                                "Authentication failed [nodeId=" + node.id() +
+                            LT.warn(log, "Authentication failed [nodeId=" + node.id() +
                                     ", addrs=" + U.addressesAsString(node) + ']',
                                 "Authentication failed [nodeId=" + U.id8(node.id()) + ", addrs=" +
                                     U.addressesAsString(node) + ']');
@@ -3371,8 +3369,7 @@ else if (log.isDebugEnabled())
                         else {
                             if (!(subj instanceof Serializable)) {
                                 // Node has not pass authentication.
-                                LT.warn(log, null,
-                                    "Authentication subject is not Serializable [nodeId=" + node.id() +
+                                LT.warn(log, "Authentication subject is not Serializable [nodeId=" + node.id() +
                                         ", addrs=" + U.addressesAsString(node) + ']',
                                     "Authentication subject is not Serializable [nodeId=" + U.id8(node.id()) +
                                         ", addrs=" +
@@ -3442,7 +3439,7 @@ else if (log.isDebugEnabled())
                                     return;
                                 }
 
-                                LT.warn(log, null, err.message());
+                                LT.warn(log, err.message());
 
                                 // Always output in debug.
                                 if (log.isDebugEnabled())
@@ -3483,7 +3480,7 @@ else if (log.isDebugEnabled())
                                     ", rmtNodeAddrs=" + U.addressesAsString(node) +
                                     ", locNodeId=" + locNode.id() + ", rmtNodeId=" + msg.creatorNodeId() + ']';
 
-                                LT.warn(log, null, errMsg);
+                                LT.warn(log, errMsg);
 
                                 // Always output in debug.
                                 if (log.isDebugEnabled())
@@ -3771,7 +3768,7 @@ else if (sendMessageToRemotes(msg))
          * @param sndMsg Message to send.
          */
         private void nodeCheckError(TcpDiscoveryNode node, String errMsg, String sndMsg) {
-            LT.warn(log, null, errMsg);
+            LT.warn(log, errMsg);
 
             // Always output in debug.
             if (log.isDebugEnabled())
@@ -4056,8 +4053,7 @@ else if (!locNodeId.equals(node.id()) && ring.node(node.id()) != null) {
 
                             if (!permissionsEqual(coordSubj.subject().permissions(), subj.subject().permissions())) {
                                 // Node has not pass authentication.
-                                LT.warn(log, null,
-                                    "Authentication failed [nodeId=" + node.id() +
+                                LT.warn(log, "Authentication failed [nodeId=" + node.id() +
                                         ", addrs=" + U.addressesAsString(node) + ']',
                                     "Authentication failed [nodeId=" + U.id8(node.id()) + ", addrs=" +
                                         U.addressesAsString(node) + ']');
@@ -4148,7 +4144,6 @@ else if (!locNodeId.equals(node.id()) && ring.node(node.id()) != null) {
                                         rmCrd.subject().permissions())) {
                                         // Node has not pass authentication.
                                         LT.warn(log,
-                                            null,
                                             "Failed to authenticate local node " +
                                                 "(local authentication result is different from rest of topology) " +
                                                 "[nodeId=" + node.id() + ", addrs=" + U.addressesAsString(node) + ']',
@@ -5593,7 +5588,7 @@ private class SocketReader extends IgniteSpiThread {
                                     "[rmtAddr=" + sock.getRemoteSocketAddress() +
                                     ", locAddr=" + sock.getLocalSocketAddress() + ']');
 
-                            LT.warn(log, null, "Failed to read magic header (too few bytes received) [rmtAddr=" +
+                            LT.warn(log, "Failed to read magic header (too few bytes received) [rmtAddr=" +
                                 sock.getRemoteSocketAddress() + ", locAddr=" + sock.getLocalSocketAddress() + ']');
 
                             return;
@@ -5609,7 +5604,7 @@ private class SocketReader extends IgniteSpiThread {
                                 "[rmtAddr=" + sock.getRemoteSocketAddress() +
                                 ", locAddr=" + sock.getLocalSocketAddress() + ']');
 
-                        LT.warn(log, null, "Unknown connection detected (is some other software connecting to " +
+                        LT.warn(log, "Unknown connection detected (is some other software connecting to " +
                             "this Ignite port?" +
                             (!spi.isSslEnabled() ? " missing SSL configuration on remote node?" : "" ) +
                             ") [rmtAddr=" + sock.getInetAddress() + ']', true);
@@ -5729,7 +5724,7 @@ else if (log.isDebugEnabled())
                         U.error(log, "Caught exception on handshake [err=" + e +", sock=" + sock + ']', e);
 
                     if (X.hasCause(e, SSLException.class) && spi.isSslEnabled() && !spi.isNodeStopping0())
-                        LT.warn(log, null, "Failed to initialize connection " +
+                        LT.warn(log, "Failed to initialize connection " +
                             "(missing SSL configuration on remote node?) " +
                             "[rmtAddr=" + sock.getInetAddress() + ']', true);
                     else if ((X.hasCause(e, ObjectStreamException.class) || !sock.isClosed())
@@ -5758,12 +5753,12 @@ else if ((X.hasCause(e, ObjectStreamException.class) || !sock.isClosed())
                     onException("Caught exception on handshake [err=" + e +", sock=" + sock + ']', e);
 
                     if (e.hasCause(SocketTimeoutException.class))
-                        LT.warn(log, null, "Socket operation timed out on handshake " +
+                        LT.warn(log, "Socket operation timed out on handshake " +
                             "(consider increasing 'networkTimeout' configuration property) " +
                             "[netTimeout=" + spi.netTimeout + ']');
 
                     else if (e.hasCause(ClassNotFoundException.class))
-                        LT.warn(log, null, "Failed to read message due to ClassNotFoundException " +
+                        LT.warn(log, "Failed to read message due to ClassNotFoundException " +
                             "(make sure same versions of all classes are available on all nodes) " +
                             "[rmtAddr=" + sock.getRemoteSocketAddress() +
                             ", err=" + X.cause(e, ClassNotFoundException.class).getMessage() + ']');
@@ -5995,7 +5990,7 @@ else if (msg instanceof TcpDiscoveryLoopbackProblemMessage) {
                             return;
 
                         if (e.hasCause(ClassNotFoundException.class))
-                            LT.warn(log, null, "Failed to read message due to ClassNotFoundException " +
+                            LT.warn(log, "Failed to read message due to ClassNotFoundException " +
                                 "(make sure same versions of all classes are available on all nodes) " +
                                 "[rmtNodeId=" + nodeId +
                                 ", err=" + X.cause(e, ClassNotFoundException.class).getMessage() + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index 0816cbca120eb..f199c20e17edb 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -329,7 +329,7 @@ protected final void registerLocalNodeAddress() throws IgniteSpiException {
      */
     protected boolean checkAckTimeout(long ackTimeout) {
         if (ackTimeout > spi.getMaxAckTimeout()) {
-            LT.warn(log, null, "Acknowledgement timeout is greater than maximum acknowledgement timeout " +
+            LT.warn(log, "Acknowledgement timeout is greater than maximum acknowledgement timeout " +
                 "(consider increasing 'maxAckTimeout' configuration property) " +
                 "[ackTimeout=" + ackTimeout + ", maxAckTimeout=" + spi.getMaxAckTimeout() + ']');
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index a8704e7ab66f5..45933e112f78d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1471,7 +1471,7 @@ protected <T> T readMessage(Socket sock, @Nullable InputStream in, long timeout)
         }
         catch (IOException | IgniteCheckedException e) {
             if (X.hasCause(e, SocketTimeoutException.class))
-                LT.warn(log, null, "Timed out waiting for message to be read (most probably, the reason is " +
+                LT.warn(log, "Timed out waiting for message to be read (most probably, the reason is " +
                     "in long GC pauses on remote node) [curTimeout=" + timeout + ']');
 
             throw e;
@@ -1511,7 +1511,7 @@ protected int readReceipt(Socket sock, long timeout) throws IOException {
             return res;
         }
         catch (SocketTimeoutException e) {
-            LT.warn(log, null, "Timed out waiting for message delivery receipt (most probably, the reason is " +
+            LT.warn(log, "Timed out waiting for message delivery receipt (most probably, the reason is " +
                 "in long GC pauses on remote node; consider tuning GC and increasing 'ackTimeout' " +
                 "configuration property). Will retry to send message with increased timeout. " +
                 "Current timeout: " + timeout + '.');
@@ -1575,7 +1575,7 @@ protected Collection<InetSocketAddress> resolvedAddresses() throws IgniteSpiExce
                     res.add(resolved);
             }
             catch (UnknownHostException ignored) {
-                LT.warn(log, null, "Failed to resolve address from IP finder (host is unknown): " + addr);
+                LT.warn(log, "Failed to resolve address from IP finder (host is unknown): " + addr);
 
                 // Add address in any case.
                 res.add(addr);
@@ -2045,7 +2045,7 @@ boolean cancel() {
                 // Close socket - timeout occurred.
                 U.closeQuiet(sock);
 
-                LT.warn(log, null, "Socket write has timed out (consider increasing " +
+                LT.warn(log, "Socket write has timed out (consider increasing " +
                     (failureDetectionTimeoutEnabled() ?
                     "'IgniteConfiguration.failureDetectionTimeout' configuration property) [" +
                     "failureDetectionTimeout=" + failureDetectionTimeout() + ']' :
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/multicast/TcpDiscoveryMulticastIpFinder.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/multicast/TcpDiscoveryMulticastIpFinder.java
index e96abe96550f6..8fe8a65719c16 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/multicast/TcpDiscoveryMulticastIpFinder.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/multicast/TcpDiscoveryMulticastIpFinder.java
@@ -598,7 +598,7 @@ private T2<Collection<InetSocketAddress>, Boolean> requestAddresses(InetAddress
                                 addrRes = new AddressResponse(data);
                             }
                             catch (IgniteCheckedException e) {
-                                LT.warn(log, e, "Failed to deserialize multicast response.");
+                                LT.error(log, e, "Failed to deserialize multicast response.");
 
                                 continue;
                             }
@@ -876,7 +876,7 @@ private MulticastSocket createSocket() throws IOException {
                 }
                 catch (IOException e) {
                     if (!isInterrupted()) {
-                        LT.warn(log, e, "Failed to send/receive address message (will try to reconnect).");
+                        LT.error(log, e, "Failed to send/receive address message (will try to reconnect).");
 
                         synchronized (this) {
                             U.close(sock);
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index b3ce46b9ed1d4..0ae6575acc7c1 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -1061,7 +1061,7 @@ public static <K, V> void waitTopologyUpdate(@Nullable String cacheName, int bac
                     Collection<ClusterNode> nodes = top.nodes(p, AffinityTopologyVersion.NONE);
 
                     if (nodes.size() > backups + 1) {
-                        LT.warn(log, null, "Partition map was not updated yet (will wait) [grid=" + g.name() +
+                        LT.warn(log, "Partition map was not updated yet (will wait) [grid=" + g.name() +
                             ", p=" + p + ", nodes=" + F.nodeIds(nodes) + ']');
 
                         wait = true;
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index d5f6b1c3bea66..90fabd95a1b47 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -507,7 +507,7 @@ protected void awaitPartitionMapExchange(boolean waitEvicts,
 
                                 if (affNodes.size() != owners.size() || !affNodes.containsAll(owners) ||
                                     (waitEvicts && loc != null && loc.state() != GridDhtPartitionState.OWNING)) {
-                                    LT.warn(log(), null, "Waiting for topology map update [" +
+                                    LT.warn(log(), "Waiting for topology map update [" +
                                         "grid=" + g.name() +
                                         ", cache=" + cfg.getName() +
                                         ", cacheId=" + dht.context().cacheId() +
@@ -524,7 +524,7 @@ protected void awaitPartitionMapExchange(boolean waitEvicts,
                                     match = true;
                             }
                             else {
-                                LT.warn(log(), null, "Waiting for topology map update [" +
+                                LT.warn(log(), "Waiting for topology map update [" +
                                     "grid=" + g.name() +
                                     ", cache=" + cfg.getName() +
                                     ", cacheId=" + dht.context().cacheId() +
@@ -590,7 +590,7 @@ protected void awaitPartitionMapExchange(boolean waitEvicts,
                                     }
 
                                     if (entry.getValue() != GridDhtPartitionState.OWNING) {
-                                        LT.warn(log(), null,
+                                        LT.warn(log(),
                                             "Waiting for correct partition state, should be OWNING [state=" +
                                                 entry.getValue() + "]");
 
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridLogThrottleTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridLogThrottleTest.java
index d9540a803bef6..9eac0ccdcb30a 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridLogThrottleTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridLogThrottleTest.java
@@ -53,26 +53,20 @@ public void testThrottle() throws Exception {
         // LOGGED.
         LT.error(log, new RuntimeException("Test exception 2."), "Test");
 
-        // OMITTED.
-        LT.warn(log, new RuntimeException("Test exception 1."), "Test");
-
-        // OMITTED.
-        LT.warn(log, new RuntimeException("Test exception 2."), "Test1");
-
-        // OMITTED.
-        LT.warn(log, new RuntimeException("Test exception 2."), "Test3");
-
         // LOGGED.
         LT.error(log, null, "Test - without throwable.");
 
         // OMITTED.
         LT.error(log, null, "Test - without throwable.");
 
+        // OMITTED.
+        LT.warn(log, "Test - without throwable.");
+
         // LOGGED.
-        LT.warn(log, null, "Test - without throwable1.");
+        LT.warn(log, "Test - without throwable1.");
 
         // OMITTED.
-        LT.warn(log, null, "Test - without throwable1.");
+        LT.warn(log, "Test - without throwable1.");
 
         Thread.sleep(LT.throttleTimeout());
 
@@ -90,14 +84,11 @@ public void testThrottle() throws Exception {
         // LOGGED.
         LT.error(log, new RuntimeException("Test exception 2."), "Test");
 
-        // OMITTED.
-        LT.warn(log, new RuntimeException("Test exception 1."), "Test");
-
-        // OMITTED.
-        LT.warn(log, new RuntimeException("Test exception 2."), "Test1");
+        // LOGGED.
+        LT.warn(log, "Test - without throwable.");
 
         // OMITTED.
-        LT.warn(log, new RuntimeException("Test exception 2."), "Test3");
+        LT.warn(log, "Test - without throwable.");
 
         Thread.sleep(LT.throttleTimeout());
 
@@ -121,4 +112,4 @@ public void testThrottle() throws Exception {
         //OMMITED.
         LT.info(log(), "Test info message.");
     }
-}
\ No newline at end of file
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
index 6d903d8b821a2..bc047e7e2fb3d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
@@ -856,7 +856,7 @@ private HadoopCommunicationClient reserveClient(HadoopProcessDescriptor desc) th
             catch (IgniteCheckedException e) {
                 if (e.hasCause(IpcOutOfSystemResourcesException.class))
                     // Has cause or is itself the IpcOutOfSystemResourcesException.
-                    LT.warn(log, null, OUT_OF_RESOURCES_TCP_MSG);
+                    LT.warn(log, OUT_OF_RESOURCES_TCP_MSG);
                 else if (log.isDebugEnabled())
                     log.debug("Failed to establish shared memory connection with local hadoop process: " +
                         desc);
@@ -1059,7 +1059,7 @@ protected HadoopCommunicationClient createTcpClient(HadoopProcessDescriptor desc
                         ", err=" + e + ']');
 
                 if (X.hasCause(e, SocketTimeoutException.class))
-                    LT.warn(log, null, "Connect timed out (consider increasing 'connTimeout' " +
+                    LT.warn(log, "Connect timed out (consider increasing 'connTimeout' " +
                         "configuration property) [addr=" + addr + ", port=" + port + ']');
 
                 if (errs == null)
@@ -1084,7 +1084,7 @@ protected HadoopCommunicationClient createTcpClient(HadoopProcessDescriptor desc
             assert errs != null;
 
             if (X.hasCause(errs, ConnectException.class))
-                LT.warn(log, null, "Failed to connect to a remote Hadoop process (is process still running?). " +
+                LT.warn(log, "Failed to connect to a remote Hadoop process (is process still running?). " +
                     "Make sure operating system firewall is disabled on local and remote host) " +
                     "[addrs=" + addr + ", port=" + port + ']');
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index c3f701a97108e..6da8758b5e728 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -996,7 +996,7 @@ private ResultSet executeSqlQueryWithTimer(String space, PreparedStatement stmt,
                 String longMsg = "Query execution is too long [time=" + time + " ms, sql='" + sql + '\'' +
                     ", plan=" + U.nl() + plan.getString(1) + U.nl() + ", parameters=" + params + "]";
 
-                LT.warn(log, null, longMsg, msg);
+                LT.warn(log, longMsg, msg);
             }
 
             return rs;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapEvictQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapEvictQueryTest.java
index c24fed4a4bc5e..bddef66091243 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapEvictQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapEvictQueryTest.java
@@ -181,7 +181,7 @@ public void testEvictAndRemove() throws Exception {
                             }
                         }
 
-                        LT.warn(log, null, e.getMessage());
+                        LT.warn(log, e.getMessage());
 
                         return;
                     }
diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java
index 48bfd7f4be7cf..bb7260d0ef989 100644
--- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java
+++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java
@@ -343,11 +343,11 @@ private void processHttp(Collection<String> files, UriDeploymentScannerContext s
                     catch (IOException e) {
                         if (!scanCtx.isCancelled()) {
                             if (X.hasCause(e, ConnectException.class)) {
-                                LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server " +
+                                LT.error(scanCtx.getLogger(), e, "Failed to connect to HTTP server " +
                                     "(connection refused): " + U.hidePassword(url));
                             }
                             else if (X.hasCause(e, UnknownHostException.class)) {
-                                LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server " +
+                                LT.error(scanCtx.getLogger(), e, "Failed to connect to HTTP server " +
                                     "(host is unknown): " + U.hidePassword(url));
                             }
                             else
@@ -404,11 +404,11 @@ private Set<String> getUrls(URL url, UriDeploymentScannerContext scanCtx) {
             catch (IOException e) {
                 if (!scanCtx.isCancelled()) {
                     if (X.hasCause(e, ConnectException.class)) {
-                        LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server (connection refused): " +
+                        LT.error(scanCtx.getLogger(), e, "Failed to connect to HTTP server (connection refused): " +
                             U.hidePassword(url.toString()));
                     }
                     else if (X.hasCause(e, UnknownHostException.class)) {
-                        LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server (host is unknown): " +
+                        LT.error(scanCtx.getLogger(), e, "Failed to connect to HTTP server (host is unknown): " +
                             U.hidePassword(url.toString()));
                     }
                     else

From 18598574bb2992aa193eed1d72ca333a1e21ad72 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 8 Dec 2016 12:36:07 +0300
Subject: [PATCH 024/446] GG-11746: Backport of IGNITE-4379:  Fixed broken
 local SqlFieldsQuery.

---
 .../processors/query/h2/IgniteH2Indexing.java | 36 +++++++++----------
 ...teCachePartitionedFieldsQuerySelfTest.java | 25 +++++++++++++
 2 files changed, 43 insertions(+), 18 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 5c2fab564e421..3ea238bf705b6 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -737,33 +737,33 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
         @Nullable final Collection<Object> params, final IndexingQueryFilter filters,
         final int timeout, final GridQueryCancel cancel)
         throws IgniteCheckedException {
-        setFilters(filters);
+        final Connection conn = connectionForThread(schema(spaceName));
 
-        try {
-            final Connection conn = connectionForThread(schema(spaceName));
+        final PreparedStatement stmt = preparedStatementWithParams(conn, qry, params, true);
 
-            final PreparedStatement stmt = preparedStatementWithParams(conn, qry, params, true);
+        List<GridQueryFieldMetadata> meta;
 
-            List<GridQueryFieldMetadata> meta;
+        try {
+            meta = meta(stmt.getMetaData());
+        }
+        catch (SQLException e) {
+            throw new IgniteCheckedException("Cannot prepare query metadata", e);
+        }
 
-            try {
-                meta = meta(stmt.getMetaData());
-            }
-            catch (SQLException e) {
-                throw new IgniteCheckedException("Cannot prepare query metadata", e);
-            }
+        return new GridQueryFieldsResultAdapter(meta, null) {
+            @Override public GridCloseableIterator<List<?>> iterator() throws IgniteCheckedException {
+                setFilters(filters);
 
-            return new GridQueryFieldsResultAdapter(meta, null) {
-                @Override public GridCloseableIterator<List<?>> iterator() throws IgniteCheckedException{
+                try {
                     ResultSet rs = executeSqlQueryWithTimer(spaceName, stmt, conn, qry, params, timeout, cancel);
 
                     return new FieldsIterator(rs);
                 }
-            };
-        }
-        finally {
-            setFilters(null);
-        }
+                finally {
+                    setFilters(null);
+                }
+            }
+        };
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java
index 653947e391e03..af5845d708196 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java
@@ -17,7 +17,13 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.near;
 
+import java.util.List;
+import javax.cache.Cache;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractFieldsQuerySelfTest;
@@ -54,4 +60,23 @@ protected NearCacheConfiguration nearConfiguration() {
 
         return cc;
     }
+
+    /** @throws Exception If failed. */
+    public void testLocalQuery() throws Exception {
+        IgniteCache<Object, Object> cache = grid(0).cache( null);
+
+        awaitPartitionMapExchange(true, true, null);
+
+        int expected = 0;
+
+        for(Cache.Entry e: cache.localEntries(CachePeekMode.PRIMARY)){
+            if(e.getValue() instanceof Integer)
+                expected++;
+        }
+
+        QueryCursor<List<?>> qry = cache
+            .query(new SqlFieldsQuery("select _key, _val from Integer").setLocal(true));
+
+        assertEquals(expected, qry.getAll().size());
+    }
 }
\ No newline at end of file

From 671a77a2d81cac401765dddf25f30fba4e4ab17f Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 8 Dec 2016 12:56:46 +0300
Subject: [PATCH 025/446] ignite-4154 Fixed node version check for compression
 feature usage

---
 .../GridCachePartitionExchangeManager.java    | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index f04a6ce2fa025..7f11dc45aacb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -826,7 +826,7 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage(Collection<Clust
 
                     break;
                 }
-                else if (node.version().compareToIgnoreTimestamp(GridDhtPartitionsAbstractMessage.PART_MAP_COMPRESS_SINCE) < 0)
+                else if (!canUsePartitionMapCompression(node))
                     compress = false;
             }
         }
@@ -964,8 +964,7 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(ClusterNode
         boolean clientOnlyExchange,
         boolean sndCounters)
     {
-        boolean compress =
-            targetNode.version().compareToIgnoreTimestamp(GridDhtPartitionsSingleMessage.PART_MAP_COMPRESS_SINCE) >= 0;
+        boolean compress = canUsePartitionMapCompression(targetNode);
 
         GridDhtPartitionsSingleMessage m = new GridDhtPartitionsSingleMessage(exchangeId,
             clientOnlyExchange,
@@ -1554,6 +1553,24 @@ private void dumpPendingObjects(@Nullable AffinityTopologyVersion exchTopVer) {
         return deque.poll(time, MILLISECONDS);
     }
 
+    /**
+     * @param node Target node.
+     * @return {@code True} if can use compression for partition map messages.
+     */
+    @SuppressWarnings("SimplifiableIfStatement")
+    private boolean canUsePartitionMapCompression(ClusterNode node) {
+        IgniteProductVersion ver = node.version();
+
+        if (ver.compareToIgnoreTimestamp(GridDhtPartitionsAbstractMessage.PART_MAP_COMPRESS_SINCE) >= 0) {
+            if (ver.minor() == 7 && ver.maintenance() < 4)
+                return false;
+
+            return true;
+        }
+
+        return false;
+    }
+
     /**
      * Exchange future thread. All exchanges happen only by one thread and next
      * exchange will not start until previous one completes.

From 391f4be4c687a7f325aeec8b727c9c85ca003454 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 7 Dec 2016 20:11:50 +0300
Subject: [PATCH 026/446] ignite-2358 toString() method for cache store
 implementations

---
 .../cache/store/cassandra/CassandraCacheStore.java    |  6 ++++++
 .../cache/store/cassandra/datasource/DataSource.java  |  9 +++++++++
 .../apache/ignite/cache/store/CacheStoreAdapter.java  |  6 ++++++
 .../ignite/cache/store/jdbc/CacheJdbcPojoStore.java   |  6 ++++++
 .../processors/cache/CacheStoreBalancingWrapper.java  |  6 ++++++
 .../processors/cache/GridCacheLoaderWriterStore.java  |  6 ++++++
 .../platform/dotnet/PlatformDotNetCacheStore.java     | 11 +++++++++++
 7 files changed, 50 insertions(+)

diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
index f7e7917a3db37..e8da3a7c8e511 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
@@ -41,6 +41,7 @@
 import org.apache.ignite.cache.store.cassandra.session.ExecutionAssistant;
 import org.apache.ignite.cache.store.cassandra.session.GenericBatchExecutionAssistant;
 import org.apache.ignite.cache.store.cassandra.session.LoadCacheCustomQueryWorker;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.logger.NullLogger;
@@ -406,4 +407,9 @@ private void closeCassandraSession(CassandraSession ses) {
         if (ses != null && (storeSes == null || storeSes.transaction() == null))
             U.closeQuiet(ses);
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(CassandraCacheStore.class, this);
+    }
 }
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
index 1ecb28f04d3f2..915eebde73f01 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
@@ -39,6 +39,8 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.store.cassandra.session.CassandraSession;
 import org.apache.ignite.cache.store.cassandra.session.CassandraSessionImpl;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
  * Data source abstraction to specify configuration of the Cassandra session to be used.
@@ -54,9 +56,11 @@ public class DataSource {
     private ConsistencyLevel writeConsistency;
 
     /** Username to use for authentication. */
+    @GridToStringExclude
     private String user;
 
     /** Password to use for authentication. */
+    @GridToStringExclude
     private String pwd;
 
     /** Port to use for Cassandra connection. */
@@ -547,4 +551,9 @@ private ConsistencyLevel parseConsistencyLevel(String level) {
     private synchronized void invalidate() {
         ses = null;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DataSource.class, this);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/CacheStoreAdapter.java b/modules/core/src/main/java/org/apache/ignite/cache/store/CacheStoreAdapter.java
index eaf522590e403..f3436f6befd80 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/CacheStoreAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/CacheStoreAdapter.java
@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import javax.cache.Cache;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteBiPredicate;
 
@@ -93,4 +94,9 @@ public abstract class CacheStoreAdapter<K, V> implements CacheStore<K, V> {
     @Override public void sessionEnd(boolean commit) {
         // No-op.
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(CacheStoreAdapter.class, this);
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
index b348020a4777b..209dd117b964a 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.binary.BinaryObjectEx;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
@@ -335,6 +336,11 @@ protected Object buildBinaryObject(String typeName, JdbcTypeField[] fields,
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(CacheJdbcPojoStore.class, this);
+    }
+
     /**
      * Description of type property.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStoreBalancingWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStoreBalancingWrapper.java
index 8992326f19373..ce2330c879160 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStoreBalancingWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStoreBalancingWrapper.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -227,6 +228,11 @@ public void loadAll(Collection<? extends K> keys, final IgniteBiInClosure<K, V>
         delegate.sessionEnd(commit);
     }
 
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(CacheStoreBalancingWrapper.class, this);
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
index c497ac0a429e0..03beaf0973314 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
@@ -25,6 +25,7 @@
 import javax.cache.integration.CacheLoader;
 import javax.cache.integration.CacheWriter;
 import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lifecycle.LifecycleAware;
 import org.jetbrains.annotations.Nullable;
@@ -142,4 +143,9 @@ CacheWriter<K, V> writer() {
     @Override public void sessionEnd(boolean commit) {
         // No-op.
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(GridCacheLoaderWriterStore.class, this);
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java
index 3563dd668cadd..7505d4820a20f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java
@@ -31,7 +31,10 @@
 import org.apache.ignite.internal.processors.platform.utils.PlatformUtils;
 import org.apache.ignite.internal.util.lang.GridTuple;
 import org.apache.ignite.internal.util.lang.IgniteInClosureX;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lifecycle.LifecycleAware;
@@ -98,12 +101,15 @@ public class PlatformDotNetCacheStore<K, V> implements CacheStore<K, V>, Platfor
     private Map<String, ?> props;
 
     /** Native factory. */
+    @GridToStringInclude
     private final Object nativeFactory;
 
     /** Interop processor. */
+    @GridToStringExclude
     protected PlatformContext platformCtx;
 
     /** Pointer to native store. */
+    @GridToStringExclude
     protected long ptr;
 
     /**
@@ -441,4 +447,9 @@ protected int doInvoke(IgniteInClosure<BinaryRawWriterEx> task, IgniteInClosure<
             return res;
         }
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(PlatformDotNetCacheStore.class, this);
+    }
 }

From bc977d3211906ef94e1f7d3f0f988efbed65034f Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 9 Dec 2016 16:11:31 +0700
Subject: [PATCH 027/446] IGNITE-4350 Reworked JdbcTypesDefaultTransformed
 logic. Improved improved error messages in CacheJdbcPojoStore.

---
 .../cache/store/jdbc/CacheJdbcPojoStore.java  |  13 +-
 .../jdbc/JdbcTypesDefaultTransformer.java     | 112 +++----
 .../jdbc/JdbcTypesDefaultTransformerTest.java | 283 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite.java      |   2 +
 4 files changed, 349 insertions(+), 61 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformerTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
index 209dd117b964a..b9f6e8ac6ffe8 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
@@ -150,15 +150,15 @@ private Object extractBinaryParameter(String fieldName, Object obj) throws Cache
      */
     private Object buildBuiltinObject(String typeName, JdbcTypeField[] fields, Map<String, Integer> loadColIdxs,
         ResultSet rs) throws CacheLoaderException {
-        try {
-            JdbcTypeField field = fields[0];
+        JdbcTypeField field = fields[0];
 
+        try {
             Integer colIdx = columnIndex(loadColIdxs, field.getDatabaseFieldName());
 
             return transformer.getColumnValue(rs, colIdx, field.getJavaFieldType());
         }
         catch (SQLException e) {
-            throw new CacheLoaderException("Failed to read object of class: " + typeName, e);
+            throw new CacheLoaderException("Failed to read object: [cls=" + typeName + ", prop=" + field + "]", e);
         }
     }
 
@@ -211,12 +211,13 @@ private Object buildPojoObject(@Nullable String cacheName, String typeName,
                     }
                     catch (Exception e) {
                         throw new CacheLoaderException("Failed to set property in POJO class [type=" + typeName +
-                            ", prop=" + fldJavaName + ", col=" + colIdx + ", dbName=" + dbName + "]", e);
+                            ", colIdx=" + colIdx + ", prop=" + fld +
+                            ", dbValCls=" + colVal.getClass().getName() + ", dbVal=" + colVal + "]", e);
                     }
                 }
                 catch (SQLException e) {
-                    throw new CacheLoaderException("Failed to read object property [type= " + typeName +
-                        ", prop=" + fldJavaName + ", col=" + colIdx + ", dbName=" + dbName + "]", e);
+                    throw new CacheLoaderException("Failed to read object property [type=" + typeName +
+                        ", colIdx=" + colIdx + ", prop=" + fld + "]", e);
                 }
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
index e91c2d3a5fce8..c32eaa227dfd0 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
@@ -17,9 +17,12 @@
 
 package org.apache.ignite.cache.store.jdbc;
 
+import java.math.BigDecimal;
 import java.nio.ByteBuffer;
+import java.sql.Date;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.sql.Time;
 import java.sql.Timestamp;
 import java.util.UUID;
 
@@ -35,56 +38,71 @@ public class JdbcTypesDefaultTransformer implements JdbcTypesTransformer {
 
     /** {@inheritDoc} */
     @Override public Object getColumnValue(ResultSet rs, int colIdx, Class<?> type) throws SQLException {
-        Object val = rs.getObject(colIdx);
+        if (type == String.class)
+            return rs.getString(colIdx);
 
-        if (val == null)
-            return null;
+        if (type == int.class || type == Integer.class) {
+            int res = rs.getInt(colIdx);
 
-        if (type == int.class)
-            return rs.getInt(colIdx);
+            return rs.wasNull() && type == Integer.class ? null : res;
+        }
+
+        if (type == long.class || type == Long.class) {
+            long res = rs.getLong(colIdx);
+
+            return rs.wasNull() && type == Long.class ? null : res;
+        }
+
+        if (type == double.class || type == Double.class) {
+            double res = rs.getDouble(colIdx);
+
+            return rs.wasNull() && type == Double.class ? null : res;
+        }
+
+        if (type == Date.class || type == java.util.Date.class)
+            return rs.getDate(colIdx);
+
+        if (type == Timestamp.class)
+            return rs.getTimestamp(colIdx);
+
+        if (type == Time.class)
+            return rs.getTime(colIdx);
 
-        if (type == long.class)
-            return rs.getLong(colIdx);
+        if (type == boolean.class || type == Boolean.class) {
+            boolean res = rs.getBoolean(colIdx);
 
-        if (type == double.class)
-            return rs.getDouble(colIdx);
+            return rs.wasNull() && type == Boolean.class ? null : res;
+        }
 
-        if (type == boolean.class || type == Boolean.class)
-            return rs.getBoolean(colIdx);
+        if (type == byte.class || type == Byte.class) {
+            byte res = rs.getByte(colIdx);
 
-        if (type == byte.class)
-            return rs.getByte(colIdx);
+            return rs.wasNull() && type == Byte.class ? null : res;
+        }
 
-        if (type == short.class)
-            return rs.getShort(colIdx);
+        if (type == short.class || type == Short.class) {
+            short res = rs.getShort(colIdx);
 
-        if (type == float.class)
-            return rs.getFloat(colIdx);
+            return rs.wasNull() && type == Short.class ? null : res;
+        }
 
-        if (type == Integer.class || type == Long.class || type == Double.class ||
-            type == Byte.class || type == Short.class ||  type == Float.class) {
-            Number num = (Number)val;
+        if (type == float.class || type == Float.class) {
+            float res = rs.getFloat(colIdx);
 
-            if (type == Integer.class)
-                return num.intValue();
-            else if (type == Long.class)
-                return num.longValue();
-            else if (type == Double.class)
-                return num.doubleValue();
-            else if (type == Byte.class)
-                return num.byteValue();
-            else if (type == Short.class)
-                return num.shortValue();
-            else if (type == Float.class)
-                return num.floatValue();
+            return rs.wasNull() && type == Float.class ? null : res;
         }
 
+        if (type == BigDecimal.class)
+            return rs.getBigDecimal(colIdx);
+
         if (type == UUID.class) {
-            if (val instanceof UUID)
-                return val;
+            Object res = rs.getObject(colIdx);
+
+            if (res instanceof UUID)
+                return res;
 
-            if (val instanceof byte[]) {
-                ByteBuffer bb = ByteBuffer.wrap((byte[])val);
+            if (res instanceof byte[]) {
+                ByteBuffer bb = ByteBuffer.wrap((byte[])res);
 
                 long most = bb.getLong();
                 long least = bb.getLong();
@@ -92,26 +110,10 @@ else if (type == Float.class)
                 return new UUID(most, least);
             }
 
-            if (val instanceof String)
-                return UUID.fromString((String)val);
-        }
-
-        // Workaround for known issue with Oracle JDBC driver https://community.oracle.com/thread/2355464?tstart=0
-        if (type == java.sql.Date.class && val instanceof java.util.Date)
-            return new java.sql.Date(((java.util.Date)val).getTime());
-
-        // Workaround for known issue with Oracle JDBC driver and timestamp.
-        // http://stackoverflow.com/questions/13269564/java-lang-classcastexception-oracle-sql-timestamp-cannot-be-cast-to-java-sql-ti
-        if (type == Timestamp.class && !(val instanceof Timestamp) &&
-            val.getClass().getName().startsWith("oracle.sql.TIMESTAMP")) {
-            try {
-                return val.getClass().getMethod("timestampValue").invoke(val);
-            }
-            catch (Exception e) {
-                throw new SQLException("Failed to read data of oracle.sql.TIMESTAMP type.", e);
-            }
+            if (res instanceof String)
+                return UUID.fromString((String)res);
         }
 
-        return val;
+        return rs.getObject(colIdx);
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformerTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformerTest.java
new file mode 100644
index 0000000000000..5e490f7368053
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformerTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.jdbc;
+
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.UUID;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for {@link JdbcTypesDefaultTransformer}.
+ */
+public class JdbcTypesDefaultTransformerTest extends GridCommonAbstractTest {
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTransformer() throws Exception {
+        // Connection to H2.
+        String jdbcUrl = "jdbc:h2:mem:JdbcTypesDefaultTransformerTest";
+        String usr = "sa";
+        String pwd = "";
+
+        // Connection to Oracle.
+        // -Duser.region=us -Duser.language=en
+//        Class.forName("oracle.jdbc.OracleDriver");
+//        String jdbcUrl = "jdbc:oracle:thin:@localhost:1521:XE";
+//        String usr = "test";
+//        String pwd = "test";
+
+        // Connection to MS SQL.
+//        Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
+//        String jdbcUrl = "jdbc:sqlserver://localhost;databaseName=master";
+//        String usr = "test";
+//        String pwd = "test";
+
+        // Connection to DB2.
+//        Class.forName("com.ibm.db2.jcc.DB2Driver");
+//        String jdbcUrl = "jdbc:db2://localhost:50000/sample";
+//        String usr = "test";
+//        String pwd = "test";
+
+        // Connection to Postgre SQL.
+//        Class.forName("org.postgresql.Driver");
+//        String jdbcUrl = "jdbc:postgresql://localhost:5433/postgres";
+//        String usr = "test";
+//        String pwd = "test";
+
+        // Connection to My SQL.
+//        Class.forName("com.mysql.jdbc.Driver");
+//        String jdbcUrl = "jdbc:mysql://localhost:3306/test";
+//        String usr = "test";
+//        String pwd = "test";
+
+        try (Connection conn = DriverManager.getConnection(jdbcUrl, usr, pwd)) {
+            Statement stmt = conn.createStatement();
+
+            try {
+                stmt.executeUpdate("DROP TABLE TEST_TRANSFORMER");
+            }
+            catch (SQLException ignored) {
+                // No-op.
+            }
+
+            // Create table in H2.
+            stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+                "c1 BOOLEAN, c2 INTEGER, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 DECIMAL(20, 2), c7 DOUBLE PRECISION, c8 REAL, " +
+                "c9 TIME, c10 DATE, c11 TIMESTAMP, c12 VARCHAR(100), c13 UUID)");
+
+            // Create table in ORACLE.
+//                stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+//                    "c1 NUMBER(1), c2 INTEGER, c3 NUMBER(3), c4 NUMBER(4), c5 NUMBER(20), c6 NUMBER(20, 2), c7 NUMBER(20, 2), c8 NUMBER(10, 2), " +
+//                    "c9 TIMESTAMP, c10 DATE, c11 TIMESTAMP, c12 VARCHAR(100), c13 VARCHAR(36))");
+
+            // Create table in MS SQL.
+//            stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+//                "c1 BIT, c2 INTEGER, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 DECIMAL(20, 2), c7 DOUBLE PRECISION, c8 REAL, " +
+//                "c9 TIME, c10 DATE, c11 DATETIME, c12 VARCHAR(100), c13 VARCHAR(36))");
+
+            // Create table in DB2.
+//            stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+//                "c1 SMALLINT , c2 INTEGER, c3 SMALLINT , c4 SMALLINT, c5 BIGINT, c6 DECIMAL(20, 2), c7 DOUBLE PRECISION, c8 REAL, " +
+//                "c9 TIME, c10 DATE, c11 TIMESTAMP, c12 VARCHAR(100), c13 VARCHAR(36))");
+
+            // Create table in Postgre SQL.
+//            stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+//                "c1 BOOLEAN, c2 INTEGER, c3 SMALLINT, c4 SMALLINT, c5 BIGINT, c6 DECIMAL(20, 2), c7 DOUBLE PRECISION, c8 REAL, " +
+//                "c9 TIME, c10 DATE, c11 TIMESTAMP, c12 VARCHAR(100), c13 UUID)");
+
+            // Create table in MySQL.
+//            stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+//                "c1 BOOLEAN, c2 INTEGER, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 DECIMAL(20, 2), c7 DOUBLE PRECISION, c8 REAL, " +
+//                "c9 TIME, c10 DATE, c11 TIMESTAMP(3), c12 VARCHAR(100), c13 VARCHAR(36))");
+
+            // Add data to H2, Postgre SQL and MySQL.
+            stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+                "VALUES (1, true, 1, 2, 3, 4, 5.35, 6.15, 7.32, '00:01:08', '2016-01-01', '2016-01-01 00:01:08.296', " +
+                "'100', '736bc956-090c-40d2-94da-916f2161f8a2')");
+            stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+                "VALUES (2, false, 10, 20, 30, 40, 50, 60, 70, current_time, current_date, current_timestamp, " +
+                "'100.55', '736bc956-090c-40d2-94da-916f2161cdea')");
+
+            // Add data to Oracle.
+//            stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+//                "VALUES (1, 1, 1, 2, 3, 4, 5.35, 6.15, 7.32, " +
+//                "TO_TIMESTAMP('2016-01-01 00:01:08', 'YYYY-MM-DD HH24:MI:SS'), " +
+//                "TO_DATE('2016-01-01', 'YYYY-MM-DD')," +
+//                "TO_TIMESTAMP('2016-01-01 00:01:08.296', 'YYYY-MM-DD HH24:MI:SS.FF3'), " +
+//                "'100', '736bc956-090c-40d2-94da-916f2161f8a2')");
+//            stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+//                "VALUES (2, 0, 10, 20, 30, 40, 50, 60, 70," +
+//                "TO_TIMESTAMP('2016-01-01 00:01:08', 'YYYY-MM-DD HH24:MI:SS'), " +
+//                "TO_DATE('2016-01-01', 'YYYY-MM-DD')," +
+//                "TO_TIMESTAMP('2016-01-01 00:01:08.296', 'YYYY-MM-DD HH24:MI:SS.FF3'), " +
+//                "'100.55', '736bc956-090c-40d2-94da-916f2161cdea')");
+
+            // Add data to MS SQL or IBM DB2.
+//            stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+//                "VALUES (1, 1, 1, 2, 3, 4, 5.35, 6.15, 7.32, '00:01:08', '2016-01-01', '2016-01-01 00:01:08.296', " +
+//                "'100', '736bc956-090c-40d2-94da-916f2161f8a2')");
+//            stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+//                "VALUES (2, 0, 10, 20, 30, 40, 50, 60, 70, '00:01:08', '2016-01-01', '2016-01-01 00:01:08.296', " +
+//                "'100.55', '736bc956-090c-40d2-94da-916f2161cdea')");
+
+            stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+                "VALUES (3, null, null, null, null, null, null, null, null, null, null, null, null, null)");
+
+            ResultSet rs = stmt.executeQuery("select c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13 from TEST_TRANSFORMER order by id");
+
+            assertTrue(rs.next());
+
+            JdbcTypesDefaultTransformer transformer = JdbcTypesDefaultTransformer.INSTANCE;
+
+            // c1: Test BOOLEAN column.
+            assertTrue((Boolean)transformer.getColumnValue(rs, 1, boolean.class));
+            assertTrue((Boolean)transformer.getColumnValue(rs, 1, Boolean.class));
+            assertEquals(1, transformer.getColumnValue(rs, 1, int.class));
+            assertEquals(1.0, transformer.getColumnValue(rs, 1, double.class));
+
+            String s = (String)transformer.getColumnValue(rs, 1, String.class);
+            assertTrue("true".equalsIgnoreCase(s) || "1".equals(s) || "t".equalsIgnoreCase(s));
+
+            // c2: Test INTEGER column.
+            assertEquals(1, transformer.getColumnValue(rs, 2, int.class));
+            assertEquals(1, transformer.getColumnValue(rs, 2, Integer.class));
+            assertEquals(1L, transformer.getColumnValue(rs, 2, Long.class));
+            assertEquals(1.0, transformer.getColumnValue(rs, 2, double.class));
+            assertEquals("1", transformer.getColumnValue(rs, 2, String.class));
+
+            // c3: Test TINYINT column.
+            byte b = 2;
+            assertEquals(b, transformer.getColumnValue(rs, 3, byte.class));
+            assertEquals(b, transformer.getColumnValue(rs, 3, Byte.class));
+            assertEquals(2.0, transformer.getColumnValue(rs, 3, double.class));
+            assertEquals("2", transformer.getColumnValue(rs, 3, String.class));
+
+            // c4: Test SMALLINT column.
+            short shrt = 3;
+            assertEquals(shrt, transformer.getColumnValue(rs, 4, short.class));
+            assertEquals(shrt, transformer.getColumnValue(rs, 4, Short.class));
+            assertEquals(3.0, transformer.getColumnValue(rs, 4, double.class));
+            assertEquals("3", transformer.getColumnValue(rs, 4, String.class));
+
+            // c5: Test BIGINT column.
+            assertEquals(4L, transformer.getColumnValue(rs, 5, long.class));
+            assertEquals(4L, transformer.getColumnValue(rs, 5, Long.class));
+            assertEquals(4, transformer.getColumnValue(rs, 5, int.class));
+            assertEquals(4, transformer.getColumnValue(rs, 5, Integer.class));
+            assertEquals(4.0, transformer.getColumnValue(rs, 5, double.class));
+            assertEquals("4", transformer.getColumnValue(rs, 5, String.class));
+            assertEquals(new BigDecimal("4"), transformer.getColumnValue(rs, 5, BigDecimal.class));
+
+            // c6: Test DECIMAL column.
+            assertEquals(new BigDecimal("5.35"), transformer.getColumnValue(rs, 6, BigDecimal.class));
+            assertEquals(5L, transformer.getColumnValue(rs, 6, long.class));
+            assertEquals("5.35", transformer.getColumnValue(rs, 6, String.class));
+
+            // c7: Test DOUBLE column.
+            assertEquals(6.15, transformer.getColumnValue(rs, 7, double.class));
+            assertEquals(6.15, transformer.getColumnValue(rs, 7, Double.class));
+            assertEquals(6, transformer.getColumnValue(rs, 7, int.class));
+            assertEquals(6, transformer.getColumnValue(rs, 7, Integer.class));
+            assertTrue(transformer.getColumnValue(rs, 7, String.class).toString().startsWith("6.15"));
+
+            // c8: Test REAL column.
+            assertTrue((7.32f - (Float)transformer.getColumnValue(rs, 8, float.class)) < 0.01);
+            assertTrue((7.32f - (Float)transformer.getColumnValue(rs, 8, Float.class)) < 0.01);
+            assertTrue((7.32 - (Double)transformer.getColumnValue(rs, 8, double.class)) < 0.01);
+            assertTrue((7.32 - (Double)transformer.getColumnValue(rs, 8, Double.class)) < 0.01);
+            assertEquals(7, transformer.getColumnValue(rs, 8, int.class));
+            assertEquals(7, transformer.getColumnValue(rs, 8, Integer.class));
+            assertTrue(transformer.getColumnValue(rs, 8, String.class).toString().startsWith("7.32"));
+
+            // c9: Test TIME column.
+            assertTrue(transformer.getColumnValue(rs, 9, Time.class) instanceof Time);
+            assertTrue(transformer.getColumnValue(rs, 9, String.class).toString().contains("00:01:08"));
+
+            // c10: Test DATE column.
+            assertTrue(transformer.getColumnValue(rs, 10, Date.class) instanceof Date);
+            assertTrue(transformer.getColumnValue(rs, 10, String.class).toString().startsWith("2016-01-01"));
+
+            // c11: Test TIMESTAMP column.
+            transformer.getColumnValue(rs, 11, Timestamp.class);
+            assertTrue(transformer.getColumnValue(rs, 11, String.class).toString().startsWith("2016-01-01 00:01:08.29"));
+
+            // c12: Test VARCHAR column.
+            assertEquals("100", transformer.getColumnValue(rs, 12, String.class));
+            assertEquals(100, transformer.getColumnValue(rs, 12, int.class));
+
+            // c13: Test UUID column.
+            transformer.getColumnValue(rs, 13, UUID.class);
+            assertEquals("736bc956-090c-40d2-94da-916f2161f8a2", transformer.getColumnValue(rs, 13, String.class));
+
+            assertTrue(rs.next());
+
+            // Test BOOLEAN column.
+            assertFalse((Boolean)transformer.getColumnValue(rs, 1, boolean.class));
+            assertFalse((Boolean)transformer.getColumnValue(rs, 1, Boolean.class));
+            assertEquals(0, transformer.getColumnValue(rs, 1, int.class));
+            assertEquals(0.0, transformer.getColumnValue(rs, 1, double.class));
+
+            s = (String)transformer.getColumnValue(rs, 1, String.class);
+            assertTrue("false".equalsIgnoreCase(s) || "0".equals(s) || "f".equalsIgnoreCase(s));
+
+            assertTrue(rs.next());
+
+            // Check how null values will be transformed.
+            assertNotNull(transformer.getColumnValue(rs, 1, boolean.class));
+            assertNull(transformer.getColumnValue(rs, 1, Boolean.class));
+
+            assertEquals(0, transformer.getColumnValue(rs, 2, int.class));
+            assertNull(transformer.getColumnValue(rs, 2, Integer.class));
+
+            assertEquals((byte)0, transformer.getColumnValue(rs, 3, byte.class));
+            assertNull(transformer.getColumnValue(rs, 3, Byte.class));
+
+            assertEquals((short)0, transformer.getColumnValue(rs, 4, short.class));
+            assertNull(transformer.getColumnValue(rs, 4, Short.class));
+
+            assertEquals(0L, transformer.getColumnValue(rs, 5, long.class));
+            assertNull(transformer.getColumnValue(rs, 5, Long.class));
+
+            assertNull(transformer.getColumnValue(rs, 6, BigDecimal.class));
+
+            assertEquals(0d, transformer.getColumnValue(rs, 7, double.class));
+            assertNull(transformer.getColumnValue(rs, 7, Double.class));
+
+            assertEquals(0f, transformer.getColumnValue(rs, 8, float.class));
+            assertNull(transformer.getColumnValue(rs, 8, Float.class));
+
+            assertNull(transformer.getColumnValue(rs, 9, Time.class));
+
+            assertNull(transformer.getColumnValue(rs, 10, Date.class));
+
+            assertNull(transformer.getColumnValue(rs, 11, Timestamp.class));
+
+            assertNull(transformer.getColumnValue(rs, 12, String.class));
+
+            assertNull(transformer.getColumnValue(rs, 13, UUID.class));
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index fd13e98d838f9..deec72af95cd8 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreTest;
 import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreMultithreadedSelfTest;
 import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreSelfTest;
+import org.apache.ignite.cache.store.jdbc.JdbcTypesDefaultTransformerTest;
 import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest;
 import org.apache.ignite.internal.processors.cache.CacheEntryProcessorCopySelfTest;
@@ -246,6 +247,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridCacheConfigurationConsistencySelfTest.class);
         suite.addTestSuite(GridCacheJdbcBlobStoreSelfTest.class);
         suite.addTestSuite(GridCacheJdbcBlobStoreMultithreadedSelfTest.class);
+        suite.addTestSuite(JdbcTypesDefaultTransformerTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreOptimizedMarshallerSelfTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreOptimizedMarshallerWithSqlEscapeSelfTest.class);

From b83ec8e57c7c48f2baa4780cf3b2e46df075f3df Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 9 Dec 2016 14:32:42 +0300
Subject: [PATCH 028/446] IGNITE-500 CacheLoadingConcurrentGridStartSelfTest
 fails: prevent 'localUpdate' execution while top read lock is held.

---
 .../datastreamer/DataStreamProcessor.java     | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index 32fda87375cb4..fee4dd6616b28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -328,6 +328,8 @@ private void localUpdate(final UUID nodeId,
             if (!allowOverwrite)
                 cctx.topology().readLock();
 
+            GridDhtTopologyFuture topWaitFut = null;
+
             try {
                 GridDhtTopologyFuture fut = cctx.topologyVersionFuture();
 
@@ -352,19 +354,25 @@ else if (allowOverwrite || fut.isDone()) {
 
                     waitFut = allowOverwrite ? null : cctx.mvcc().addDataStreamerFuture(topVer);
                 }
-                else {
-                    fut.listen(new IgniteInClosure<IgniteInternalFuture<AffinityTopologyVersion>>() {
-                        @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> e) {
-                            localUpdate(nodeId, req, updater, topic);
-                        }
-                    });
-                }
+                else
+                    topWaitFut = fut;
             }
             finally {
                 if (!allowOverwrite)
                     cctx.topology().readUnlock();
             }
 
+            if (topWaitFut != null) {
+                // Need call 'listen' after topology read lock is released.
+                topWaitFut.listen(new IgniteInClosure<IgniteInternalFuture<AffinityTopologyVersion>>() {
+                    @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> e) {
+                        localUpdate(nodeId, req, updater, topic);
+                    }
+                });
+
+                return;
+            }
+
             if (job != null) {
                 try {
                     job.call();

From 6e485637e2738a7e809eac1a802f0964dc12383d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 9 Dec 2016 15:42:40 +0300
Subject: [PATCH 029/446] IGNITE-4379: Fixed local query execution. This closes
 #1323.

---
 .../processors/query/h2/IgniteH2Indexing.java | 54 +++++++++----------
 ...gniteCacheAbstractFieldsQuerySelfTest.java | 19 +++++++
 2 files changed, 45 insertions(+), 28 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 6da8758b5e728..362ddd853443c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -774,31 +774,38 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
         throws IgniteCheckedException {
         final Connection conn = connectionForSpace(spaceName);
 
-        initLocalQueryContext(conn, enforceJoinOrder, filters);
+        setupConnection(conn, false, enforceJoinOrder);
+
+        final PreparedStatement stmt = preparedStatementWithParams(conn, qry, params, true);
+
+        List<GridQueryFieldMetadata> meta;
 
         try {
-            final PreparedStatement stmt = preparedStatementWithParams(conn, qry, params, true);
+            meta = meta(stmt.getMetaData());
+        }
+        catch (SQLException e) {
+            throw new IgniteCheckedException("Cannot prepare query metadata", e);
+        }
 
-            List<GridQueryFieldMetadata> meta;
+        final GridH2QueryContext ctx = new GridH2QueryContext(nodeId, nodeId, 0, LOCAL)
+            .filter(filters).distributedJoins(false);
 
-            try {
-                meta = meta(stmt.getMetaData());
-            }
-            catch (SQLException e) {
-                throw new IgniteCheckedException("Cannot prepare query metadata", e);
-            }
+        return new GridQueryFieldsResultAdapter(meta, null) {
+            @Override public GridCloseableIterator<List<?>> iterator() throws IgniteCheckedException {
+                assert GridH2QueryContext.get() == null;
+
+                GridH2QueryContext.set(ctx);
 
-            return new GridQueryFieldsResultAdapter(meta, null) {
-                @Override public GridCloseableIterator<List<?>> iterator() throws IgniteCheckedException{
+                try {
                     ResultSet rs = executeSqlQueryWithTimer(spaceName, stmt, conn, qry, params, timeout, cancel);
 
                     return new FieldsIterator(rs);
                 }
-            };
-        }
-        finally {
-            GridH2QueryContext.clearThreadLocal();
-        }
+                finally {
+                    GridH2QueryContext.clearThreadLocal();
+                }
+            }
+        };
     }
 
     /**
@@ -1024,17 +1031,6 @@ public void bindParameters(PreparedStatement stmt, @Nullable Collection<Object>
         }
     }
 
-    /**
-     * @param conn Connection.
-     * @param enforceJoinOrder Enforce join order of tables.
-     * @param filter Filter.
-     */
-    private void initLocalQueryContext(Connection conn, boolean enforceJoinOrder, IndexingQueryFilter filter) {
-        setupConnection(conn, false, enforceJoinOrder);
-
-        GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter).distributedJoins(false));
-    }
-
     /**
      * @param conn Connection to use.
      * @param distributedJoins If distributed joins are enabled.
@@ -1061,7 +1057,9 @@ public void setupConnection(Connection conn, boolean distributedJoins, boolean e
 
         Connection conn = connectionForThread(tbl.schemaName());
 
-        initLocalQueryContext(conn, false, filter);
+        setupConnection(conn, false, false);
+
+        GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter).distributedJoins(false));
 
         try {
             ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, null);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
index d5f02ebdf980d..2c355011114ed 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
@@ -28,10 +28,12 @@
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.Callable;
+import javax.cache.Cache;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.affinity.AffinityKey;
@@ -649,6 +651,23 @@ public void testQueryString() throws Exception {
         }
     }
 
+    /** @throws Exception If failed. */
+    public void testLocalQuery() throws Exception {
+        IgniteCache<Object, Object> cache = grid(0).cache( null);
+
+        int expected = 0;
+
+        for(Cache.Entry e: cache.localEntries(CachePeekMode.PRIMARY)){
+            if(e.getValue() instanceof Integer)
+                expected++;
+        }
+
+        QueryCursor<List<?>> qry = cache
+            .query(new SqlFieldsQuery("select _key, _val from Integer").setLocal(true));
+
+        assertEquals(expected, qry.getAll().size());
+    }
+
     /** @throws Exception If failed. */
     public void testQueryIntegersWithJoin() throws Exception {
         QueryCursor<List<?>> qry = grid(0).cache(null).query(new SqlFieldsQuery(

From 6fd8bf6338470275e687a686044c7d935d3714ff Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 7 Dec 2016 18:49:06 +0300
Subject: [PATCH 030/446] Fixed tests for IGNITE-4379 commit.

---
 ...gniteCacheAbstractFieldsQuerySelfTest.java | 19 --------------
 ...teCachePartitionedFieldsQuerySelfTest.java | 25 +++++++++++++++++++
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
index 2c355011114ed..d5f02ebdf980d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
@@ -28,12 +28,10 @@
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.Callable;
-import javax.cache.Cache;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.affinity.AffinityKey;
@@ -651,23 +649,6 @@ public void testQueryString() throws Exception {
         }
     }
 
-    /** @throws Exception If failed. */
-    public void testLocalQuery() throws Exception {
-        IgniteCache<Object, Object> cache = grid(0).cache( null);
-
-        int expected = 0;
-
-        for(Cache.Entry e: cache.localEntries(CachePeekMode.PRIMARY)){
-            if(e.getValue() instanceof Integer)
-                expected++;
-        }
-
-        QueryCursor<List<?>> qry = cache
-            .query(new SqlFieldsQuery("select _key, _val from Integer").setLocal(true));
-
-        assertEquals(expected, qry.getAll().size());
-    }
-
     /** @throws Exception If failed. */
     public void testQueryIntegersWithJoin() throws Exception {
         QueryCursor<List<?>> qry = grid(0).cache(null).query(new SqlFieldsQuery(
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java
index 653947e391e03..af5845d708196 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java
@@ -17,7 +17,13 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.near;
 
+import java.util.List;
+import javax.cache.Cache;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractFieldsQuerySelfTest;
@@ -54,4 +60,23 @@ protected NearCacheConfiguration nearConfiguration() {
 
         return cc;
     }
+
+    /** @throws Exception If failed. */
+    public void testLocalQuery() throws Exception {
+        IgniteCache<Object, Object> cache = grid(0).cache( null);
+
+        awaitPartitionMapExchange(true, true, null);
+
+        int expected = 0;
+
+        for(Cache.Entry e: cache.localEntries(CachePeekMode.PRIMARY)){
+            if(e.getValue() instanceof Integer)
+                expected++;
+        }
+
+        QueryCursor<List<?>> qry = cache
+            .query(new SqlFieldsQuery("select _key, _val from Integer").setLocal(true));
+
+        assertEquals(expected, qry.getAll().size());
+    }
 }
\ No newline at end of file

From c143bc1a77baa13f61d6ba00509fa1fcb33757b1 Mon Sep 17 00:00:00 2001
From: tledkov-gridgain <tledkov@gridgain.com>
Date: Fri, 9 Dec 2016 16:05:03 +0300
Subject: [PATCH 031/446] IGNITE-4063:  Preserved order of fields in the
 metadata according with schema. This closes #1270.

---
 .../binary/BinaryClassDescriptor.java         | 12 ++--
 .../ignite/internal/binary/BinaryUtils.java   | 10 ++-
 .../builder/BinaryObjectBuilderImpl.java      | 11 +++-
 .../platform/PlatformContextImpl.java         |  2 +-
 .../platform/utils/PlatformUtils.java         | 28 ++++++++
 .../binary/BinaryMarshallerSelfTest.java      | 66 +++++++++++++++++++
 6 files changed, 119 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
index b121337287d6d..5ec519abca641 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
@@ -28,7 +28,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -275,15 +274,20 @@ else if (useOptMarshaller)
             case OBJECT:
                 // Must not use constructor to honor transient fields semantics.
                 ctor = null;
-                stableFieldsMeta = metaDataEnabled ? new HashMap<String, Integer>() : null;
 
                 Map<Object, BinaryFieldAccessor> fields0;
 
-                if (BinaryUtils.FIELDS_SORTED_ORDER)
+                if (BinaryUtils.FIELDS_SORTED_ORDER) {
                     fields0 = new TreeMap<>();
-                else
+
+                    stableFieldsMeta = metaDataEnabled ? new TreeMap<String, Integer>() : null;
+                }
+                else {
                     fields0 = new LinkedHashMap<>();
 
+                    stableFieldsMeta = metaDataEnabled ? new LinkedHashMap<String, Integer>() : null;
+                }
+
                 Set<String> duplicates = duplicateFields(cls);
 
                 Collection<String> names = new HashSet<>();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index b3040826fead9..bbf50213fdd6d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -947,11 +947,17 @@ public static BinaryMetadata mergeMetadata(@Nullable BinaryMetadata oldMeta, Bin
             }
 
             // Check and merge fields.
-            boolean changed = false;
+            Map<String, Integer> mergedFields;
+
+            if (FIELDS_SORTED_ORDER)
+                mergedFields = new TreeMap<>(oldMeta.fieldsMap());
+            else
+                mergedFields = new LinkedHashMap<>(oldMeta.fieldsMap());
 
-            Map<String, Integer> mergedFields = new HashMap<>(oldMeta.fieldsMap());
             Map<String, Integer> newFields = newMeta.fieldsMap();
 
+            boolean changed = false;
+
             for (Map.Entry<String, Integer> newField : newFields.entrySet()) {
                 Integer oldFieldType = mergedFields.put(newField.getKey(), newField.getValue());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
index 6ea9e7ec3a728..68a0ff3e35986 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
@@ -400,8 +400,12 @@ else if (newVal.getClass().isArray() && newVal.getClass().getComponentType() ==
 
         if (oldFldTypeName == null) {
             // It's a new field, we have to add it to metadata.
-            if (fieldsMeta == null)
-                fieldsMeta = new HashMap<>();
+            if (fieldsMeta == null) {
+                if (BinaryUtils.FIELDS_SORTED_ORDER)
+                    fieldsMeta = new TreeMap<>();
+                else
+                    fieldsMeta = new LinkedHashMap<>();
+            }
 
             fieldsMeta.put(name, newFldTypeId);
         }
@@ -532,11 +536,12 @@ private void ensureReadCacheInit() {
     @Override public BinaryObjectBuilder setField(String name, Object val0) {
         Object val = val0 == null ? new BinaryValueWithType(BinaryUtils.typeByClass(Object.class), null) : val0;
 
-        if (assignedVals == null)
+        if (assignedVals == null) {
             if (BinaryUtils.FIELDS_SORTED_ORDER)
                 assignedVals = new TreeMap<>();
             else
                 assignedVals = new LinkedHashMap<>();
+        }
 
         Object oldVal = assignedVals.put(name, val);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java
index e7fdb0a2dcc15..6cec7a1a204ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java
@@ -361,7 +361,7 @@ public PlatformContextImpl(GridKernalContext ctx, PlatformCallbackGateway gate,
                     String typeName = reader.readString();
                     String affKey = reader.readString();
 
-                    Map<String, Integer> fields = PlatformUtils.readMap(reader,
+                    Map<String, Integer> fields = PlatformUtils.readLinkedMap(reader,
                         new PlatformReaderBiClosure<String, Integer>() {
                             @Override public IgniteBiTuple<String, Integer> read(BinaryRawReaderEx reader) {
                                 return F.t(reader.readString(), reader.readInt());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
index 0d30ad95582a9..959ff68a4e19f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
@@ -387,6 +387,34 @@ public static <K, V> Map<K, V> readMap(BinaryRawReaderEx reader,
         return map;
     }
 
+    /**
+     * Read linked map.
+     *
+     * @param reader Reader.
+     * @param readClo Reader closure.
+     * @return Map.
+     */
+    public static <K, V> Map<K, V> readLinkedMap(BinaryRawReaderEx reader,
+        @Nullable PlatformReaderBiClosure<K, V> readClo) {
+        int cnt = reader.readInt();
+
+        Map<K, V> map = U.newLinkedHashMap(cnt);
+
+        if (readClo == null) {
+            for (int i = 0; i < cnt; i++)
+                map.put((K)reader.readObjectDetached(), (V)reader.readObjectDetached());
+        }
+        else {
+            for (int i = 0; i < cnt; i++) {
+                IgniteBiTuple<K, V> entry = readClo.read(reader);
+
+                map.put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        return map;
+    }
+
     /**
      * Read nullable map.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index 39a4d32383f7a..6d07c9ba42770 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -53,6 +53,7 @@
 import java.util.concurrent.ConcurrentSkipListSet;
 import junit.framework.Assert;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryBasicIdMapper;
 import org.apache.ignite.binary.BinaryBasicNameMapper;
 import org.apache.ignite.binary.BinaryCollectionFactory;
@@ -3108,6 +3109,71 @@ public void testBinaryEqualsComplexObject() throws Exception {
         assertNotEquals(binObj02, binObj11);
     }
 
+
+    /**
+     * The test must be refactored after {@link IgniteSystemProperties#IGNITE_BINARY_SORT_OBJECT_FIELDS}
+     * is removed.
+     *
+     * @throws Exception If failed.
+     */
+    public void testFieldOrder() throws Exception {
+        if (BinaryUtils.FIELDS_SORTED_ORDER)
+            return;
+
+        BinaryMarshaller m = binaryMarshaller();
+
+        BinaryObjectImpl binObj = marshal(simpleObject(), m);
+
+        Collection<String> fieldsBin =  binObj.type().fieldNames();
+
+        Field[] fields = SimpleObject.class.getDeclaredFields();
+
+        assertEquals(fields.length, fieldsBin.size());
+
+        int i = 0;
+
+        for (String fieldName : fieldsBin) {
+            assertEquals(fields[i].getName(), fieldName);
+
+            ++i;
+        }
+    }
+
+    /**
+     * The test must be refactored after {@link IgniteSystemProperties#IGNITE_BINARY_SORT_OBJECT_FIELDS}
+     * is removed.
+     *
+     * @throws Exception If failed.
+     */
+    public void testFieldOrderByBuilder() throws Exception {
+        if (BinaryUtils.FIELDS_SORTED_ORDER)
+            return;
+
+        BinaryMarshaller m = binaryMarshaller();
+
+        BinaryObjectBuilder builder = new BinaryObjectBuilderImpl(binaryContext(m), "MyFakeClass");
+
+        String[] fieldNames = {"field9", "field8", "field0", "field1", "field2"};
+
+        for (String fieldName : fieldNames)
+            builder.setField(fieldName, 0);
+
+        BinaryObject binObj = builder.build();
+
+
+        Collection<String> fieldsBin =  binObj.type().fieldNames();
+
+        assertEquals(fieldNames.length, fieldsBin.size());
+
+        int i = 0;
+
+        for (String fieldName : fieldsBin) {
+            assertEquals(fieldNames[i], fieldName);
+
+            ++i;
+        }
+    }
+
     /**
      * @param obj Instance of the BinaryObjectImpl to offheap marshalling.
      * @param marsh Binary marshaller.

From f087b661c466c9fd27d049efa8e55a2d7bdf9834 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 9 Dec 2016 17:45:33 +0300
Subject: [PATCH 032/446] IGNITE-4264: Fixed cluster wide cache metrics. This
 closes #1336.

---
 .../processors/cache/GridCacheAdapter.java    |   2 +-
 ...apCacheMetricsForClusterGroupSelfTest.java | 141 ++++++++++++++++++
 ...alCacheOffHeapAndSwapMetricsSelfTest.java} |   2 +-
 .../IgniteCacheMetricsSelfTestSuite.java      |   6 +-
 4 files changed, 147 insertions(+), 4 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java
 rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/{CacheOffHeapAndSwapMetricsSelfTest.java => LocalCacheOffHeapAndSwapMetricsSelfTest.java} (99%)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 2e24e67cae195..b30ec709870d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3234,7 +3234,7 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
 
     /** {@inheritDoc} */
     @Override public CacheMetrics clusterMetrics() {
-        return clusterMetrics(ctx.grid().cluster().forCacheNodes(ctx.name()));
+        return clusterMetrics(ctx.grid().cluster().forDataNodes(ctx.name()));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java
new file mode 100644
index 0000000000000..53a20fcd83861
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import java.util.concurrent.CountDownLatch;
+
+import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
+
+/**
+ * Test for cluster wide offheap cache metrics.
+ */
+public class OffheapCacheMetricsForClusterGroupSelfTest extends GridCommonAbstractTest {
+    /** Grid count. */
+    private static final int GRID_CNT = 3;
+
+    /** Client count */
+    private static final int CLIENT_CNT = 3;
+
+    /** Grid client mode */
+    private boolean clientMode;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(clientMode);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        // start grids
+        clientMode = false;
+        for (int i = 0; i < GRID_CNT; i++)
+            startGrid("server-" + i);
+
+        // start clients
+        clientMode = true;
+        for (int i = 0; i < CLIENT_CNT; i++)
+            startGrid("client-" + i);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    public void testGetOffHeapPrimaryEntriesCount() throws Exception {
+        String cacheName = "testGetOffHeapPrimaryEntriesCount";
+        IgniteCache<Integer, Integer> cache = grid("client-0").createCache(cacheConfiguration(cacheName));
+
+        for (int i = 0; i < 1000; i++)
+            cache.put(i, i);
+
+        awaitMetricsUpdate();
+
+        assertGetOffHeapPrimaryEntriesCount(cacheName, 1000);
+
+        for (int j = 0; j < 1000; j++)
+            cache.get(j);
+
+        awaitMetricsUpdate();
+
+        assertGetOffHeapPrimaryEntriesCount(cacheName, 1000);
+
+        cache = grid("client-1").cache(cacheName);
+
+        for (int j = 0; j < 1000; j++)
+            cache.get(j);
+
+        awaitMetricsUpdate();
+
+        assertGetOffHeapPrimaryEntriesCount(cacheName, 1000);
+    }
+
+    /**
+     * Wait for {@link EventType#EVT_NODE_METRICS_UPDATED} event will be receieved.
+     */
+    private void awaitMetricsUpdate() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch((GRID_CNT + 1) * 2);
+
+        IgnitePredicate<Event> lsnr = new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event ignore) {
+                latch.countDown();
+
+                return true;
+            }
+        };
+
+        for (int i = 0; i < GRID_CNT; i++)
+            grid("server-" + i).events().localListen(lsnr, EVT_NODE_METRICS_UPDATED);
+
+        latch.await();
+    }
+
+    private void assertGetOffHeapPrimaryEntriesCount(String cacheName, int count) throws Exception {
+        for (int i = 0; i < GRID_CNT; i++) {
+            IgniteCache<Integer, Integer> cache = grid("server-" + i).cache(cacheName);
+            assertEquals(count, cache.metrics().getOffHeapPrimaryEntriesCount());
+        }
+
+        for (int i = 0; i < CLIENT_CNT; i++) {
+            IgniteCache<Integer, Integer> cache = grid("client-" + i).cache(cacheName);
+            assertEquals(count, cache.metrics().getOffHeapPrimaryEntriesCount());
+        }
+    }
+
+    private static CacheConfiguration<Integer, Integer> cacheConfiguration(String cacheName) {
+        CacheConfiguration<Integer, Integer> cfg = new CacheConfiguration<>(cacheName);
+
+        cfg.setBackups(1);
+        cfg.setStatisticsEnabled(true);
+        cfg.setMemoryMode(CacheMemoryMode.OFFHEAP_TIERED);
+        cfg.setOffHeapMaxMemory(1024 * 1024 * 1024);
+        return cfg;
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/CacheOffHeapAndSwapMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/LocalCacheOffHeapAndSwapMetricsSelfTest.java
similarity index 99%
rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/CacheOffHeapAndSwapMetricsSelfTest.java
rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/LocalCacheOffHeapAndSwapMetricsSelfTest.java
index 6837a272b0d0d..ae8807f706be2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/CacheOffHeapAndSwapMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/LocalCacheOffHeapAndSwapMetricsSelfTest.java
@@ -30,7 +30,7 @@
 /**
  *
  */
-public class CacheOffHeapAndSwapMetricsSelfTest extends GridCommonAbstractTest {
+public class LocalCacheOffHeapAndSwapMetricsSelfTest extends GridCommonAbstractTest {
     /** Grid count. */
     private static final int GRID_CNT = 1;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
index 90ecbdaff4ded..f3dd1c83d5924 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.CacheMetricsForClusterGroupSelfTest;
+import org.apache.ignite.internal.processors.cache.OffheapCacheMetricsForClusterGroupSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedMetricsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedTckMetricsSelfTestImpl;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearMetricsSelfTest;
@@ -26,7 +27,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMetricsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheAtomicReplicatedMetricsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMetricsSelfTest;
-import org.apache.ignite.internal.processors.cache.local.CacheOffHeapAndSwapMetricsSelfTest;
+import org.apache.ignite.internal.processors.cache.local.LocalCacheOffHeapAndSwapMetricsSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheAtomicLocalMetricsNoStoreSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheAtomicLocalMetricsSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheAtomicLocalTckMetricsSelfTestImpl;
@@ -48,7 +49,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheReplicatedMetricsSelfTest.class);
         suite.addTestSuite(GridCachePartitionedMetricsSelfTest.class);
         suite.addTestSuite(GridCachePartitionedHitsAndMissesSelfTest.class);
-        suite.addTestSuite(CacheOffHeapAndSwapMetricsSelfTest.class);
+        suite.addTestSuite(LocalCacheOffHeapAndSwapMetricsSelfTest.class);
 
         // Atomic cache.
         suite.addTestSuite(GridCacheAtomicLocalMetricsSelfTest.class);
@@ -60,6 +61,7 @@ public static TestSuite suite() throws Exception {
 
         // Cluster wide metrics.
         suite.addTestSuite(CacheMetricsForClusterGroupSelfTest.class);
+        suite.addTestSuite(OffheapCacheMetricsForClusterGroupSelfTest.class);
 
         return suite;
     }

From 6d744db27111c68e13b06ec99428a4c4148d97b6 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 12 Dec 2016 11:44:57 +0300
Subject: [PATCH 033/446] IGNITE-4231 - Hangs on compute result serialization
 error. Fix

---
 .../processors/job/GridJobWorker.java         |  69 ++++++-
 .../closure/GridClosureSerializationTest.java | 177 ++++++++++++++++++
 .../testsuites/IgniteBasicTestSuite.java      |   2 +
 3 files changed, 241 insertions(+), 7 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index 5f38b29025584..f5c6a2738e115 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -19,6 +19,7 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.Callable;
@@ -790,6 +791,64 @@ void finishJob(@Nullable Object res,
                     }
                     else {
                         try {
+                            byte[] resBytes = null;
+                            byte[] exBytes = null;
+                            byte[] attrBytes = null;
+
+                            boolean loc = ctx.localNodeId().equals(sndNode.id()) && !ctx.config().isMarshalLocalJobs();
+
+                            Map<Object, Object> attrs = jobCtx.getAttributes();
+
+                            // Try serialize response, and if exception - return to client.
+                            if (!loc) {
+                                try {
+                                    resBytes = U.marshal(marsh, res);
+                                }
+                                catch (IgniteCheckedException e) {
+                                    resBytes = U.marshal(marsh, null);
+
+                                    if (ex != null)
+                                        ex.addSuppressed(e);
+                                    else
+                                        ex = U.convertException(e);
+
+                                    U.error(log, "Failed to serialize job response [nodeId=" + taskNode.id() +
+                                        ", ses=" + ses + ", jobId=" + ses.getJobId() + ", job=" + job +
+                                        ", resCls=" + (res == null ? null : res.getClass()) + ']', e);
+                                }
+
+                                try {
+                                    attrBytes = U.marshal(marsh, attrs);
+                                }
+                                catch (IgniteCheckedException e) {
+                                    attrBytes = U.marshal(marsh, Collections.emptyMap());
+
+                                    if (ex != null)
+                                        ex.addSuppressed(e);
+                                    else
+                                        ex = U.convertException(e);
+
+                                    U.error(log, "Failed to serialize job attributes [nodeId=" + taskNode.id() +
+                                        ", ses=" + ses + ", jobId=" + ses.getJobId() + ", job=" + job +
+                                        ", attrs=" + attrs + ']', e);
+                                }
+
+                                try {
+                                    exBytes = U.marshal(marsh, ex);
+                                }
+                                catch (IgniteCheckedException e) {
+                                    String msg = "Failed to serialize job exception [nodeId=" + taskNode.id() +
+                                        ", ses=" + ses + ", jobId=" + ses.getJobId() + ", job=" + job +
+                                        ", msg=\"" + e.getMessage() + "\"]";
+
+                                    ex = new IgniteException(msg);
+
+                                    U.error(log, msg, e);
+
+                                    exBytes = U.marshal(marsh, ex);
+                                }
+                            }
+
                             if (ex != null) {
                                 if (isStarted) {
                                     // Job failed.
@@ -804,19 +863,15 @@ else if (!internal && ctx.event().isRecordable(EVT_JOB_REJECTED))
                             else if (!internal && ctx.event().isRecordable(EVT_JOB_FINISHED))
                                 evts = addEvent(evts, EVT_JOB_FINISHED, /*no message for success. */null);
 
-                            boolean loc = ctx.localNodeId().equals(sndNode.id()) && !ctx.config().isMarshalLocalJobs();
-
-                            Map<Object, Object> attrs = jobCtx.getAttributes();
-
                             GridJobExecuteResponse jobRes = new GridJobExecuteResponse(
                                 ctx.localNodeId(),
                                 ses.getId(),
                                 ses.getJobId(),
-                                loc ? null : U.marshal(marsh, ex),
+                                exBytes,
                                 loc ? ex : null,
-                                loc ? null: U.marshal(marsh, res),
+                                resBytes,
                                 loc ? res : null,
-                                loc ? null : U.marshal(marsh, attrs),
+                                attrBytes,
                                 loc ? attrs : null,
                                 isCancelled(),
                                 retry ? ctx.cache().context().exchange().readyAffinityVersion() : null);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
new file mode 100644
index 0000000000000..2426dd7bdbc4e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.closure;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.compute.ComputeJobContext;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.resources.JobContextResource;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests handling of job result serialization error.
+ */
+public class GridClosureSerializationTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setMarshaller(null);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid(0);
+        startGrid(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "Convert2Lambda"})
+    public void testSerializationFailure() throws Exception {
+        final IgniteEx ignite0 = grid(0);
+        final IgniteEx ignite1 = grid(1);
+
+        GridTestUtils.assertThrows(null, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                ignite1.compute(ignite1.cluster().forNode(ignite0.localNode())).call(new IgniteCallable<Object>() {
+                    @Override public Object call() throws Exception {
+                        return new CaseClass.CaseClass2();
+                    }
+                });
+
+                return null;
+            }
+        }, BinaryObjectException.class, null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "Convert2Lambda"})
+    public void testExceptionSerializationFailure() throws Exception {
+        final IgniteEx ignite0 = grid(0);
+        final IgniteEx ignite1 = grid(1);
+
+        GridTestUtils.assertThrows(null, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                ignite1.compute(ignite1.cluster().forNode(ignite0.localNode())).call(new IgniteCallable<Object>() {
+                    @Override public Object call() throws Exception {
+                        throw new BrokenException();
+                    }
+                });
+
+                return null;
+            }
+        }, IgniteException.class, null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "Convert2Lambda"})
+    public void testAttributesSerializationFailure() throws Exception {
+        final IgniteEx ignite0 = grid(0);
+        final IgniteEx ignite1 = grid(1);
+
+        GridTestUtils.assertThrows(null, new Callable<Object>() {
+            @JobContextResource
+            private ComputeJobContext jobCtx;
+
+            @Override public Object call() throws Exception {
+                ignite1.compute(ignite1.cluster().forNode(ignite0.localNode())).call(new IgniteCallable<Object>() {
+                    @Override public Object call() throws Exception {
+                        jobCtx.setAttribute("test-attr", new BrokenAttribute());
+
+                        return null;
+                    }
+                });
+
+                return null;
+            }
+        }, IgniteException.class, null);
+    }
+
+    /**
+     * Binary marshaller will fail because subclass defines other field with different case.
+     */
+    @SuppressWarnings("unused")
+    private static class CaseClass {
+        /** */
+        private String val;
+
+        /**
+         *
+         */
+        private static class CaseClass2 extends CaseClass {
+            /** */
+            private String vAl;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class BrokenAttribute implements Externalizable {
+        /** {@inheritDoc} */
+        @Override public void writeExternal(final ObjectOutput out) throws IOException {
+            throw new IOException("Test exception");
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException {
+            throw new IOException("Test exception");
+        }
+    }
+
+    /**
+     *
+     */
+    private static class BrokenException extends Exception implements Externalizable {
+        /** {@inheritDoc} */
+        @Override public void writeExternal(final ObjectOutput out) throws IOException {
+            throw new IOException("Test exception");
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException {
+            throw new IOException("Test exception");
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index 6ab0885976f10..1c1fcf7b494a9 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -41,6 +41,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheConcurrentReadWriteTest;
 import org.apache.ignite.internal.processors.cache.OffHeapTieredTransactionSelfTest;
 import org.apache.ignite.internal.processors.closure.GridClosureProcessorSelfTest;
+import org.apache.ignite.internal.processors.closure.GridClosureSerializationTest;
 import org.apache.ignite.internal.processors.continuous.GridEventConsumeSelfTest;
 import org.apache.ignite.internal.processors.continuous.GridMessageListenSelfTest;
 import org.apache.ignite.internal.processors.odbc.OdbcProcessorValidationSelfTest;
@@ -109,6 +110,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridProductVersionSelfTest.class);
         suite.addTestSuite(GridAffinityProcessorRendezvousSelfTest.class);
         suite.addTestSuite(GridClosureProcessorSelfTest.class);
+        suite.addTestSuite(GridClosureSerializationTest.class);
         suite.addTestSuite(ClosureServiceClientsNodesTest.class);
         suite.addTestSuite(GridStartStopSelfTest.class);
         suite.addTestSuite(GridProjectionForCachesSelfTest.class);

From eeb2f2a38fb792b711eb665e380d14bc00f6e078 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 12 Dec 2016 12:14:01 +0300
Subject: [PATCH 034/446] IGNITE-3862 - GridServiceProxy invocation never times
 out. Fix

---
 .../org/apache/ignite/IgniteServices.java     |  18 +-
 .../ignite/internal/IgniteServicesImpl.java   |  11 +-
 .../affinity/GridAffinityProcessor.java       |   2 +-
 .../processors/cache/GridCacheAdapter.java    |   6 +-
 .../CacheDataStructuresManager.java           |   6 +-
 .../cache/query/GridCacheQueryManager.java    |   4 +-
 .../closure/GridClosureProcessor.java         |  32 +-
 .../processors/job/GridJobWorker.java         |   7 +
 .../platform/services/PlatformServices.java   |   2 +-
 .../service/GridServiceProcessor.java         |  11 +-
 .../processors/service/GridServiceProxy.java  |  18 +-
 .../processors/task/GridTaskWorker.java       |   7 +
 .../IgniteComputeTopologyExceptionTest.java   |   5 +-
 ...iteServiceProxyTimeoutInitializedTest.java | 284 ++++++++++++++++++
 .../testsuites/IgniteKernalSelfTestSuite.java |   2 +
 15 files changed, 390 insertions(+), 25 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceProxyTimeoutInitializedTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
index 08577c55118db..83fd487c88f1b 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
@@ -350,6 +350,22 @@ public void deployKeyAffinitySingleton(String name, Service svc, @Nullable Strin
      */
     public <T> T serviceProxy(String name, Class<? super T> svcItf, boolean sticky) throws IgniteException;
 
+    /**
+     * Gets a remote handle on the service with timeout. If service is available locally,
+     * then local instance is returned and timeout ignored, otherwise, a remote proxy is dynamically
+     * created and provided for the specified service.
+     *
+     * @param name Service name.
+     * @param svcItf Interface for the service.
+     * @param sticky Whether or not Ignite should always contact the same remote
+     *      service or try to load-balance between services.
+     * @param timeout If greater than 0 created proxy will wait for service availability only specified time,
+     *  and will limit remote service invocation time.
+     * @return Either proxy over remote service or local service if it is deployed locally.
+     * @throws IgniteException If failed to create service proxy.
+     */
+    public <T> T serviceProxy(String name, Class<? super T> svcItf, boolean sticky, long timeout) throws IgniteException;
+
     /** {@inheritDoc} */
     @Override public IgniteServices withAsync();
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
index b8042c349995f..400f28d1926a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
@@ -222,14 +222,21 @@ public IgniteServicesImpl(GridKernalContext ctx, ClusterGroupAdapter prj, boolea
     /** {@inheritDoc} */
     @Override public <T> T serviceProxy(String name, Class<? super T> svcItf, boolean sticky)
         throws IgniteException {
+        return (T) serviceProxy(name, svcItf, sticky, 0);
+    }
+
+    /** {@inheritDoc} */
+    @Override public <T> T serviceProxy(final String name, final Class<? super T> svcItf, final boolean sticky,
+        final long timeout) throws IgniteException {
         A.notNull(name, "name");
         A.notNull(svcItf, "svcItf");
         A.ensure(svcItf.isInterface(), "Service class must be an interface: " + svcItf);
+        A.ensure(timeout >= 0, "Timeout cannot be negative: " + timeout);
 
         guard();
 
         try {
-            return (T)ctx.service().serviceProxy(prj, name, svcItf, sticky);
+            return (T)ctx.service().serviceProxy(prj, name, svcItf, sticky, timeout);
         }
         finally {
             unguard();
@@ -289,4 +296,4 @@ private void unguard() {
     protected Object readResolve() throws ObjectStreamException {
         return prj.services();
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
index b9182ae5c246e..b6efafbab394c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
@@ -497,7 +497,7 @@ private <K> Map<ClusterNode, Collection<K>> keysToNodes(@Nullable final String c
     private AffinityInfo affinityInfoFromNode(@Nullable String cacheName, AffinityTopologyVersion topVer, ClusterNode n)
         throws IgniteCheckedException {
         GridTuple3<GridAffinityMessage, GridAffinityMessage, GridAffinityAssignment> t = ctx.closure()
-            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/).get();
+            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0).get();
 
         AffinityFunction f = (AffinityFunction)unmarshall(ctx, n.id(), t.get1());
         AffinityKeyMapper m = (AffinityKeyMapper)unmarshall(ctx, n.id(), t.get2());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index b30ec709870d9..4d59d50ace4d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3628,13 +3628,15 @@ private IgniteInternalFuture<?> runLoadKeysCallable(final Set<? extends K> keys,
             return ctx.closures().callAsyncNoFailover(BROADCAST,
                 new LoadKeysCallableV2<>(ctx.name(), keys, update, plc, keepBinary),
                 nodes,
-                true);
+                true,
+                0);
         }
         else {
             return ctx.closures().callAsyncNoFailover(BROADCAST,
                 new LoadKeysCallable<>(ctx.name(), keys, update, plc),
                 nodes,
-                true);
+                true,
+                0);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index c018f71b3e183..c1983df4f4d86 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -491,7 +491,8 @@ public void removeSetData(IgniteUuid id) throws IgniteCheckedException {
                     cctx.closures().callAsyncNoFailover(BROADCAST,
                         new BlockSetCallable(cctx.name(), id),
                         nodes,
-                        true).get();
+                        true,
+                        0).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
@@ -514,7 +515,8 @@ else if (!pingNodes(nodes)) {
                     cctx.closures().callAsyncNoFailover(BROADCAST,
                         new RemoveSetDataCallable(cctx.name(), id, topVer),
                         nodes,
-                        true).get();
+                        true,
+                        0).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index d4decb47ae489..1165157784b5e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -220,7 +220,7 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
 
         if (detailMetricsSz > 0)
             detailMetrics = new ConcurrentHashMap8<>(detailMetricsSz);
-        
+
         lsnr = new GridLocalEventListener() {
             @Override public void onEvent(Event evt) {
                 UUID nodeId = ((DiscoveryEvent)evt).eventNode().id();
@@ -2237,7 +2237,7 @@ public Collection<GridCacheSqlMetadata> sqlMetadata() throws IgniteCheckedExcept
 
             // Get metadata from remote nodes.
             if (!nodes.isEmpty())
-                rmtFut = cctx.closures().callAsyncNoFailover(BROADCAST, Collections.singleton(job), nodes, true);
+                rmtFut = cctx.closures().callAsyncNoFailover(BROADCAST, Collections.singleton(job), nodes, true, 0);
 
             // Get local metadata.
             IgniteInternalFuture<Collection<CacheSqlMetadata>> locFut = cctx.closures().callLocalSafe(job, true);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 9d295d3ced3b8..3ed985ef00972 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -81,6 +81,7 @@
 import static org.apache.ignite.compute.ComputeJobResultPolicy.REDUCE;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
 
 /**
  *
@@ -520,9 +521,15 @@ public ComputeTaskInternalFuture<?> affinityRun(@NotNull Collection<String> cach
      * @param <R> Type.
      * @return Grid future for collection of closure results.
      */
-    public <R> IgniteInternalFuture<R> callAsyncNoFailover(GridClosureCallMode mode, @Nullable Callable<R> job,
-        @Nullable Collection<ClusterNode> nodes, boolean sys) {
+    public <R> IgniteInternalFuture<R> callAsyncNoFailover(
+        GridClosureCallMode mode,
+        @Nullable Callable<R> job,
+        @Nullable Collection<ClusterNode> nodes,
+        boolean sys,
+        long timeout
+    ) {
         assert mode != null;
+        assert timeout >= 0 : timeout;
 
         busyLock.readLock();
 
@@ -536,6 +543,9 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(GridClosureCallMode mode,
             ctx.task().setThreadContext(TC_NO_FAILOVER, true);
             ctx.task().setThreadContext(TC_SUBGRID, nodes);
 
+            if (timeout > 0)
+                ctx.task().setThreadContext(TC_TIMEOUT, timeout);
+
             return ctx.task().execute(new T7<>(mode, job), null, sys);
         }
         finally {
@@ -548,13 +558,19 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(GridClosureCallMode mode,
      * @param jobs Closures to execute.
      * @param nodes Grid nodes.
      * @param sys If {@code true}, then system pool will be used.
+     * @param timeout If greater than 0 limits task execution. Cannot be negative.
      * @param <R> Type.
      * @return Grid future for collection of closure results.
      */
-    public <R> IgniteInternalFuture<Collection<R>> callAsyncNoFailover(GridClosureCallMode mode,
-        @Nullable Collection<? extends Callable<R>> jobs, @Nullable Collection<ClusterNode> nodes,
-        boolean sys) {
+    public <R> IgniteInternalFuture<Collection<R>> callAsyncNoFailover(
+        GridClosureCallMode mode,
+        @Nullable Collection<? extends Callable<R>> jobs,
+        @Nullable Collection<ClusterNode> nodes,
+        boolean sys,
+        long timeout
+    ) {
         assert mode != null;
+        assert timeout >= 0 : timeout;
 
         busyLock.readLock();
 
@@ -568,6 +584,9 @@ public <R> IgniteInternalFuture<Collection<R>> callAsyncNoFailover(GridClosureCa
             ctx.task().setThreadContext(TC_NO_FAILOVER, true);
             ctx.task().setThreadContext(TC_SUBGRID, nodes);
 
+            if (timeout > 0)
+                ctx.task().setThreadContext(TC_TIMEOUT, timeout);
+
             return ctx.task().execute(new T6<>(mode, jobs), null, sys);
         }
         finally {
@@ -580,6 +599,7 @@ public <R> IgniteInternalFuture<Collection<R>> callAsyncNoFailover(GridClosureCa
      * @param job Closure to execute.
      * @param nodes Grid nodes.
      * @param sys If {@code true}, then system pool will be used.
+     * @param timeout If greater than 0 limits task execution. Cannot be negative.
      * @param <R> Type.
      * @return Grid future for collection of closure results.
      */
@@ -2304,4 +2324,4 @@ private C4MLAV2(Runnable r) {
             return S.toString(C4MLAV2.class, this, super.toString());
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index f5c6a2738e115..9bee8490fca4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -43,14 +43,17 @@
 import org.apache.ignite.internal.GridJobSessionImpl;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.deployment.GridDeployment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
 import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.service.GridServiceNotFoundException;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
@@ -587,6 +590,10 @@ else if (X.hasCause(e, InterruptedException.class)) {
                         else
                             U.warn(log, msg);
                     }
+                    else if (X.hasCause(e, GridServiceNotFoundException.class) ||
+                        X.hasCause(e, ClusterTopologyCheckedException.class))
+                        // Should be throttled, because GridServiceProxy continuously retry getting service.
+                        LT.error(log, e, "Failed to execute job [jobId=" + ses.getJobId() + ", ses=" + ses + ']');
                     else
                         U.error(log, "Failed to execute job [jobId=" + ses.getJobId() + ", ses=" + ses + ']', e);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
index 962a4c0af00c3..c266986eb6e76 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
@@ -340,7 +340,7 @@ private ServiceDescriptor findDescriptor(String name) {
 
                 Object proxy = PlatformService.class.isAssignableFrom(d.serviceClass())
                     ? services.serviceProxy(name, PlatformService.class, sticky)
-                    : new GridServiceProxy<>(services.clusterGroup(), name, Service.class, sticky,
+                    : new GridServiceProxy<>(services.clusterGroup(), name, Service.class, sticky, 0,
                         platformCtx.kernalContext());
 
                 return new ServiceProxyHolder(proxy, d.serviceClass());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index b9b92b856b6c2..3690f357f1e5e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -694,9 +694,10 @@ public IgniteInternalFuture<?> cancelAll() {
 
     /**
      * @param name Service name.
+     * @param timeout If greater than 0 limits task execution time. Cannot be negative.
      * @return Service topology.
      */
-    public Map<UUID, Integer> serviceTopology(String name) throws IgniteCheckedException {
+    public Map<UUID, Integer> serviceTopology(String name, long timeout) throws IgniteCheckedException {
         ClusterNode node = cache.affinity().mapKeyToNode(name);
 
         if (node.version().compareTo(ServiceTopologyCallable.SINCE_VER) >= 0) {
@@ -708,7 +709,8 @@ public Map<UUID, Integer> serviceTopology(String name) throws IgniteCheckedExcep
                 GridClosureCallMode.BROADCAST,
                 call,
                 Collections.singletonList(node),
-                false
+                false,
+                timeout
             ).get();
         }
         else
@@ -828,12 +830,13 @@ public ServiceContextImpl serviceContext(String name) {
      * @param name Service name.
      * @param svcItf Service class.
      * @param sticky Whether multi-node request should be done.
+     * @param timeout If greater than 0 limits service acquire time. Cannot be negative.
      * @param <T> Service interface type.
      * @return The proxy of a service by its name and class.
      * @throws IgniteException If failed to create proxy.
      */
     @SuppressWarnings("unchecked")
-    public <T> T serviceProxy(ClusterGroup prj, String name, Class<? super T> svcItf, boolean sticky)
+    public <T> T serviceProxy(ClusterGroup prj, String name, Class<? super T> svcItf, boolean sticky, long timeout)
         throws IgniteException {
         if (hasLocalNode(prj)) {
             ServiceContextImpl ctx = serviceContext(name);
@@ -851,7 +854,7 @@ public <T> T serviceProxy(ClusterGroup prj, String name, Class<? super T> svcItf
             }
         }
 
-        return new GridServiceProxy<T>(prj, name, svcItf, sticky, ctx).proxy();
+        return new GridServiceProxy<T>(prj, name, svcItf, sticky, timeout, ctx).proxy();
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index 564a13ae7e6c7..aa609340b3bfb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -84,11 +84,15 @@ public class GridServiceProxy<T> implements Serializable {
     /** Whether multi-node request should be done. */
     private final boolean sticky;
 
+    /** Service availability wait timeout. */
+    private final long waitTimeout;
+
     /**
      * @param prj Grid projection.
      * @param name Service name.
      * @param svc Service type class.
      * @param sticky Whether multi-node request should be done.
+     * @param timeout Service availability wait timeout. Cannot be negative.
      * @param ctx Context.
      */
     @SuppressWarnings("unchecked")
@@ -96,12 +100,16 @@ public GridServiceProxy(ClusterGroup prj,
         String name,
         Class<? super T> svc,
         boolean sticky,
+        long timeout,
         GridKernalContext ctx)
     {
+        assert timeout >= 0 : timeout;
+
         this.prj = prj;
         this.ctx = ctx;
         this.name = name;
         this.sticky = sticky;
+        this.waitTimeout = timeout;
         hasLocNode = hasLocalNode(prj);
 
         log = ctx.log(getClass());
@@ -145,6 +153,8 @@ else if (U.isToStringMethod(mtd))
         ctx.gateway().readLock();
 
         try {
+            final long startTime = U.currentTimeMillis();
+
             while (true) {
                 ClusterNode node = null;
 
@@ -171,7 +181,8 @@ else if (U.isToStringMethod(mtd))
                             GridClosureCallMode.BROADCAST,
                             new ServiceProxyCallable(mtd.getName(), name, mtd.getParameterTypes(), args),
                             Collections.singleton(node),
-                            false
+                            false,
+                            waitTimeout
                         ).get();
                     }
                 }
@@ -203,6 +214,9 @@ else if (U.isToStringMethod(mtd))
 
                     throw new IgniteException(e);
                 }
+
+                if (waitTimeout > 0 && U.currentTimeMillis() - startTime >= waitTimeout)
+                    throw new IgniteException("Service acquire timeout was reached, stopping. [timeout=" + waitTimeout + "]");
             }
         }
         finally {
@@ -246,7 +260,7 @@ private ClusterNode randomNodeForService(String name) throws IgniteCheckedExcept
         if (hasLocNode && ctx.service().service(name) != null)
             return ctx.discovery().localNode();
 
-        Map<UUID, Integer> snapshot = ctx.service().serviceTopology(name);
+        Map<UUID, Integer> snapshot = ctx.service().serviceTopology(name, waitTimeout);
 
         if (snapshot == null || snapshot.isEmpty())
             return null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
index 3478c70a86e11..d89e80bc99825 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
@@ -68,6 +68,7 @@
 import org.apache.ignite.internal.managers.deployment.GridDeployment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.closure.AffinityTask;
+import org.apache.ignite.internal.processors.service.GridServiceNotFoundException;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.typedef.CO;
 import org.apache.ignite.internal.util.typedef.F;
@@ -1065,6 +1066,12 @@ else if (X.hasCause(e, ComputeJobFailoverException.class)) {
 
                         return null;
                     }
+                    else if (X.hasCause(e, GridServiceNotFoundException.class) ||
+                        X.hasCause(e, ClusterTopologyCheckedException.class)) {
+                        // Should be throttled, because GridServiceProxy continuously retry getting service.
+                        LT.error(log, e, "Failed to obtain remote job result policy for result from " +
+                            "ComputeTask.result(..) method (will fail the whole task): " + jobRes);
+                    }
                     else
                         U.error(log, "Failed to obtain remote job result policy for result from " +
                             "ComputeTask.result(..) method (will fail the whole task): " + jobRes, e);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
index c74dacacede51..3ed91e805d656 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
@@ -88,7 +88,8 @@ public void testCorrectCheckedException() throws Exception {
                 }
             },
             nodes,
-            false);
+            false,
+            0);
 
         try {
             fut.get();
@@ -99,4 +100,4 @@ public void testCorrectCheckedException() throws Exception {
             log.info("Expected exception: " + e);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceProxyTimeoutInitializedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceProxyTimeoutInitializedTest.java
new file mode 100644
index 0000000000000..41eef310b2dac
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceProxyTimeoutInitializedTest.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.binary.BinaryBasicIdMapper;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeTaskTimeoutException;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tests service proxy timeouts.
+ */
+public class IgniteServiceProxyTimeoutInitializedTest extends GridCommonAbstractTest {
+    /** */
+    private static Service srvc;
+
+    /** */
+    private static CountDownLatch latch1;
+
+    /** */
+    private static CountDownLatch latch2;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        final ServiceConfiguration scfg = new ServiceConfiguration();
+
+        if (gridName.endsWith("0")) {
+            scfg.setName("testService");
+            scfg.setService(srvc);
+            scfg.setMaxPerNodeCount(1);
+            scfg.setTotalCount(1);
+            scfg.setNodeFilter(new NodeFilter());
+
+            final Map<String, String> attrs = new HashMap<>();
+
+            attrs.put("clusterGroup", "0");
+
+            cfg.setUserAttributes(attrs);
+
+            cfg.setServiceConfiguration(scfg);
+        }
+
+        cfg.setMarshaller(null);
+
+        final BinaryConfiguration binCfg = new BinaryConfiguration();
+
+        // Despite defaults explicitly set to lower case.
+        binCfg.setIdMapper(new BinaryBasicIdMapper(true));
+
+        cfg.setBinaryConfiguration(binCfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * Checks that we limit retries to get not available service by timeout.
+     *
+     * @throws Exception If fail.
+     */
+    @SuppressWarnings({"Convert2Lambda", "ThrowableResultOfMethodCallIgnored"})
+    public void testUnavailableService() throws Exception {
+        srvc = new TestWaitServiceImpl();
+
+        latch1 = new CountDownLatch(1);
+        latch2 = new CountDownLatch(1);
+
+        try {
+            GridTestUtils.runAsync(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    startGrid(0);
+
+                    return null;
+                }
+            });
+
+            assert latch1.await(1, TimeUnit.MINUTES);
+
+            final IgniteEx ignite1 = startGrid(1);
+
+            final TestService testSrvc = ignite1.services().serviceProxy("testService", TestService.class, false, 500);
+
+            GridTestUtils.assertThrows(null, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    testSrvc.test();
+
+                    return null;
+                }
+            }, IgniteException.class, null);
+        }
+        finally {
+            latch2.countDown();
+        }
+    }
+
+    /**
+     * Checks that service not hangs if timeout set. Here we get hang with marshalling exception.
+     *
+     * @throws Exception If fail.
+     */
+    @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "Convert2Lambda"})
+    public void testServiceException() throws Exception {
+        srvc = new HangServiceImpl();
+
+        // Start service grid.
+        startGrid(0);
+        final IgniteEx ignite1 = startGrid(1);
+
+        final HangService testSrvc = ignite1.services().serviceProxy("testService", HangService.class, false, 1_000);
+
+        GridTestUtils.assertThrows(null, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                testSrvc.hang();
+
+                return null;
+            }
+        }, ComputeTaskTimeoutException.class, null);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 60_000;
+    }
+
+    /**
+     *
+     */
+    private static class NodeFilter implements IgnitePredicate<ClusterNode> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(final ClusterNode clusterNode) {
+            return "0".equals(clusterNode.attribute("clusterGroup"));
+        }
+    }
+
+    /**
+     *
+     */
+    private interface TestService {
+        /** */
+        void test();
+    }
+
+    /**
+     *
+     */
+    private static class TestWaitServiceImpl implements Service, TestService {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public void test() {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancel(final ServiceContext ctx) {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(final ServiceContext ctx) throws Exception {
+            latch1.countDown();
+
+            // Simulate long initialization.
+            latch2.await(1, TimeUnit.MINUTES);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(final ServiceContext ctx) throws Exception {
+            // No-op
+        }
+    }
+
+    /**
+     *
+     */
+    private static class HangClass implements Binarylizable {
+
+        /** {@inheritDoc} */
+        @Override public void writeBinary(final BinaryWriter writer) throws BinaryObjectException {
+            try {
+                U.sleep(10_000);
+            }
+            catch (IgniteInterruptedCheckedException e) {
+                throw new BinaryObjectException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readBinary(final BinaryReader reader) throws BinaryObjectException {
+            try {
+                U.sleep(10_000);
+            }
+            catch (IgniteInterruptedCheckedException e) {
+                throw new BinaryObjectException(e);
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private interface HangService {
+        /**
+         * @return Hangs deserialization.
+         */
+        HangClass hang();
+    }
+
+    /**
+     *
+     */
+    private static class HangServiceImpl implements HangService, Service {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public HangClass hang() {
+            return new HangClass();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancel(final ServiceContext ctx) {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(final ServiceContext ctx) throws Exception {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(final ServiceContext ctx) throws Exception {
+
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index d9cc8c0473522..350b7152c36d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingDefaultMarshallerTest;
 import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingJdkMarshallerTest;
 import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest;
+import org.apache.ignite.internal.processors.service.IgniteServiceProxyTimeoutInitializedTest;
 import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
 import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
 import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
@@ -141,6 +142,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridServiceProxyNodeStopSelfTest.class);
         suite.addTestSuite(GridServiceProxyClientReconnectSelfTest.class);
         suite.addTestSuite(IgniteServiceReassignmentTest.class);
+        suite.addTestSuite(IgniteServiceProxyTimeoutInitializedTest.class);
 
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class);

From 8dd4ada6287af49a243aba1e58daffa7e44f39a9 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Mon, 12 Dec 2016 14:56:44 +0300
Subject: [PATCH 035/446] GridClosureProcessor: fixed javadoc.

---
 .../ignite/internal/processors/closure/GridClosureProcessor.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 3ed985ef00972..a07dbf8e897c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -599,7 +599,6 @@ public <R> IgniteInternalFuture<Collection<R>> callAsyncNoFailover(
      * @param job Closure to execute.
      * @param nodes Grid nodes.
      * @param sys If {@code true}, then system pool will be used.
-     * @param timeout If greater than 0 limits task execution. Cannot be negative.
      * @param <R> Type.
      * @return Grid future for collection of closure results.
      */

From 83d961cff88cf2ead0bbc4ded3285f4faf9157fc Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Mon, 12 Dec 2016 17:52:22 +0300
Subject: [PATCH 036/446] IGNITE-4413 .NET: Fix DateTime argument handling in
 SqlQuery

This closes #1341

# Conflicts:
#	modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
---
 .../CacheQueriesCodeConfigurationTest.cs      | 24 +++++++++++++--
 .../Cache/Query/CacheQueriesTest.cs           |  8 +++++
 .../Cache/Query/QueryBase.cs                  | 15 ++++++++--
 .../Impl/Cache/CacheImpl.cs                   | 29 +------------------
 4 files changed, 43 insertions(+), 33 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
index d5f98ac97e4ce..92e2891e76878 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
@@ -52,7 +52,8 @@ public void TestQueryEntityConfiguration()
                         Fields = new[]
                         {
                             new QueryField("Name", typeof (string)),
-                            new QueryField("Age", typeof (int))
+                            new QueryField("Age", typeof (int)),
+                            new QueryField("Birthday", typeof(DateTime)),
                         },
                         Indexes = new[]
                         {
@@ -71,7 +72,8 @@ public void TestQueryEntityConfiguration()
                 cache[1] = new QueryPerson("Arnold", 10);
                 cache[2] = new QueryPerson("John", 20);
 
-                using (var cursor = cache.Query(new SqlQuery(typeof (QueryPerson), "age > 10")))
+                using (var cursor = cache.Query(new SqlQuery(typeof (QueryPerson), "age > ? and birthday < ?",
+                    10, DateTime.UtcNow)))
                 {
                     Assert.AreEqual(2, cursor.GetAll().Single().Key);
                 }
@@ -145,7 +147,9 @@ public void TestAttributeConfigurationQuery()
 
                 cache[2] = new AttributeQueryPerson("John", 20);
 
-                using (var cursor = cache.Query(new SqlQuery(typeof(AttributeQueryPerson), "age > ?", 10)))
+                using (var cursor = cache.Query(new SqlQuery(typeof(AttributeQueryPerson),
+                    "age > ? and age < ? and birthday > ? and birthday < ?", 10, 30,
+                    DateTime.UtcNow.AddYears(-21), DateTime.UtcNow.AddYears(-19))))
                 {
                     Assert.AreEqual(2, cursor.GetAll().Single().Key);
                 }
@@ -186,6 +190,8 @@ public AttributeQueryPerson(string name, int age)
             {
                 Name = name;
                 Age = age;
+                Salary = age;
+                Birthday = DateTime.UtcNow.AddYears(-age);
             }
 
             /// <summary>
@@ -214,6 +220,18 @@ public AttributeQueryPerson(string name, int age)
             /// </value>
             [QuerySqlField]
             public AttributeQueryAddress Address { get; set; }
+
+            /// <summary>
+            /// Gets or sets the salary.
+            /// </summary>
+            [QuerySqlField]
+            public decimal? Salary { get; set; }
+
+            /// <summary>
+            /// Gets or sets the birthday.
+            /// </summary>
+            [QuerySqlField]
+            public DateTime Birthday { get; set; }
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
index a8ffe13d5f9da..af79db99d4e9e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 
+// ReSharper disable UnusedAutoPropertyAccessor.Global
 namespace Apache.Ignite.Core.Tests.Cache.Query
 {
     using System;
@@ -758,6 +759,7 @@ public QueryPerson(string name, int age)
         {
             Name = name;
             Age = age;
+            Birthday = DateTime.UtcNow.AddYears(-age);
         }
 
         /// <summary>
@@ -769,6 +771,12 @@ public QueryPerson(string name, int age)
         /// Age.
         /// </summary>
         public int Age { get; set; }
+
+        /// <summary>
+        /// Gets or sets the birthday.
+        /// </summary>
+        [QuerySqlField]  // Enforce Timestamp serialization
+        public DateTime Birthday { get; set; }
     }
 
     /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs
index cf1f637ec1389..d992845d304fa 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs
@@ -17,6 +17,8 @@
 
 namespace Apache.Ignite.Core.Cache.Query
 {
+    using System;
+    using Apache.Ignite.Core.Binary;
     using Apache.Ignite.Core.Impl.Binary;
     using Apache.Ignite.Core.Impl.Cache;
 
@@ -66,7 +68,7 @@ protected internal QueryBase()
         /// </summary>
         /// <param name="writer">Writer.</param>
         /// <param name="args">Arguments.</param>
-        internal static void WriteQueryArgs(BinaryWriter writer, object[] args)
+        internal static void WriteQueryArgs(IBinaryRawWriter writer, object[] args)
         {
             if (args == null)
                 writer.WriteInt(0);
@@ -75,7 +77,16 @@ internal static void WriteQueryArgs(BinaryWriter writer, object[] args)
                 writer.WriteInt(args.Length);
 
                 foreach (var arg in args)
-                    writer.WriteObject(arg);
+                {
+                    // Write DateTime as TimeStamp always, otherwise it does not make sense
+                    // Wrapped DateTime comparison does not work in SQL
+                    var dt = arg as DateTime?;  // Works with DateTime also
+
+                    if (dt != null)
+                        writer.WriteTimestamp(dt);
+                    else
+                        writer.WriteObject(arg);
+                }
             }
         }
     }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
index 359611d960a90..57e70e18de0e2 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
@@ -991,7 +991,7 @@ public IQueryCursor<T> QueryFields<T>(SqlFieldsQuery qry, Func<IBinaryRawReader,
                 writer.WriteString(qry.Sql);
                 writer.WriteInt(qry.PageSize);
 
-                WriteQueryArgs(writer, qry.Arguments);
+                QueryBase.WriteQueryArgs(writer, qry.Arguments);
 
                 writer.WriteBoolean(qry.EnableDistributedJoins);
                 writer.WriteBoolean(qry.EnforceJoinOrder);
@@ -1010,33 +1010,6 @@ public IQueryCursor<T> QueryFields<T>(SqlFieldsQuery qry, Func<IBinaryRawReader,
             return new QueryCursor<TK, TV>(cursor, Marshaller, _flagKeepBinary);
         }
                 
-        /// <summary>
-        /// Write query arguments.
-        /// </summary>
-        /// <param name="writer">Writer.</param>
-        /// <param name="args">Arguments.</param>
-        private static void WriteQueryArgs(BinaryWriter writer, object[] args)
-        {
-            if (args == null)
-                writer.WriteInt(0);
-            else
-            {
-                writer.WriteInt(args.Length);
-
-                foreach (var arg in args)
-                {
-                    // Write DateTime as TimeStamp always, otherwise it does not make sense
-                    // Wrapped DateTime comparison does not work in SQL
-                    var dt = arg as DateTime?;  // Works with DateTime also
-
-                    if (dt != null)
-                        writer.WriteTimestamp(dt);
-                    else
-                        writer.WriteObject(arg);
-                }
-            }
-        }
-
         /** <inheritdoc /> */
         public IContinuousQueryHandle QueryContinuous(ContinuousQuery<TK, TV> qry)
         {

From 83710a9d1bb7379e5f3d891ed95c86096263740b Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Mon, 12 Dec 2016 17:52:22 +0300
Subject: [PATCH 037/446] IGNITE-4413 .NET: Fix DateTime argument handling in
 SqlQuery

This closes #1341
---
 .../CacheQueriesCodeConfigurationTest.cs      | 17 +++++++++--
 .../Cache/Query/CacheQueriesTest.cs           |  8 +++++
 .../Cache/Query/QueryBase.cs                  | 15 ++++++++--
 .../Impl/Cache/CacheImpl.cs                   | 29 +------------------
 4 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
index 7cb999f0ce452..863e14f53d50d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
@@ -52,7 +52,8 @@ public void TestQueryEntityConfiguration()
                         Fields = new[]
                         {
                             new QueryField("Name", typeof (string)),
-                            new QueryField("Age", typeof (int))
+                            new QueryField("Age", typeof (int)),
+                            new QueryField("Birthday", typeof(DateTime)), 
                         },
                         Indexes = new[]
                         {
@@ -71,7 +72,8 @@ public void TestQueryEntityConfiguration()
                 cache[1] = new QueryPerson("Arnold", 10);
                 cache[2] = new QueryPerson("John", 20);
 
-                using (var cursor = cache.Query(new SqlQuery(typeof (QueryPerson), "age > 10")))
+                using (var cursor = cache.Query(new SqlQuery(typeof (QueryPerson), "age > ? and birthday < ?", 
+                    10, DateTime.UtcNow)))
                 {
                     Assert.AreEqual(2, cursor.GetAll().Single().Key);
                 }
@@ -145,7 +147,9 @@ public void TestAttributeConfigurationQuery()
 
                 cache[2] = new AttributeQueryPerson("John", 20);
 
-                using (var cursor = cache.Query(new SqlQuery(typeof(AttributeQueryPerson), "age > ?", 10)))
+                using (var cursor = cache.Query(new SqlQuery(typeof(AttributeQueryPerson),
+                    "age > ? and age < ? and birthday > ? and birthday < ?", 10, 30,
+                    DateTime.UtcNow.AddYears(-21), DateTime.UtcNow.AddYears(-19))))
                 {
                     Assert.AreEqual(2, cursor.GetAll().Single().Key);
                 }
@@ -192,6 +196,7 @@ public AttributeQueryPerson(string name, int age)
                 Name = name;
                 Age = age;
                 Salary = age;
+                Birthday = DateTime.UtcNow.AddYears(-age);
             }
 
             /// <summary>
@@ -226,6 +231,12 @@ public AttributeQueryPerson(string name, int age)
             /// </summary>
             [QuerySqlField]
             public decimal? Salary { get; set; }
+
+            /// <summary>
+            /// Gets or sets the birthday.
+            /// </summary>
+            [QuerySqlField]
+            public DateTime Birthday { get; set; }
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
index 49f87c6a3853c..20fd93a550cb5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 
+// ReSharper disable UnusedAutoPropertyAccessor.Global
 namespace Apache.Ignite.Core.Tests.Cache.Query
 {
     using System;
@@ -797,6 +798,7 @@ public QueryPerson(string name, int age)
         {
             Name = name;
             Age = age;
+            Birthday = DateTime.UtcNow.AddYears(-age);
         }
 
         /// <summary>
@@ -808,6 +810,12 @@ public QueryPerson(string name, int age)
         /// Age.
         /// </summary>
         public int Age { get; set; }
+
+        /// <summary>
+        /// Gets or sets the birthday.
+        /// </summary>
+        [QuerySqlField]  // Enforce Timestamp serialization
+        public DateTime Birthday { get; set; }
     }
 
     /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs
index cf1f637ec1389..d992845d304fa 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs
@@ -17,6 +17,8 @@
 
 namespace Apache.Ignite.Core.Cache.Query
 {
+    using System;
+    using Apache.Ignite.Core.Binary;
     using Apache.Ignite.Core.Impl.Binary;
     using Apache.Ignite.Core.Impl.Cache;
 
@@ -66,7 +68,7 @@ protected internal QueryBase()
         /// </summary>
         /// <param name="writer">Writer.</param>
         /// <param name="args">Arguments.</param>
-        internal static void WriteQueryArgs(BinaryWriter writer, object[] args)
+        internal static void WriteQueryArgs(IBinaryRawWriter writer, object[] args)
         {
             if (args == null)
                 writer.WriteInt(0);
@@ -75,7 +77,16 @@ internal static void WriteQueryArgs(BinaryWriter writer, object[] args)
                 writer.WriteInt(args.Length);
 
                 foreach (var arg in args)
-                    writer.WriteObject(arg);
+                {
+                    // Write DateTime as TimeStamp always, otherwise it does not make sense
+                    // Wrapped DateTime comparison does not work in SQL
+                    var dt = arg as DateTime?;  // Works with DateTime also
+
+                    if (dt != null)
+                        writer.WriteTimestamp(dt);
+                    else
+                        writer.WriteObject(arg);
+                }
             }
         }
     }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
index e0d1a3ce872cd..186737c47906d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
@@ -980,7 +980,7 @@ public IQueryCursor<T> QueryFields<T>(SqlFieldsQuery qry, Func<IBinaryRawReader,
                 writer.WriteString(qry.Sql);
                 writer.WriteInt(qry.PageSize);
 
-                WriteQueryArgs(writer, qry.Arguments);
+                QueryBase.WriteQueryArgs(writer, qry.Arguments);
 
                 writer.WriteBoolean(qry.EnableDistributedJoins);
                 writer.WriteBoolean(qry.EnforceJoinOrder);
@@ -999,33 +999,6 @@ public IQueryCursor<T> QueryFields<T>(SqlFieldsQuery qry, Func<IBinaryRawReader,
             return new QueryCursor<TK, TV>(cursor, Marshaller, _flagKeepBinary);
         }
                 
-        /// <summary>
-        /// Write query arguments.
-        /// </summary>
-        /// <param name="writer">Writer.</param>
-        /// <param name="args">Arguments.</param>
-        private static void WriteQueryArgs(BinaryWriter writer, object[] args)
-        {
-            if (args == null)
-                writer.WriteInt(0);
-            else
-            {
-                writer.WriteInt(args.Length);
-
-                foreach (var arg in args)
-                {
-                    // Write DateTime as TimeStamp always, otherwise it does not make sense
-                    // Wrapped DateTime comparison does not work in SQL
-                    var dt = arg as DateTime?;  // Works with DateTime also
-
-                    if (dt != null)
-                        writer.WriteTimestamp(dt);
-                    else
-                        writer.WriteObject(arg);
-                }
-            }
-        }
-
         /** <inheritdoc /> */
         public IContinuousQueryHandle QueryContinuous(ContinuousQuery<TK, TV> qry)
         {

From bf1770bcebfb17d36ac90e157373901b5a181c05 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Wed, 14 Dec 2016 12:26:27 +0300
Subject: [PATCH 038/446] IGNITE-4421: Added BinaryObject handling in ODBC.
 Added test.

---
 .../cpp/odbc-test/config/queries-test.xml     |  37 ++++++
 .../cpp/odbc-test/include/Makefile.am         |   1 +
 .../cpp/odbc-test/include/complex_type.h      | 122 ++++++++++++++++++
 .../odbc-test/project/vs/odbc-test.vcxproj    |   1 +
 .../project/vs/odbc-test.vcxproj.filters      |   3 +
 .../cpp/odbc-test/src/queries_test.cpp        | 101 ++++++++++++---
 modules/platforms/cpp/odbc/src/column.cpp     |  41 +++---
 .../platforms/cpp/odbc/src/type_traits.cpp    |   3 +
 8 files changed, 277 insertions(+), 32 deletions(-)
 create mode 100644 modules/platforms/cpp/odbc-test/include/complex_type.h

diff --git a/modules/platforms/cpp/odbc-test/config/queries-test.xml b/modules/platforms/cpp/odbc-test/config/queries-test.xml
index f511c7d6bd1b2..906fadfc2d1fe 100644
--- a/modules/platforms/cpp/odbc-test/config/queries-test.xml
+++ b/modules/platforms/cpp/odbc-test/config/queries-test.xml
@@ -85,6 +85,43 @@
                         </list>
                     </property>
                 </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="name" value="cache2"/>
+                    <property name="cacheMode" value="PARTITIONED"/>
+                    <property name="atomicityMode" value="TRANSACTIONAL"/>
+                    <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+
+                    <!-- Configure type metadata to enable queries. -->
+                    <property name="queryEntities">
+                        <list>
+                            <bean class="org.apache.ignite.cache.QueryEntity">
+                                <property name="keyType" value="java.lang.Long"/>
+                                <property name="valueType" value="ComplexType"/>
+
+                                <property name="fields">
+                                    <map>
+                                        <entry key="i32Field" value="java.lang.Integer"/>
+                                        <entry key="objField" value="TestObject"/>
+                                        <entry key="strField" value="java.lang.String"/>
+                                    </map>
+                                </property>
+
+                                <property name="keyFields">
+                                    <list></list>
+                                </property>
+
+                                <property name="indexes">
+                                    <list>
+                                        <bean class="org.apache.ignite.cache.QueryIndex">
+                                            <constructor-arg value="i32Field"/>
+                                        </bean>
+                                    </list>
+                                </property>
+                            </bean>
+                        </list>
+                    </property>
+                </bean>
             </list>
         </property>
 
diff --git a/modules/platforms/cpp/odbc-test/include/Makefile.am b/modules/platforms/cpp/odbc-test/include/Makefile.am
index 832103ce7ab3a..d13e7b62de7aa 100644
--- a/modules/platforms/cpp/odbc-test/include/Makefile.am
+++ b/modules/platforms/cpp/odbc-test/include/Makefile.am
@@ -19,6 +19,7 @@ ACLOCAL_AMFLAGS =-I m4
 
 noinst_HEADERS = \
     teamcity/teamcity_messages.h \
+    complex_type.h \
     test_type.h \
     test_utils.h \
     sql_test_suite_fixture.h
diff --git a/modules/platforms/cpp/odbc-test/include/complex_type.h b/modules/platforms/cpp/odbc-test/include/complex_type.h
new file mode 100644
index 0000000000000..a84b03394b701
--- /dev/null
+++ b/modules/platforms/cpp/odbc-test/include/complex_type.h
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _IGNITE_ODBC_TEST_COMPLEX_TYPE
+#define _IGNITE_ODBC_TEST_COMPLEX_TYPE
+
+#include <string>
+
+#include "ignite/ignite.h"
+#include "ignite/ignition.h"
+
+namespace ignite
+{
+    struct TestObject
+    {
+        TestObject() :
+            f1(412),
+            f2("Lorem ipsum")
+        {
+            // No-op.
+        }
+
+        int32_t f1;
+        std::string f2;
+    };
+
+    struct ComplexType
+    {
+        ComplexType() :
+            i32Field(0)
+        {
+            // No-op.
+        }
+
+        int32_t i32Field;
+        TestObject objField;
+        std::string strField;
+    };
+}
+
+namespace ignite
+{
+    namespace binary
+    {
+
+        IGNITE_BINARY_TYPE_START(ignite::TestObject)
+
+            typedef ignite::TestObject TestObject;
+
+            IGNITE_BINARY_GET_TYPE_ID_AS_HASH(TestObject)
+            IGNITE_BINARY_GET_TYPE_NAME_AS_IS(TestObject)
+            IGNITE_BINARY_GET_FIELD_ID_AS_HASH
+            IGNITE_BINARY_GET_HASH_CODE_ZERO(TestObject)
+            IGNITE_BINARY_IS_NULL_FALSE(TestObject)
+            IGNITE_BINARY_GET_NULL_DEFAULT_CTOR(TestObject)
+
+            void Write(BinaryWriter& writer, TestObject obj)
+            {
+                writer.WriteInt32("f1", obj.f1);
+                writer.WriteString("f2", obj.f2);
+            }
+
+            TestObject Read(BinaryReader& reader)
+            {
+                TestObject obj;
+
+                obj.f1 = reader.ReadInt32("f1");
+                obj.f2 = reader.ReadString("f2");
+
+                return obj;
+            }
+
+        IGNITE_BINARY_TYPE_END
+
+        IGNITE_BINARY_TYPE_START(ignite::ComplexType)
+
+            typedef ignite::ComplexType ComplexType;
+
+            IGNITE_BINARY_GET_TYPE_ID_AS_HASH(ComplexType)
+            IGNITE_BINARY_GET_TYPE_NAME_AS_IS(ComplexType)
+            IGNITE_BINARY_GET_FIELD_ID_AS_HASH
+            IGNITE_BINARY_GET_HASH_CODE_ZERO(ComplexType)
+            IGNITE_BINARY_IS_NULL_FALSE(ComplexType)
+            IGNITE_BINARY_GET_NULL_DEFAULT_CTOR(ComplexType)
+
+            void Write(BinaryWriter& writer, ComplexType obj)
+            {
+                writer.WriteInt32("i32Field", obj.i32Field);
+                writer.WriteObject("objField", obj.objField);
+                writer.WriteString("strField", obj.strField);
+            }
+
+            ComplexType Read(BinaryReader& reader)
+            {
+                ComplexType obj;
+
+                obj.i32Field = reader.ReadInt32("i32Field");
+                obj.objField = reader.ReadObject<TestObject>("objField");
+                obj.strField = reader.ReadString("strField");
+
+                return obj;
+            }
+
+        IGNITE_BINARY_TYPE_END
+    }
+};
+
+#endif // _IGNITE_ODBC_TEST_COMPLEX_TYPE
diff --git a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
index 91603dc5f01f9..753ae4c0e8d3d 100644
--- a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
+++ b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
@@ -186,6 +186,7 @@
     <ClCompile Include="..\..\src\utility_test.cpp" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="..\..\include\complex_type.h" />
     <ClInclude Include="..\..\include\sql_test_suite_fixture.h" />
     <ClInclude Include="..\..\include\teamcity\teamcity_messages.h" />
     <ClInclude Include="..\..\include\test_type.h" />
diff --git a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
index eef6abbd60bfa..bedceaa6c205d 100644
--- a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
+++ b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
@@ -129,6 +129,9 @@
     <ClInclude Include="..\..\include\sql_test_suite_fixture.h">
       <Filter>Code</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\complex_type.h">
+      <Filter>Code\CacheTypes</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\..\config\queries-test.xml">
diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 554cee2090ed3..52b885d1d2298 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -37,6 +37,7 @@
 #include "ignite/impl/binary/binary_utils.h"
 
 #include "test_type.h"
+#include "complex_type.h"
 #include "test_utils.h"
 
 using namespace ignite;
@@ -150,11 +151,17 @@ struct QueriesTestSuiteFixture
     /**
      * Constructor.
      */
-    QueriesTestSuiteFixture() : testCache(0), env(NULL), dbc(NULL), stmt(NULL)
+    QueriesTestSuiteFixture() :
+        cache1(0),
+        cache2(0),
+        env(NULL),
+        dbc(NULL),
+        stmt(NULL)
     {
         grid = StartNode("NodeMain", "queries-test.xml");
 
-        testCache = grid.GetCache<int64_t, TestType>("cache");
+        cache1 = grid.GetCache<int64_t, TestType>("cache");
+        cache2 = grid.GetCache<int64_t, ComplexType>("cache2");
     }
 
     /**
@@ -180,8 +187,8 @@ struct QueriesTestSuiteFixture
         TestType in2(8, 7, 6, 5, "4", 3.0f, 2.0, false, Guid(1, 0), BinaryUtils::MakeDateGmt(1976, 1, 12),
             BinaryUtils::MakeTimestampGmt(1978, 8, 21, 23, 13, 45, 456));
 
-        testCache.Put(1, in1);
-        testCache.Put(2, in2);
+        cache1.Put(1, in1);
+        cache1.Put(2, in2);
 
         const size_t columnsCnt = 11;
 
@@ -362,8 +369,11 @@ struct QueriesTestSuiteFixture
     /** Node started during the test. */
     Ignite grid;
 
-    /** Test cache instance. */
-    Cache<int64_t, TestType> testCache;
+    /** Frist cache instance. */
+    Cache<int64_t, TestType> cache1;
+
+    /** Second cache instance. */
+    Cache<int64_t, ComplexType> cache2;
 
     /** ODBC Environment. */
     SQLHENV env;
@@ -444,8 +454,8 @@ BOOST_AUTO_TEST_CASE(TestTwoRowsString)
     TestType in2(8, 7, 6, 5, "4", 3.0f, 2.0, false, Guid(1, 0), BinaryUtils::MakeDateGmt(1976, 1, 12),
         BinaryUtils::MakeTimestampGmt(1978, 8, 21, 23, 13, 45, 999999999));
 
-    testCache.Put(1, in1);
-    testCache.Put(2, in2);
+    cache1.Put(1, in1);
+    cache1.Put(2, in2);
 
     const size_t columnsCnt = 11;
 
@@ -541,7 +551,7 @@ BOOST_AUTO_TEST_CASE(TestOneRowString)
     TestType in(1, 2, 3, 4, "5", 6.0f, 7.0, true, Guid(8, 9), BinaryUtils::MakeDateGmt(1987, 6, 5),
         BinaryUtils::MakeTimestampGmt(1998, 12, 27, 1, 2, 3, 456));
 
-    testCache.Put(1, in);
+    cache1.Put(1, in);
 
     const size_t columnsCnt = 11;
 
@@ -607,7 +617,7 @@ BOOST_AUTO_TEST_CASE(TestOneRowStringLen)
     TestType in(1, 2, 3, 4, "5", 6.0f, 7.0, true, Guid(8, 9), BinaryUtils::MakeDateGmt(1987, 6, 5),
         BinaryUtils::MakeTimestampGmt(1998, 12, 27, 1, 2, 3, 456));
 
-    testCache.Put(1, in);
+    cache1.Put(1, in);
 
     const size_t columnsCnt = 11;
 
@@ -649,6 +659,63 @@ BOOST_AUTO_TEST_CASE(TestOneRowStringLen)
     BOOST_CHECK(ret == SQL_NO_DATA);
 }
 
+BOOST_AUTO_TEST_CASE(TestOneRowObject)
+{
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache2");
+
+    SQLRETURN ret;
+
+    ComplexType obj;
+
+    obj.i32Field = 123;
+    obj.strField = "Some string";
+
+    obj.objField.f1 = 54321;
+    obj.objField.f2 = "Hello Ignite";
+
+    cache2.Put(1, obj);
+
+    int64_t column1 = 0;
+    int8_t column2[ODBC_BUFFER_SIZE] = { 0 };
+    char column3[ODBC_BUFFER_SIZE] = { 0 };
+
+    SQLLEN column1Len = sizeof(column1);
+    SQLLEN column2Len = sizeof(column2);
+    SQLLEN column3Len = sizeof(column3);
+
+    // Binding columns.
+    ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &column1, column1Len, &column1Len);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLBindCol(stmt, 2, SQL_C_BINARY, &column2, column2Len, &column2Len);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLBindCol(stmt, 3, SQL_C_CHAR, &column3, column3Len, &column3Len);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    SQLCHAR request[] = "SELECT i32Field, objField, strField FROM ComplexType";
+
+    ret = SQLExecDirect(stmt, request, SQL_NTS);
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLFetch(stmt);
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    BOOST_CHECK_EQUAL(column1, obj.i32Field);
+    BOOST_CHECK_EQUAL(column3, obj.strField);
+
+    ret = SQLFetch(stmt);
+    BOOST_CHECK(ret == SQL_NO_DATA);
+}
+
 BOOST_AUTO_TEST_CASE(TestDataAtExecution)
 {
     Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
@@ -661,8 +728,8 @@ BOOST_AUTO_TEST_CASE(TestDataAtExecution)
     TestType in2(8, 7, 6, 5, "4", 3.0f, 2.0, false, Guid(1, 0), BinaryUtils::MakeDateGmt(1976, 1, 12),
         BinaryUtils::MakeTimestampGmt(1978, 8, 21, 23, 13, 45, 999999999));
 
-    testCache.Put(1, in1);
-    testCache.Put(2, in2);
+    cache1.Put(1, in1);
+    cache1.Put(2, in2);
 
     const size_t columnsCnt = 11;
 
@@ -785,9 +852,9 @@ BOOST_AUTO_TEST_CASE(TestNullFields)
 
     inNull.allNulls = true;
 
-    testCache.Put(1, in);
-    testCache.Put(2, inNull);
-    testCache.Put(3, in);
+    cache1.Put(1, in);
+    cache1.Put(2, inNull);
+    cache1.Put(3, in);
 
     const size_t columnsCnt = 10;
 
@@ -900,7 +967,7 @@ BOOST_AUTO_TEST_CASE(TestDistributedJoins)
         entry.i32Field = i;
         entry.i64Field = entriesNum - i - 1;
 
-        testCache.Put(i, entry);
+        cache1.Put(i, entry);
     }
 
     Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
@@ -974,7 +1041,7 @@ BOOST_AUTO_TEST_CASE(TestDistributedJoinsWithOldVersion)
         entry.i32Field = i;
         entry.i64Field = entriesNum - i - 1;
 
-        testCache.Put(i, entry);
+        cache1.Put(i, entry);
     }
 
     Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache;DISTRIBUTED_JOINS=true;PROTOCOL_VERSION=1.6.0");
diff --git a/modules/platforms/cpp/odbc/src/column.cpp b/modules/platforms/cpp/odbc/src/column.cpp
index b076a12ee8c5e..3e6bbca6268f7 100644
--- a/modules/platforms/cpp/odbc/src/column.cpp
+++ b/modules/platforms/cpp/odbc/src/column.cpp
@@ -31,24 +31,32 @@ namespace
 
         int8_t hdr = stream.ReadInt8();
 
-        if (hdr != IGNITE_HDR_FULL)
-            return false;
-
-        int8_t protoVer = stream.ReadInt8();
+        switch (hdr)
+        {
+            case IGNITE_TYPE_BINARY:
+            {
+                // Header field + Length field + Object itself + Offset field
+                len = 1 + 4 + stream.ReadInt32() + 4;
 
-        if (protoVer != IGNITE_PROTO_VER)
-            return false;
+                break;
+            }
+            
+            case IGNITE_TYPE_OBJECT:
+            {
+                int8_t protoVer = stream.ReadInt8();
 
-        // Skipping flags
-        stream.ReadInt16();
+                if (protoVer != IGNITE_PROTO_VER)
+                    return false;
 
-        // Skipping typeId
-        stream.ReadInt32();
+                // Skipping flags, typeId and hash code
+                len = stream.ReadInt32(stream.Position() + 2 + 4 + 4);
 
-        // Skipping hash code
-        stream.ReadInt32();
+                break;
+            }
 
-        len = stream.ReadInt32();
+            default:
+                return false;
+        }
 
         return true;
     }
@@ -238,7 +246,8 @@ namespace ignite
                     break;
                 }
 
-                case IGNITE_HDR_FULL:
+                case IGNITE_TYPE_BINARY:
+                case IGNITE_TYPE_OBJECT:
                 {
                     int32_t len;
 
@@ -284,6 +293,7 @@ namespace ignite
                 default:
                 {
                     // This is a fail case.
+                    assert(false);
                     return;
                 }
             }
@@ -413,7 +423,8 @@ namespace ignite
                     break;
                 }
 
-                case IGNITE_HDR_FULL:
+                case IGNITE_TYPE_BINARY:
+                case IGNITE_TYPE_OBJECT:
                 {
                     int32_t len;
 
diff --git a/modules/platforms/cpp/odbc/src/type_traits.cpp b/modules/platforms/cpp/odbc/src/type_traits.cpp
index fced2e16be366..643e1b4e6991c 100644
--- a/modules/platforms/cpp/odbc/src/type_traits.cpp
+++ b/modules/platforms/cpp/odbc/src/type_traits.cpp
@@ -299,12 +299,14 @@ namespace ignite
                     return IGNITE_ODBC_C_TYPE_WCHAR;
 
                 case SQL_C_SSHORT:
+                case SQL_C_SHORT:
                     return IGNITE_ODBC_C_TYPE_SIGNED_SHORT;
 
                 case SQL_C_USHORT:
                     return IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT;
 
                 case SQL_C_SLONG:
+                case SQL_C_LONG:
                     return IGNITE_ODBC_C_TYPE_SIGNED_LONG;
 
                 case SQL_C_ULONG:
@@ -320,6 +322,7 @@ namespace ignite
                     return IGNITE_ODBC_C_TYPE_BIT;
 
                 case SQL_C_STINYINT:
+                case SQL_C_TINYINT:
                     return IGNITE_ODBC_C_TYPE_SIGNED_TINYINT;
 
                 case SQL_C_UTINYINT:

From ae903c59c10bb9fad58842221f3e720d1b4d54b8 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Mon, 5 Dec 2016 14:57:25 +0300
Subject: [PATCH 039/446] IGNITE-4276: Hadoop: added configurable throttle for
 shuffle message sending. Disabled by default.

---
 .../internal/processors/hadoop/HadoopJobProperty.java  | 10 +++++++++-
 .../processors/hadoop/shuffle/HadoopShuffleJob.java    |  9 ++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index dcfbcba660b86..e7bf56584f57e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -81,7 +81,15 @@ public enum HadoopJobProperty {
      * <p>
      * By default is {@code false}.
      */
-    SHUFFLE_REDUCER_NO_SORTING;
+    SHUFFLE_REDUCER_NO_SORTING,
+
+    /**
+     * Shuffle job throttle in milliseconds. When job is executed with separate shuffle thread, this parameter
+     * controls sleep duration between iterations through intermediate reducer maps.
+     * <p>
+     * Defaults to {@code 0}.
+     */
+    SHUFFLE_JOB_THROTTLE;
 
     /** */
     private final String ptyName;
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index b940c72301aae..8c731c0dce513 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.thread.IgniteThread;
 
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.PARTITION_HASHMAP_SIZE;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_JOB_THROTTLE;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_REDUCER_NO_SORTING;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.get;
 
@@ -108,6 +109,9 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
     /** */
     private final IgniteLogger log;
 
+    /** */
+    private final long throttle;
+
     /**
      * @param locReduceAddr Local reducer address.
      * @param log Logger.
@@ -136,6 +140,8 @@ public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUn
 
         maps = new AtomicReferenceArray<>(totalReducerCnt);
         msgs = new HadoopShuffleMessage[totalReducerCnt];
+
+        throttle = get(job.info(), SHUFFLE_JOB_THROTTLE, 0);
     }
 
     /**
@@ -175,7 +181,8 @@ public void startSending(String gridName, IgniteInClosure2X<T, HadoopShuffleMess
                 @Override protected void body() throws InterruptedException {
                     try {
                         while (!isCancelled()) {
-                            Thread.sleep(5);
+                            if (throttle > 0)
+                                Thread.sleep(throttle);
 
                             collectUpdatesAndSend(false);
                         }

From 2488f340c85301c0ec39cac80a8426f5f4c3caf0 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Mon, 5 Dec 2016 15:28:54 +0300
Subject: [PATCH 040/446] IGNITE-4274: Hadoop: added new property to control
 shuffle message size.

---
 .../processors/hadoop/HadoopJobProperty.java  | 37 ++++++++++++-------
 .../hadoop/shuffle/HadoopShuffleJob.java      | 12 ++++--
 2 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index e7bf56584f57e..e713caa626501 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -28,40 +28,40 @@ public enum HadoopJobProperty {
      * <p>
      * Setting it right allows to avoid rehashing.
      */
-    COMBINER_HASHMAP_SIZE,
+    COMBINER_HASHMAP_SIZE("ignite.combiner.hashmap.size"),
 
     /**
      * Initial size for hashmap which stores output of mapper or combiner and will be used as input of reducer.
      * <p>
      * Setting it right allows to avoid rehashing.
      */
-    PARTITION_HASHMAP_SIZE,
+    PARTITION_HASHMAP_SIZE("ignite.partition.hashmap.size"),
 
     /**
      * Specifies number of concurrently running mappers for external execution mode.
      * <p>
      * If not specified, defaults to {@code Runtime.getRuntime().availableProcessors()}.
      */
-    EXTERNAL_CONCURRENT_MAPPERS,
+    EXTERNAL_CONCURRENT_MAPPERS("ignite.external.concurrent.mappers"),
 
     /**
      * Specifies number of concurrently running reducers for external execution mode.
      * <p>
      * If not specified, defaults to {@code Runtime.getRuntime().availableProcessors()}.
      */
-    EXTERNAL_CONCURRENT_REDUCERS,
+    EXTERNAL_CONCURRENT_REDUCERS("ignite.external.concurrent.reducers"),
 
     /**
      * Delay in milliseconds after which Ignite server will reply job status.
      */
-    JOB_STATUS_POLL_DELAY,
+    JOB_STATUS_POLL_DELAY("ignite.job.status.poll.delay"),
 
     /**
      * Size in bytes of single memory page which will be allocated for data structures in shuffle.
      * <p>
      * By default is {@code 32 * 1024}.
      */
-    SHUFFLE_OFFHEAP_PAGE_SIZE,
+    SHUFFLE_OFFHEAP_PAGE_SIZE("ignite.shuffle.offheap.page.size"),
 
     /**
      * If set to {@code true} then input for combiner will not be sorted by key.
@@ -71,7 +71,7 @@ public enum HadoopJobProperty {
      * <p>
      * By default is {@code false}.
      */
-    SHUFFLE_COMBINER_NO_SORTING,
+    SHUFFLE_COMBINER_NO_SORTING("ignite.shuffle.combiner.no.sorting"),
 
     /**
      * If set to {@code true} then input for reducer will not be sorted by key.
@@ -81,7 +81,14 @@ public enum HadoopJobProperty {
      * <p>
      * By default is {@code false}.
      */
-    SHUFFLE_REDUCER_NO_SORTING,
+    SHUFFLE_REDUCER_NO_SORTING("ignite.shuffle.reducer.no.sorting"),
+
+    /**
+     * Defines approximate size in bytes of shuffle message which will be passed over wire from mapper to reducer.
+     * <p>
+     * Defaults to 128Kb.
+     */
+    SHUFFLE_MSG_SIZE("ignite.shuffle.message.size"),
 
     /**
      * Shuffle job throttle in milliseconds. When job is executed with separate shuffle thread, this parameter
@@ -89,23 +96,25 @@ public enum HadoopJobProperty {
      * <p>
      * Defaults to {@code 0}.
      */
-    SHUFFLE_JOB_THROTTLE;
+    SHUFFLE_JOB_THROTTLE("ignite.shuffle.job.throttle");
 
-    /** */
-    private final String ptyName;
+    /** Property name. */
+    private final String propName;
 
     /**
+     * Constrcutor.
      *
+     * @param propName Property name.
      */
-    HadoopJobProperty() {
-        ptyName = "ignite." + name().toLowerCase().replace('_', '.');
+    HadoopJobProperty(String propName) {
+        this.propName = propName;
     }
 
     /**
      * @return Property name.
      */
     public String propertyName() {
-        return ptyName;
+        return propName;
     }
 
     /**
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 8c731c0dce513..e5af8f116a07f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.thread.IgniteThread;
 
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.PARTITION_HASHMAP_SIZE;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MSG_SIZE;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_JOB_THROTTLE;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_REDUCER_NO_SORTING;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.get;
@@ -64,7 +65,7 @@
  */
 public class HadoopShuffleJob<T> implements AutoCloseable {
     /** */
-    private static final int MSG_BUF_SIZE = 128 * 1024;
+    private static final int DFLT_SHUFFLE_MSG_SIZE = 128 * 1024;
 
     /** */
     private final HadoopJob job;
@@ -109,6 +110,9 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
     /** */
     private final IgniteLogger log;
 
+    /** Message size. */
+    private final int msgSize;
+
     /** */
     private final long throttle;
 
@@ -128,6 +132,8 @@ public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUn
         this.mem = mem;
         this.log = log.getLogger(HadoopShuffleJob.class);
 
+        msgSize = get(job.info(), SHUFFLE_MSG_SIZE, DFLT_SHUFFLE_MSG_SIZE);
+
         if (!F.isEmpty(locReducers)) {
             for (int rdc : locReducers) {
                 HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, job.id(), rdc, 0, null);
@@ -320,7 +326,7 @@ private void collectUpdatesAndSend(boolean flush) throws IgniteCheckedException
                 continue; // Skip empty map and local node.
 
             if (msgs[i] == null)
-                msgs[i] = new HadoopShuffleMessage(job.id(), i, MSG_BUF_SIZE);
+                msgs[i] = new HadoopShuffleMessage(job.id(), i, msgSize);
 
             final int idx = i;
 
@@ -425,7 +431,7 @@ private void send(final int idx, int newBufMinSize) {
         });
 
         msgs[idx] = newBufMinSize == 0 ? null : new HadoopShuffleMessage(job.id(), idx,
-            Math.max(MSG_BUF_SIZE, newBufMinSize));
+            Math.max(msgSize, newBufMinSize));
     }
 
     /** {@inheritDoc} */

From 3c384212603dd31b933205d1c127dc9d0f1785e1 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Mon, 5 Dec 2016 15:42:24 +0300
Subject: [PATCH 041/446] IGNITE-4295: Added specialized methods for memory
 copying, depending on their nature (heap/offheap). Heap->heap copying is now
 performed using System.arraycopy when possible. Also it is possible to
 fallback to byte-by-byte copying until certain threshold is reached (see
 IgniteSystemProperties.IGNITE_MEMORY_PER_BYTE_COPY_THRESHOLD); it could be
 useful on some JDK implementations with inefficient Unsafe copy routines.
 E.g. on OpenJDK 8 for PowerPC. This closes #1282.

---
 .../apache/ignite/IgniteSystemProperties.java |  8 +++
 .../internal/binary/BinaryPrimitives.java     |  6 +-
 .../binary/streams/BinaryHeapInputStream.java |  4 +-
 .../streams/BinaryHeapOutputStream.java       |  2 +-
 .../streams/BinaryMemoryAllocatorChunk.java   |  3 +-
 .../streams/BinaryOffheapInputStream.java     |  4 +-
 .../streams/BinaryOffheapOutputStream.java    |  4 +-
 .../memory/PlatformInputStreamImpl.java       |  4 +-
 .../memory/PlatformOutputStreamImpl.java      |  2 +-
 .../ignite/internal/util/GridHandleTable.java | 10 ++--
 .../ignite/internal/util/GridUnsafe.java      | 60 +++++++++++++++++--
 .../ignite/internal/util/IgniteUtils.java     | 32 +---------
 .../internal/util/io/GridUnsafeDataInput.java | 12 ++--
 .../util/io/GridUnsafeDataOutput.java         | 12 ++--
 .../util/offheap/unsafe/GridUnsafeMemory.java | 10 ++--
 .../binary/BinaryFieldsOffheapSelfTest.java   |  2 +-
 .../BinaryFooterOffsetsOffheapSelfTest.java   |  2 +-
 .../binary/BinaryMarshallerSelfTest.java      |  2 +-
 ...ryObjectBuilderDefaultMappersSelfTest.java |  2 +-
 .../hadoop/shuffle/HadoopShuffleJob.java      |  2 +-
 .../hadoop/shuffle/HadoopShuffleMessage.java  |  2 +-
 .../shuffle/streams/HadoopDataOutStream.java  |  2 +-
 .../HadoopConcurrentHashMultimapSelftest.java |  2 +-
 .../collections/HadoopSkipListSelfTest.java   |  2 +-
 24 files changed, 108 insertions(+), 83 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 16fc759477f60..d0c0d5ef96652 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -493,6 +493,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_MEMORY_UNALIGNED_ACCESS = "IGNITE_MEMORY_UNALIGNED_ACCESS";
 
+    /**
+     * When unsafe memory copy if performed below this threshold, Ignite will do it on per-byte basis instead of
+     * calling to Unsafe.copyMemory().
+     * <p>
+     * Defaults to 0, meaning that threshold is disabled.
+     */
+    public static final String IGNITE_MEMORY_PER_BYTE_COPY_THRESHOLD = "IGNITE_MEMORY_PER_BYTE_COPY_THRESHOLD";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryPrimitives.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryPrimitives.java
index 86f504068d566..556282d3570c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryPrimitives.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryPrimitives.java
@@ -60,7 +60,7 @@ public static byte readByte(long ptr, int off) {
     public static byte[] readByteArray(byte[] arr, int off, int len) {
         byte[] arr0 = new byte[len];
 
-        GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF + off, arr0, GridUnsafe.BYTE_ARR_OFF, len);
+        System.arraycopy(arr, off, arr0, 0, len);
 
         return arr0;
     }
@@ -73,7 +73,7 @@ public static byte[] readByteArray(byte[] arr, int off, int len) {
     public static byte[] readByteArray(long ptr, int off, int len) {
         byte[] arr0 = new byte[len];
 
-        GridUnsafe.copyMemory(null, ptr + off, arr0, GridUnsafe.BYTE_ARR_OFF, len);
+        GridUnsafe.copyOffheapHeap(ptr + off, arr0, GridUnsafe.BYTE_ARR_OFF, len);
 
         return arr0;
     }
@@ -215,7 +215,7 @@ public static char[] readCharArray(byte[] arr, int off, int len) {
     public static char[] readCharArray(long ptr, int off, int len) {
         char[] arr0 = new char[len];
 
-        GridUnsafe.copyMemory(null, ptr + off, arr0, GridUnsafe.CHAR_ARR_OFF, len << 1);
+        GridUnsafe.copyOffheapHeap(ptr + off, arr0, GridUnsafe.CHAR_ARR_OFF, len << 1);
 
         if (BIG_ENDIAN) {
             for (int i = 0; i < len; i++)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapInputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapInputStream.java
index b5edc023ec732..322894e4c03e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapInputStream.java
@@ -78,7 +78,7 @@ public byte[] resize(int len) {
         if (data.length < len) {
             byte[] data0 = new byte[len];
 
-            GridUnsafe.copyMemory(data, GridUnsafe.BYTE_ARR_OFF, data0, GridUnsafe.BYTE_ARR_OFF, data.length);
+            System.arraycopy(data, 0, data0, 0, data.length);
 
             data = data0;
         }
@@ -105,7 +105,7 @@ public byte[] resize(int len) {
     @Override public byte[] arrayCopy() {
         byte[] res = new byte[len];
 
-        GridUnsafe.copyMemory(data, GridUnsafe.BYTE_ARR_OFF, res, GridUnsafe.BYTE_ARR_OFF, res.length);
+        System.arraycopy(data, 0, res, 0, len);
 
         return res;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapOutputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapOutputStream.java
index f06c980b3db3e..17bcdf60a12f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapOutputStream.java
@@ -75,7 +75,7 @@ public BinaryHeapOutputStream(int cap, BinaryMemoryAllocatorChunk chunk) {
     @Override public byte[] arrayCopy() {
         byte[] res = new byte[pos];
 
-        GridUnsafe.copyMemory(data, GridUnsafe.BYTE_ARR_OFF, res, GridUnsafe.BYTE_ARR_OFF, pos);
+        System.arraycopy(data, 0, res, 0, pos);
 
         return res;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryMemoryAllocatorChunk.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryMemoryAllocatorChunk.java
index f9db7da24d573..09f8c3fb57786 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryMemoryAllocatorChunk.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryMemoryAllocatorChunk.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.internal.binary.streams;
 
-import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_MARSHAL_BUFFERS_RECHECK;
@@ -72,7 +71,7 @@ public byte[] reallocate(byte[] data, int size) {
         if (this.data == data)
             this.data = newData;
 
-        GridUnsafe.copyMemory(data, GridUnsafe.BYTE_ARR_OFF, newData, GridUnsafe.BYTE_ARR_OFF, data.length);
+        System.arraycopy(data, 0, newData, 0, data.length);
 
         return newData;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapInputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapInputStream.java
index 9dc92c61e2291..6f505b904a7b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapInputStream.java
@@ -79,7 +79,7 @@ public BinaryOffheapInputStream(long ptr, int cap, boolean forceHeap) {
     @Override public byte[] arrayCopy() {
         byte[] res = new byte[len];
 
-        GridUnsafe.copyMemory(null, ptr, res, GridUnsafe.BYTE_ARR_OFF, res.length);
+        GridUnsafe.copyOffheapHeap(ptr, res, GridUnsafe.BYTE_ARR_OFF, res.length);
 
         return res;
     }
@@ -96,7 +96,7 @@ public BinaryOffheapInputStream(long ptr, int cap, boolean forceHeap) {
 
     /** {@inheritDoc} */
     @Override protected void copyAndShift(Object target, long off, int len) {
-        GridUnsafe.copyMemory(null, ptr + pos, target, off, len);
+        GridUnsafe.copyOffheapHeap(ptr + pos, target, off, len);
 
         shift(len);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapOutputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapOutputStream.java
index be9f7d38cce25..d16e575875383 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapOutputStream.java
@@ -77,7 +77,7 @@ public BinaryOffheapOutputStream(long ptr, int cap) {
     @Override public byte[] arrayCopy() {
         byte[] res = new byte[pos];
 
-        GridUnsafe.copyMemory(null, ptr, res, GridUnsafe.BYTE_ARR_OFF, pos);
+        GridUnsafe.copyOffheapHeap(ptr, res, GridUnsafe.BYTE_ARR_OFF, pos);
 
         return res;
     }
@@ -101,7 +101,7 @@ public int capacity() {
 
     /** {@inheritDoc} */
     @Override protected void copyAndShift(Object src, long offset, int len) {
-        GridUnsafe.copyMemory(src, offset, null, ptr + pos, len);
+        GridUnsafe.copyHeapOffheap(src, offset, ptr + pos, len);
 
         shift(len);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformInputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformInputStreamImpl.java
index 859de395d9aca..f5b2c3adba4cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformInputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformInputStreamImpl.java
@@ -286,7 +286,7 @@ public PlatformInputStreamImpl(PlatformMemory mem) {
         if (dataCopy == null) {
             dataCopy = new byte[len];
 
-            GridUnsafe.copyMemory(null, data, dataCopy, GridUnsafe.BYTE_ARR_OFF, dataCopy.length);
+            GridUnsafe.copyOffheapHeap(data, dataCopy, GridUnsafe.BYTE_ARR_OFF, dataCopy.length);
         }
 
         return dataCopy;
@@ -334,7 +334,7 @@ private void ensureEnoughData(int cnt) {
     private void copyAndShift(Object target, long off, int cnt) {
         ensureEnoughData(cnt);
 
-        GridUnsafe.copyMemory(null, data + pos, target, off, cnt);
+        GridUnsafe.copyOffheapHeap(data + pos, target, off, cnt);
 
         shift(cnt);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java
index 1ece10ba796e7..334afb8905667 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java
@@ -334,7 +334,7 @@ protected void shift(int cnt) {
     private void copyAndShift(Object src, long off, int len) {
         ensureCapacity(pos + len);
 
-        GridUnsafe.copyMemory(src, off, null, data + pos, len);
+        GridUnsafe.copyHeapOffheap(src, off, data + pos, len);
 
         shift(len);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
index 41a4da9fef8b9..aace1ed0ba3e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
@@ -19,8 +19,6 @@
 
 import java.util.Arrays;
 
-import static org.apache.ignite.internal.util.GridUnsafe.INT_ARR_OFF;
-
 /**
  * Lightweight identity hash table which maps objects to integer handles,
  * assigned in ascending order.
@@ -109,8 +107,8 @@ public int lookup(Object obj) {
      * Resets table to its initial (empty) state.
      */
     public void clear() {
-        GridUnsafe.copyMemory(spineEmpty, INT_ARR_OFF, spine, INT_ARR_OFF, spineEmpty.length << 2);
-        GridUnsafe.copyMemory(nextEmpty, INT_ARR_OFF, next, INT_ARR_OFF, nextEmpty.length << 2);
+        System.arraycopy(spineEmpty, 0, spine, 0, spineEmpty.length);
+        System.arraycopy(nextEmpty, 0, next, 0, nextEmpty.length);
 
         Arrays.fill(objs, null);
 
@@ -151,7 +149,7 @@ private void growSpine() {
 
         Arrays.fill(spineEmpty, -1);
 
-        GridUnsafe.copyMemory(spineEmpty, INT_ARR_OFF, spine, INT_ARR_OFF, spineEmpty.length << 2);
+        System.arraycopy(spineEmpty, 0, spine, 0, spineEmpty.length);
 
         for (int i = 0; i < this.size; i++) {
             Object obj = objs[i];
@@ -169,7 +167,7 @@ private void growEntries() {
         int newLen = (next.length << 1) + 1;
         int[] newNext = new int[newLen];
 
-        GridUnsafe.copyMemory(next, INT_ARR_OFF, newNext, INT_ARR_OFF, size << 2);
+        System.arraycopy(next, 0, newNext, 0, size);
 
         next = newNext;
         nextEmpty = new int[newLen];
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridUnsafe.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridUnsafe.java
index 35b793a9a88f5..6e9efdbc041d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridUnsafe.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridUnsafe.java
@@ -51,6 +51,10 @@ public abstract class GridUnsafe {
     /** Unaligned flag. */
     private static final boolean UNALIGNED = unaligned();
 
+    /** Per-byte copy threshold. */
+    private static final long PER_BYTE_THRESHOLD =
+        IgniteSystemProperties.getLong(IgniteSystemProperties.IGNITE_MEMORY_PER_BYTE_COPY_THRESHOLD, 0L);
+
     /** Big endian. */
     public static final boolean BIG_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN;
 
@@ -1027,14 +1031,53 @@ public static void setMemory(long addr, long len, byte val) {
     }
 
     /**
-     * Copies memory.
+     * Copy memory between offheap locations.
+     *
+     * @param srcAddr Source address.
+     * @param dstAddr Destination address.
+     * @param len Length.
+     */
+    public static void copyOffheapOffheap(long srcAddr, long dstAddr, long len) {
+        if (len <= PER_BYTE_THRESHOLD) {
+            for (int i = 0; i < len; i++)
+                UNSAFE.putByte(dstAddr + i, UNSAFE.getByte(srcAddr + i));
+        }
+        else
+            UNSAFE.copyMemory(srcAddr, dstAddr, len);
+    }
+
+    /**
+     * Copy memory from offheap to heap.
      *
-     * @param src Source.
-     * @param dst Dst.
+     * @param srcAddr Source address.
+     * @param dstBase Destination base.
+     * @param dstOff Destination offset.
      * @param len Length.
      */
-    public static void copyMemory(long src, long dst, long len) {
-        UNSAFE.copyMemory(src, dst, len);
+    public static void copyOffheapHeap(long srcAddr, Object dstBase, long dstOff, long len) {
+        if (len <= PER_BYTE_THRESHOLD) {
+            for (int i = 0; i < len; i++)
+                UNSAFE.putByte(dstBase, dstOff + i, UNSAFE.getByte(srcAddr + i));
+        }
+        else
+            UNSAFE.copyMemory(null, srcAddr, dstBase, dstOff, len);
+    }
+
+    /**
+     * Copy memory from heap to offheap.
+     *
+     * @param srcBase Source base.
+     * @param srcOff Source offset.
+     * @param dstAddr Destination address.
+     * @param len Length.
+     */
+    public static void copyHeapOffheap(Object srcBase, long srcOff, long dstAddr, long len) {
+        if (len <= PER_BYTE_THRESHOLD) {
+            for (int i = 0; i < len; i++)
+                UNSAFE.putByte(dstAddr + i, UNSAFE.getByte(srcBase, srcOff + i));
+        }
+        else
+            UNSAFE.copyMemory(srcBase, srcOff, null, dstAddr, len);
     }
 
     /**
@@ -1047,7 +1090,12 @@ public static void copyMemory(long src, long dst, long len) {
      * @param len Length.
      */
     public static void copyMemory(Object srcBase, long srcOff, Object dstBase, long dstOff, long len) {
-        UNSAFE.copyMemory(srcBase, srcOff, dstBase, dstOff, len);
+        if (len <= PER_BYTE_THRESHOLD && srcBase != null && dstBase != null) {
+            for (int i = 0; i < len; i++)
+                UNSAFE.putByte(dstBase, dstOff + i, UNSAFE.getByte(srcBase, srcOff + i));
+        }
+        else
+            UNSAFE.copyMemory(srcBase, srcOff, dstBase, dstOff, len);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 3fa3f7b92eccb..e1937bbe67810 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -265,9 +265,6 @@
  */
 @SuppressWarnings({"UnusedReturnValue", "UnnecessaryFullyQualifiedName", "RedundantStringConstructorCall"})
 public abstract class IgniteUtils {
-    /** {@code True} if {@code unsafe} should be used for array copy. */
-    private static final boolean UNSAFE_BYTE_ARR_CP = unsafeByteArrayCopyAvailable();
-
     /** Sun-specific JDK constructor factory for objects that don't have empty constructor. */
     private static final Method CTOR_FACTORY;
 
@@ -8481,28 +8478,6 @@ public static List<String> jvmArgs() {
         return ManagementFactory.getRuntimeMXBean().getInputArguments();
     }
 
-    /**
-     * As long as array copying uses JVM-private API, which is not guaranteed
-     * to be available on all JVM, this method should be called to ensure
-     * logic could work properly.
-     *
-     * @return {@code True} if unsafe copying can work on the current JVM or
-     *      {@code false} if it can't.
-     */
-    @SuppressWarnings("TypeParameterExtendsFinalClass")
-    private static boolean unsafeByteArrayCopyAvailable() {
-        try {
-            Class<? extends Unsafe> unsafeCls = Unsafe.class;
-
-            unsafeCls.getMethod("copyMemory", Object.class, long.class, Object.class, long.class, long.class);
-
-            return true;
-        }
-        catch (Exception ignored) {
-            return false;
-        }
-    }
-
     /**
      * @param src Buffer to copy from (length included).
      * @param off Offset in source buffer.
@@ -8514,10 +8489,7 @@ private static boolean unsafeByteArrayCopyAvailable() {
     public static int arrayCopy(byte[] src, int off, byte[] resBuf, int resOff, int len) {
         assert resBuf.length >= resOff + len;
 
-        if (UNSAFE_BYTE_ARR_CP)
-            GridUnsafe.copyMemory(src, GridUnsafe.BYTE_ARR_OFF + off, resBuf, GridUnsafe.BYTE_ARR_OFF + resOff, len);
-        else
-            System.arraycopy(src, off, resBuf, resOff, len);
+        System.arraycopy(src, off, resBuf, resOff, len);
 
         return resOff + len;
     }
@@ -9284,7 +9256,7 @@ public static GridCacheVersion readVersion(byte[] arr, long off, boolean verEx)
     public static byte[] copyMemory(long ptr, int size) {
         byte[] res = new byte[size];
 
-        GridUnsafe.copyMemory(null, ptr, res, GridUnsafe.BYTE_ARR_OFF, size);
+        GridUnsafe.copyOffheapHeap(ptr, res, GridUnsafe.BYTE_ARR_OFF, size);
 
         return res;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
index 2f57e5946f1e8..759b95e49f74b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
@@ -135,7 +135,7 @@ else if (now - lastCheck > CHECK_FREQ) {
             if (maxOff < halfSize) {
                 byte[] newInBuf = new byte[halfSize]; // Shrink.
 
-                GridUnsafe.copyMemory(inBuf, BYTE_ARR_OFF, newInBuf, BYTE_ARR_OFF, off);
+                System.arraycopy(inBuf, 0, newInBuf, 0, off);
 
                 buf = inBuf = newInBuf;
             }
@@ -191,7 +191,7 @@ private int offset(int more) throws IOException {
 
         byte[] arr = new byte[arrSize];
 
-        GridUnsafe.copyMemory(buf, BYTE_ARR_OFF + offset(arrSize), arr, BYTE_ARR_OFF, arrSize);
+        System.arraycopy(buf, offset(arrSize), arr, 0, arrSize);
 
         return arr;
     }
@@ -364,14 +364,14 @@ private int offset(int more) throws IOException {
 
         fromStream(len);
 
-        GridUnsafe.copyMemory(buf, BYTE_ARR_OFF + offset(len), b, BYTE_ARR_OFF, len);
+        System.arraycopy(buf, offset(len), b, 0, len);
     }
 
     /** {@inheritDoc} */
     @Override public void readFully(byte[] b, int off, int len) throws IOException {
         fromStream(len);
 
-        GridUnsafe.copyMemory(buf, BYTE_ARR_OFF + offset(len), b, BYTE_ARR_OFF + off, len);
+        System.arraycopy(buf, offset(len), b, off, len);
     }
 
     /** {@inheritDoc} */
@@ -488,7 +488,7 @@ private int offset(int more) throws IOException {
         else {
             int toRead = Math.min(len, max - this.off);
 
-            GridUnsafe.copyMemory(buf, BYTE_ARR_OFF + offset(toRead), b, BYTE_ARR_OFF + off, toRead);
+            System.arraycopy(buf, offset(toRead), b, off, toRead);
 
             return toRead;
         }
@@ -552,7 +552,7 @@ private String readUTFBody(long utfLen) throws IOException {
             else {
                 // shift and refill buffer manually
                 if (avail > 0)
-                    GridUnsafe.copyMemory(utfBuf, BYTE_ARR_OFF + pos, utfBuf, BYTE_ARR_OFF, avail);
+                    System.arraycopy(utfBuf, pos, utfBuf, 0, avail);
 
                 pos = 0;
                 end = (int)Math.min(MAX_BLOCK_SIZE, utfLen);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
index c0fe0d3b225da..c45b8fdd9fe6c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
@@ -97,7 +97,7 @@ public void bytes(byte[] bytes, int off) {
     @Override public byte[] array() {
         byte[] bytes0 = new byte[off];
 
-        GridUnsafe.copyMemory(bytes, BYTE_ARR_OFF, bytes0, BYTE_ARR_OFF, off);
+        System.arraycopy(bytes, 0, bytes0, 0, off);
 
         return bytes0;
     }
@@ -130,7 +130,7 @@ private void requestFreeSize(int size) {
         if (size > bytes.length) {
             byte[] newBytes = new byte[size << 1]; // Grow.
 
-            GridUnsafe.copyMemory(bytes, BYTE_ARR_OFF, newBytes, BYTE_ARR_OFF, off);
+            System.arraycopy(bytes, 0, newBytes, 0, off);
 
             bytes = newBytes;
         }
@@ -140,7 +140,7 @@ else if (now - lastCheck > CHECK_FREQ) {
             if (maxOff < halfSize) {
                 byte[] newBytes = new byte[halfSize]; // Shrink.
 
-                GridUnsafe.copyMemory(bytes, BYTE_ARR_OFF, newBytes, BYTE_ARR_OFF, off);
+                System.arraycopy(bytes, 0, newBytes, 0, off);
 
                 bytes = newBytes;
             }
@@ -165,7 +165,7 @@ private void onWrite(int size) throws IOException {
     @Override public void write(byte[] b) throws IOException {
         requestFreeSize(b.length);
 
-        GridUnsafe.copyMemory(b, BYTE_ARR_OFF, bytes, BYTE_ARR_OFF + off, b.length);
+        System.arraycopy(b, 0, bytes, off, b.length);
 
         onWrite(b.length);
     }
@@ -174,7 +174,7 @@ private void onWrite(int size) throws IOException {
     @Override public void write(byte[] b, int off, int len) throws IOException {
         requestFreeSize(len);
 
-        GridUnsafe.copyMemory(b, BYTE_ARR_OFF + off, bytes, BYTE_ARR_OFF + this.off, len);
+        System.arraycopy(b, off, bytes, this.off, len);
 
         onWrite(len);
     }
@@ -293,7 +293,7 @@ private void onWrite(int size) throws IOException {
 
         requestFreeSize(arr.length);
 
-        GridUnsafe.copyMemory(arr, BYTE_ARR_OFF, bytes, BYTE_ARR_OFF + off, arr.length);
+        System.arraycopy(arr, 0, bytes, off, arr.length);
 
         onWrite(arr.length);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java
index 718e1a6c17ac5..41cb586ce9f65 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java
@@ -607,7 +607,7 @@ public byte[] readBytes(long ptr, int cnt) {
      * @return The same array as passed in one.
      */
     public byte[] readBytes(long ptr, byte[] arr) {
-        GridUnsafe.copyMemory(null, ptr, arr, GridUnsafe.BYTE_ARR_OFF, arr.length);
+        GridUnsafe.copyOffheapHeap(ptr, arr, GridUnsafe.BYTE_ARR_OFF, arr.length);
 
         return arr;
     }
@@ -620,7 +620,7 @@ public byte[] readBytes(long ptr, byte[] arr) {
      * @return The same array as passed in one.
      */
     public byte[] readBytes(long ptr, byte[] arr, int off, int len) {
-        GridUnsafe.copyMemory(null, ptr, arr, GridUnsafe.BYTE_ARR_OFF + off, len);
+        GridUnsafe.copyOffheapHeap(ptr, arr, GridUnsafe.BYTE_ARR_OFF + off, len);
 
         return arr;
     }
@@ -632,7 +632,7 @@ public byte[] readBytes(long ptr, byte[] arr, int off, int len) {
      * @param arr Array.
      */
     public void writeBytes(long ptr, byte[] arr) {
-        GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF, null, ptr, arr.length);
+        GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF, ptr, arr.length);
     }
 
     /**
@@ -644,7 +644,7 @@ public void writeBytes(long ptr, byte[] arr) {
      * @param len Length.
      */
     public void writeBytes(long ptr, byte[] arr, int off, int len) {
-        GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF + off, null, ptr, len);
+        GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF + off, ptr, len);
     }
 
     /**
@@ -655,7 +655,7 @@ public void writeBytes(long ptr, byte[] arr, int off, int len) {
      * @param len Length in bytes.
      */
     public void copyMemory(long srcPtr, long destPtr, long len) {
-        GridUnsafe.copyMemory(srcPtr, destPtr, len);
+        GridUnsafe.copyOffheapOffheap(srcPtr, destPtr, len);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldsOffheapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldsOffheapSelfTest.java
index 1546252c126b2..ca4bdd2f72792 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldsOffheapSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldsOffheapSelfTest.java
@@ -46,7 +46,7 @@ public class BinaryFieldsOffheapSelfTest extends BinaryFieldsAbstractSelfTest {
 
         ptrs.add(ptr);
 
-        GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF, null, ptr, arr.length);
+        GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF, ptr, arr.length);
 
         return new BinaryObjectOffheapImpl(binaryContext(marsh), ptr, 0, arr.length);
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFooterOffsetsOffheapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFooterOffsetsOffheapSelfTest.java
index 796c0279bd603..052e652ed73bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFooterOffsetsOffheapSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFooterOffsetsOffheapSelfTest.java
@@ -46,7 +46,7 @@ public class BinaryFooterOffsetsOffheapSelfTest extends BinaryFooterOffsetsAbstr
 
         ptrs.add(ptr);
 
-        GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF, null, ptr, arr.length);
+        GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF, ptr, arr.length);
 
         return new BinaryObjectOffheapImpl(ctx, ptr, 0, arr.length);
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index 39a4d32383f7a..70df1e67a658e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -3346,7 +3346,7 @@ private long copyOffheap(BinaryObjectImpl obj) {
 
         long ptr = GridUnsafe.allocateMemory(arr.length);
 
-        GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF, null, ptr, arr.length);
+        GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF, ptr, arr.length);
 
         return ptr;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderDefaultMappersSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderDefaultMappersSelfTest.java
index 4fc5c6c9d93a5..b623c620c0ffe 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderDefaultMappersSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderDefaultMappersSelfTest.java
@@ -791,7 +791,7 @@ public void testOffheapBinary() throws Exception {
             else
                 GridUnsafe.putInt(ptr0, len);
 
-            GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF, null, ptr0 + 4, arr.length);
+            GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF, ptr0 + 4, arr.length);
 
             BinaryObject offheapObj = (BinaryObject)
                 ((CacheObjectBinaryProcessorImpl)(grid(0)).context().cacheObjects()).unmarshal(ptr, false);
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index e5af8f116a07f..025c4dafe58d5 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -311,7 +311,7 @@ private UnsafeValue(byte[] buf) {
 
         /** */
         @Override public void copyTo(long ptr) {
-            GridUnsafe.copyMemory(buf, GridUnsafe.BYTE_ARR_OFF + off, null, ptr, size);
+            GridUnsafe.copyHeapOffheap(buf, GridUnsafe.BYTE_ARR_OFF + off, ptr, size);
         }
     }
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
index 69dfe643ea292..71a314bf107e0 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
@@ -169,7 +169,7 @@ private void add(byte marker, long ptr, int size) {
 
         off += 4;
 
-        GridUnsafe.copyMemory(null, ptr, buf, GridUnsafe.BYTE_ARR_OFF + off, size);
+        GridUnsafe.copyOffheapHeap(ptr, buf, GridUnsafe.BYTE_ARR_OFF + off, size);
 
         off += size;
     }
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java
index f7b1a73f5c728..bb5acba989cb7 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java
@@ -67,7 +67,7 @@ public long move(long size) {
 
     /** {@inheritDoc} */
     @Override public void write(byte[] b, int off, int len) {
-        GridUnsafe.copyMemory(b, GridUnsafe.BYTE_ARR_OFF + off, null, move(len), len);
+        GridUnsafe.copyHeapOffheap(b, GridUnsafe.BYTE_ARR_OFF + off, move(len), len);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java
index 019b17279e4a4..7862d6eff3595 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java
@@ -162,7 +162,7 @@ private void check(HadoopConcurrentHashMultimap m, Multimap<Integer, Integer> mm
             private void read(long ptr, int size, Writable w) {
                 assert size == 4 : size;
 
-                GridUnsafe.copyMemory(null, ptr, buf, GridUnsafe.BYTE_ARR_OFF, size);
+                GridUnsafe.copyOffheapHeap(ptr, buf, GridUnsafe.BYTE_ARR_OFF, size);
 
                 dataInput.bytes(buf, size);
 
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
index d04becaa607be..111ea78d8280f 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
@@ -202,7 +202,7 @@ private void check(HadoopMultimap m, Multimap<Integer, Integer> mm, final Multim
             private void read(long ptr, int size, Writable w) {
                 assert size == 4 : size;
 
-                GridUnsafe.copyMemory(null, ptr, buf, GridUnsafe.BYTE_ARR_OFF, size);
+                GridUnsafe.copyOffheapHeap(ptr, buf, GridUnsafe.BYTE_ARR_OFF, size);
 
                 dataInput.bytes(buf, size);
 

From b7b97cfd3419da4421423e189369a4931efd3996 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Mon, 5 Dec 2016 15:49:01 +0300
Subject: [PATCH 042/446] IGNITE-4271: Hadoop: shuffle messages now use
 "direct-marshallable" path this avoiding unnecessary copying as opposed to
 user messages which were used previously. This closes #1266. This closes
 #1313.

---
 .../org/apache/ignite/internal/GridTopic.java |   3 +
 .../communication/GridIoMessageFactory.java   |  18 +++
 .../processors/hadoop/HadoopJobId.java        |  79 +++++++++++-
 .../hadoop/message/HadoopMessage.java         |   0
 .../hadoop/shuffle/HadoopShuffleAck.java      |  86 ++++++++++++-
 .../hadoop/shuffle/HadoopShuffleMessage.java  | 121 +++++++++++++++++-
 .../hadoop/shuffle/HadoopShuffle.java         |  25 +++-
 .../hadoop/shuffle/HadoopShuffleJob.java      |   5 +-
 .../child/HadoopChildProcessRunner.java       |   6 +-
 9 files changed, 326 insertions(+), 17 deletions(-)
 rename modules/{hadoop => core}/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java (100%)
 rename modules/{hadoop => core}/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java (55%)
 rename modules/{hadoop => core}/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java (66%)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 248f75b9d2689..b5608db32bc8a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -90,6 +90,9 @@ public enum GridTopic {
     /** */
     TOPIC_HADOOP,
 
+    /** */
+    TOPIC_HADOOP_MSG,
+
     /** */
     TOPIC_QUERY,
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index f36191cbafcd9..dd68984651583 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -117,6 +117,9 @@
 import org.apache.ignite.internal.processors.datastreamer.DataStreamerEntry;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamerResponse;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleAck;
+import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleMessage;
 import org.apache.ignite.internal.processors.igfs.IgfsAckMessage;
 import org.apache.ignite.internal.processors.igfs.IgfsBlockKey;
 import org.apache.ignite.internal.processors.igfs.IgfsBlocksMessage;
@@ -165,6 +168,21 @@ public GridIoMessageFactory(MessageFactory[] ext) {
         Message msg = null;
 
         switch (type) {
+            case -30:
+                msg = new HadoopJobId();
+
+                break;
+
+            case -29:
+                msg = new HadoopShuffleAck();
+
+                break;
+
+            case -28:
+                msg = new HadoopShuffleMessage();
+
+                break;
+
             case -27:
                 msg = new GridDhtTxOnePhaseCommitAckRequest();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
index 8c61fab44b15c..740ab89a6dad8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
@@ -21,14 +21,18 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
 import java.util.UUID;
 import org.apache.ignite.internal.processors.cache.GridCacheInternal;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
 
 /**
  * Job ID.
  */
-public class HadoopJobId implements GridCacheInternal, Externalizable {
+public class HadoopJobId implements Message, GridCacheInternal, Externalizable {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -62,6 +66,79 @@ public int localId() {
         return jobId;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeUuid("nodeId", nodeId))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeInt("localId", jobId))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                nodeId = reader.readUuid("nodeId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                jobId = reader.readInt("jobId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        return reader.afterMessageRead(HadoopJobId.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte directType() {
+        return -30;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 2;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void writeExternal(ObjectOutput out) throws IOException {
         U.writeUuid(out, nodeId);
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
similarity index 100%
rename from modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
similarity index 55%
rename from modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
index 6013ec6a50d6a..6dd2c2db82ff0 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
@@ -17,18 +17,23 @@
 
 package org.apache.ignite.internal.processors.hadoop.shuffle;
 
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
 import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
 
 /**
  * Acknowledgement message.
  */
-public class HadoopShuffleAck implements HadoopMessage {
+public class HadoopShuffleAck implements HadoopMessage, Message {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -71,6 +76,79 @@ public HadoopJobId jobId() {
         return jobId;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeLong("msgId", msgId))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeMessage("jobId", jobId))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                msgId = reader.readLong("msgId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                jobId = reader.readMessage("jobId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        return reader.afterMessageRead(HadoopShuffleAck.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte directType() {
+        return -29;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 2;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void writeExternal(ObjectOutput out) throws IOException {
         jobId.writeExternal(out);
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
similarity index 66%
rename from modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
index 71a314bf107e0..3732bc2e204ec 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
@@ -28,11 +29,14 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
 
 /**
  * Shuffle message.
  */
-public class HadoopShuffleMessage implements HadoopMessage {
+public class HadoopShuffleMessage implements Message, HadoopMessage {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -196,6 +200,121 @@ else if (marker == MARKER_KEY)
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeLong("msgId", msgId))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeMessage("jobId", jobId))
+                    return false;
+
+                writer.incrementState();
+
+            case 2:
+                if (!writer.writeInt("reducer", reducer))
+                    return false;
+
+                writer.incrementState();
+
+            case 3:
+                if (!writer.writeByteArray("buf", this.buf))
+                    return false;
+
+                writer.incrementState();
+
+            case 4:
+                if (!writer.writeInt("off", off))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                msgId = reader.readLong("msgId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                jobId = reader.readMessage("jobId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 2:
+                reducer = reader.readInt("reducer");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 3:
+                this.buf = reader.readByteArray("buf");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 4:
+                off = reader.readInt("off");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        return reader.afterMessageRead(HadoopShuffleMessage.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte directType() {
+        return -28;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 5;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void writeExternal(ObjectOutput out) throws IOException {
         jobId.writeExternal(out);
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
index 769bdc432f66e..a69e77934fb9c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -17,13 +17,12 @@
 
 package org.apache.ignite.internal.processors.hadoop.shuffle;
 
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridTopic;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.processors.hadoop.HadoopComponent;
 import org.apache.ignite.internal.processors.hadoop.HadoopContext;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
@@ -38,6 +37,11 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
 /**
  * Shuffle.
@@ -53,6 +57,12 @@ public class HadoopShuffle extends HadoopComponent {
     @Override public void start(HadoopContext ctx) throws IgniteCheckedException {
         super.start(ctx);
 
+        ctx.kernalContext().io().addMessageListener(GridTopic.TOPIC_HADOOP_MSG, new GridMessageListener() {
+            @Override public void onMessage(UUID nodeId, Object msg) {
+                onMessageReceived(nodeId, (HadoopMessage)msg);
+            }
+        });
+
         ctx.kernalContext().io().addUserMessageListener(GridTopic.TOPIC_HADOOP,
             new IgniteBiPredicate<UUID, Object>() {
                 @Override public boolean apply(UUID nodeId, Object msg) {
@@ -117,7 +127,10 @@ private HadoopShuffleJob<UUID> newJob(HadoopJobId jobId) throws IgniteCheckedExc
     private void send0(UUID nodeId, Object msg) throws IgniteCheckedException {
         ClusterNode node = ctx.kernalContext().discovery().node(nodeId);
 
-        ctx.kernalContext().io().sendUserMessage(F.asList(node), msg, GridTopic.TOPIC_HADOOP, false, 0);
+        if (msg instanceof Message)
+            ctx.kernalContext().io().send(node, GridTopic.TOPIC_HADOOP_MSG, (Message)msg, GridIoPolicy.PUBLIC_POOL);
+        else
+            ctx.kernalContext().io().sendUserMessage(F.asList(node), msg, GridTopic.TOPIC_HADOOP, false, 0);
     }
 
     /**
@@ -151,8 +164,8 @@ else if (res.reducersInitialized())
      */
     private void startSending(HadoopShuffleJob<UUID> shuffleJob) {
         shuffleJob.startSending(ctx.kernalContext().gridName(),
-            new IgniteInClosure2X<UUID, HadoopShuffleMessage>() {
-                @Override public void applyx(UUID dest, HadoopShuffleMessage msg) throws IgniteCheckedException {
+            new IgniteInClosure2X<UUID, HadoopMessage>() {
+                @Override public void applyx(UUID dest, HadoopMessage msg) throws IgniteCheckedException {
                     send0(dest, msg);
                 }
             }
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 025c4dafe58d5..9392b2cce81aa 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskOutput;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskType;
 import org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter;
+import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
 import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopConcurrentHashMultimap;
 import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap;
 import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopSkipList;
@@ -92,7 +93,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
     private final AtomicReferenceArray<HadoopMultimap> maps;
 
     /** */
-    private volatile IgniteInClosure2X<T, HadoopShuffleMessage> io;
+    private volatile IgniteInClosure2X<T, HadoopMessage> io;
 
     /** */
     protected ConcurrentMap<Long, IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>>> sentMsgs =
@@ -176,7 +177,7 @@ public boolean reducersInitialized() {
      * @param io IO Closure for sending messages.
      */
     @SuppressWarnings("BusyWait")
-    public void startSending(String gridName, IgniteInClosure2X<T, HadoopShuffleMessage> io) {
+    public void startSending(String gridName, IgniteInClosure2X<T, HadoopMessage> io) {
         assert snd == null;
         assert io != null;
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
index 45d9a27c51dec..7001b8ca24177 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
@@ -259,9 +259,9 @@ private void updateTasks(final HadoopJobInfoUpdateRequest req) {
                 if (req.reducersAddresses() != null) {
                     if (shuffleJob.initializeReduceAddresses(req.reducersAddresses())) {
                         shuffleJob.startSending("external",
-                            new IgniteInClosure2X<HadoopProcessDescriptor, HadoopShuffleMessage>() {
-                                @Override public void applyx(HadoopProcessDescriptor dest,
-                                    HadoopShuffleMessage msg) throws IgniteCheckedException {
+                            new IgniteInClosure2X<HadoopProcessDescriptor, HadoopMessage>() {
+                                @Override public void applyx(HadoopProcessDescriptor dest, HadoopMessage msg)
+                                    throws IgniteCheckedException {
                                     comm.sendMessage(dest, msg);
                                 }
                             });

From 04cff9b53df1da21c7552b2e47df50cacbae9158 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Mon, 5 Dec 2016 16:28:05 +0300
Subject: [PATCH 043/446] IGNITE-4355: Hadoop: Implemented parallel task
 context initialization during shuffle. This closes #1310. This closes #1313.

---
 .../hadoop/shuffle/HadoopShuffle.java         | 23 +++++---
 .../hadoop/shuffle/HadoopShuffleJob.java      | 53 +++++++++++++++++--
 2 files changed, 66 insertions(+), 10 deletions(-)

diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
index a69e77934fb9c..4450bf2f23636 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -53,6 +53,9 @@ public class HadoopShuffle extends HadoopComponent {
     /** */
     protected final GridUnsafeMemory mem = new GridUnsafeMemory(0);
 
+    /** Mutex for iternal synchronization. */
+    private final Object mux = new Object();
+
     /** {@inheritDoc} */
     @Override public void start(HadoopContext ctx) throws IgniteCheckedException {
         super.start(ctx);
@@ -141,17 +144,23 @@ private HadoopShuffleJob<UUID> job(HadoopJobId jobId) throws IgniteCheckedExcept
         HadoopShuffleJob<UUID> res = jobs.get(jobId);
 
         if (res == null) {
-            res = newJob(jobId);
+            synchronized (mux) {
+                res = jobs.get(jobId);
+
+                if (res == null) {
+                    res = newJob(jobId);
 
-            HadoopShuffleJob<UUID> old = jobs.putIfAbsent(jobId, res);
+                    HadoopShuffleJob<UUID> old = jobs.putIfAbsent(jobId, res);
 
-            if (old != null) {
-                res.close();
+                    if (old != null) {
+                        res.close();
 
-                res = old;
+                        res = old;
+                    }
+                    else if (res.reducersInitialized())
+                        startSending(res);
+                }
             }
-            else if (res.reducersInitialized())
-                startSending(res);
         }
 
         return res;
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 9392b2cce81aa..aca5fdf2bfb1b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -78,7 +78,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
     private final boolean needPartitioner;
 
     /** Collection of task contexts for each reduce task. */
-    private final Map<Integer, HadoopTaskContext> reducersCtx = new HashMap<>();
+    private final Map<Integer, LocalTaskContextProxy> reducersCtx = new HashMap<>();
 
     /** Reducers addresses. */
     private T[] reduceAddrs;
@@ -139,7 +139,7 @@ public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUn
             for (int rdc : locReducers) {
                 HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, job.id(), rdc, 0, null);
 
-                reducersCtx.put(rdc, job.getTaskContext(taskInfo));
+                reducersCtx.put(rdc, new LocalTaskContextProxy(taskInfo));
             }
         }
 
@@ -237,7 +237,7 @@ public void onShuffleMessage(HadoopShuffleMessage msg) throws IgniteCheckedExcep
         assert msg.buffer() != null;
         assert msg.offset() > 0;
 
-        HadoopTaskContext taskCtx = reducersCtx.get(msg.reducer());
+        HadoopTaskContext taskCtx = reducersCtx.get(msg.reducer()).get();
 
         HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
 
@@ -623,4 +623,51 @@ private PartitionedOutput(HadoopTaskContext taskCtx) throws IgniteCheckedExcepti
             }
         }
     }
+
+    /**
+     * Local task context proxy with delayed initialization.
+     */
+    private class LocalTaskContextProxy {
+        /** Mutex for synchronization. */
+        private final Object mux = new Object();
+
+        /** Task info. */
+        private final HadoopTaskInfo taskInfo;
+
+        /** Task context. */
+        private volatile HadoopTaskContext ctx;
+
+        /**
+         * Constructor.
+         *
+         * @param taskInfo Task info.
+         */
+        public LocalTaskContextProxy(HadoopTaskInfo taskInfo) {
+            this.taskInfo = taskInfo;
+        }
+
+        /**
+         * Get task context.
+         *
+         * @return Task context.
+         * @throws IgniteCheckedException If failed.
+         */
+        public HadoopTaskContext get() throws IgniteCheckedException {
+            HadoopTaskContext ctx0 = ctx;
+
+            if (ctx0 == null) {
+                synchronized (mux) {
+                    ctx0 = ctx;
+
+                    if (ctx0 == null) {
+                        ctx0 = job.getTaskContext(taskInfo);
+
+                        ctx = ctx0;
+                    }
+                }
+            }
+
+            return ctx0;
+        }
+    }
 }
\ No newline at end of file

From be12a7ea242dedba932c15dce005540c34711e77 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Mon, 5 Dec 2016 17:09:28 +0300
Subject: [PATCH 044/446] IGNITE-4281: Hadoop: decoupled remote and local maps
 to simplify further optimizations. This closes #1264. This closes #1315.

---
 .../hadoop/shuffle/HadoopShuffleJob.java      | 85 +++++++++++++------
 1 file changed, 61 insertions(+), 24 deletions(-)

diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index aca5fdf2bfb1b..3afb55aae8262 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -17,9 +17,7 @@
 
 package org.apache.ignite.internal.processors.hadoop.shuffle;
 
-import java.util.HashMap;
 import java.util.Iterator;
-import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
@@ -56,8 +54,8 @@
 import org.apache.ignite.thread.IgniteThread;
 
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.PARTITION_HASHMAP_SIZE;
-import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MSG_SIZE;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_JOB_THROTTLE;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MSG_SIZE;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_REDUCER_NO_SORTING;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.get;
 
@@ -77,20 +75,26 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
     /** */
     private final boolean needPartitioner;
 
-    /** Collection of task contexts for each reduce task. */
-    private final Map<Integer, LocalTaskContextProxy> reducersCtx = new HashMap<>();
+    /** Task contexts for each reduce task. */
+    private final AtomicReferenceArray<LocalTaskContextProxy> locReducersCtx;
 
     /** Reducers addresses. */
     private T[] reduceAddrs;
 
+    /** Total reducer count. */
+    private final int totalReducerCnt;
+
     /** Local reducers address. */
     private final T locReduceAddr;
 
     /** */
     private final HadoopShuffleMessage[] msgs;
 
-    /** */
-    private final AtomicReferenceArray<HadoopMultimap> maps;
+    /** Maps for local reducers. */
+    private final AtomicReferenceArray<HadoopMultimap> locMaps;
+
+    /** Maps for remote reducers. */
+    private final AtomicReferenceArray<HadoopMultimap> rmtMaps;
 
     /** */
     private volatile IgniteInClosure2X<T, HadoopMessage> io;
@@ -129,23 +133,27 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
     public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUnsafeMemory mem,
         int totalReducerCnt, int[] locReducers) throws IgniteCheckedException {
         this.locReduceAddr = locReduceAddr;
+        this.totalReducerCnt = totalReducerCnt;
         this.job = job;
         this.mem = mem;
         this.log = log.getLogger(HadoopShuffleJob.class);
 
         msgSize = get(job.info(), SHUFFLE_MSG_SIZE, DFLT_SHUFFLE_MSG_SIZE);
 
+        locReducersCtx = new AtomicReferenceArray<>(totalReducerCnt);
+
         if (!F.isEmpty(locReducers)) {
             for (int rdc : locReducers) {
                 HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, job.id(), rdc, 0, null);
 
-                reducersCtx.put(rdc, new LocalTaskContextProxy(taskInfo));
+                locReducersCtx.set(rdc, new LocalTaskContextProxy(taskInfo));
             }
         }
 
         needPartitioner = totalReducerCnt > 1;
 
-        maps = new AtomicReferenceArray<>(totalReducerCnt);
+        locMaps = new AtomicReferenceArray<>(totalReducerCnt);
+        rmtMaps = new AtomicReferenceArray<>(totalReducerCnt);
         msgs = new HadoopShuffleMessage[totalReducerCnt];
 
         throttle = get(job.info(), SHUFFLE_JOB_THROTTLE, 0);
@@ -237,13 +245,13 @@ public void onShuffleMessage(HadoopShuffleMessage msg) throws IgniteCheckedExcep
         assert msg.buffer() != null;
         assert msg.offset() > 0;
 
-        HadoopTaskContext taskCtx = reducersCtx.get(msg.reducer()).get();
+        HadoopTaskContext taskCtx = locReducersCtx.get(msg.reducer()).get();
 
         HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
 
         perfCntr.onShuffleMessage(msg.reducer(), U.currentTimeMillis());
 
-        HadoopMultimap map = getOrCreateMap(maps, msg.reducer());
+        HadoopMultimap map = getOrCreateMap(locMaps, msg.reducer());
 
         // Add data from message to the map.
         try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
@@ -320,10 +328,10 @@ private UnsafeValue(byte[] buf) {
      * Sends map updates to remote reducers.
      */
     private void collectUpdatesAndSend(boolean flush) throws IgniteCheckedException {
-        for (int i = 0; i < maps.length(); i++) {
-            HadoopMultimap map = maps.get(i);
+        for (int i = 0; i < rmtMaps.length(); i++) {
+            HadoopMultimap map = rmtMaps.get(i);
 
-            if (map == null || locReduceAddr.equals(reduceAddrs[i]))
+            if (map == null)
                 continue; // Skip empty map and local node.
 
             if (msgs[i] == null)
@@ -448,7 +456,8 @@ private void send(final int idx, int newBufMinSize) {
             }
         }
 
-        close(maps);
+        close(locMaps);
+        close(rmtMaps);
     }
 
     /**
@@ -473,7 +482,7 @@ public IgniteInternalFuture<?> flush() throws IgniteCheckedException {
 
         flushed = true;
 
-        if (maps.length() == 0)
+        if (totalReducerCnt == 0)
             return new GridFinishedFuture<>();
 
         U.await(ioInitLatch);
@@ -544,7 +553,7 @@ public HadoopTaskInput input(HadoopTaskContext taskCtx) throws IgniteCheckedExce
             case REDUCE:
                 int reducer = taskCtx.taskInfo().taskNumber();
 
-                HadoopMultimap m = maps.get(reducer);
+                HadoopMultimap m = locMaps.get(reducer);
 
                 if (m != null)
                     return m.input(taskCtx);
@@ -572,12 +581,25 @@ public HadoopTaskInput input(HadoopTaskContext taskCtx) throws IgniteCheckedExce
         }
     }
 
+    /**
+     * Check if certain partition (reducer) is local.
+     *
+     * @param part Partition.
+     * @return {@code True} if local.
+     */
+    private boolean isLocalPartition(int part) {
+        return locReducersCtx.get(part) != null;
+    }
+
     /**
      * Partitioned output.
      */
     private class PartitionedOutput implements HadoopTaskOutput {
         /** */
-        private final HadoopTaskOutput[] adders = new HadoopTaskOutput[maps.length()];
+        private final HadoopTaskOutput[] locAdders = new HadoopTaskOutput[locMaps.length()];
+
+        /** */
+        private final HadoopTaskOutput[] rmtAdders = new HadoopTaskOutput[rmtMaps.length()];
 
         /** */
         private HadoopPartitioner partitioner;
@@ -601,23 +623,38 @@ private PartitionedOutput(HadoopTaskContext taskCtx) throws IgniteCheckedExcepti
             int part = 0;
 
             if (partitioner != null) {
-                part = partitioner.partition(key, val, adders.length);
+                part = partitioner.partition(key, val, totalReducerCnt);
 
-                if (part < 0 || part >= adders.length)
+                if (part < 0 || part >= totalReducerCnt)
                     throw new IgniteCheckedException("Invalid partition: " + part);
             }
 
-            HadoopTaskOutput out = adders[part];
+            HadoopTaskOutput out;
 
-            if (out == null)
-                adders[part] = out = getOrCreateMap(maps, part).startAdding(taskCtx);
+            if (isLocalPartition(part)) {
+                out = locAdders[part];
+
+                if (out == null)
+                    locAdders[part] = out = getOrCreateMap(locMaps, part).startAdding(taskCtx);
+            }
+            else {
+                out = rmtAdders[part];
+
+                if (out == null)
+                    rmtAdders[part] = out = getOrCreateMap(rmtMaps, part).startAdding(taskCtx);
+            }
 
             out.write(key, val);
         }
 
         /** {@inheritDoc} */
         @Override public void close() throws IgniteCheckedException {
-            for (HadoopTaskOutput adder : adders) {
+            for (HadoopTaskOutput adder : locAdders) {
+                if (adder != null)
+                    adder.close();
+            }
+
+            for (HadoopTaskOutput adder : rmtAdders) {
                 if (adder != null)
                     adder.close();
             }

From e857f297bb22bde984b5e99c1fc62a389fa54f9b Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Tue, 6 Dec 2016 13:12:23 +0300
Subject: [PATCH 045/446] IGNITE-4271: Hadoop: set proper message type IDs to
 avoid clashing with SQL messages.

---
 .../communication/GridIoMessageFactory.java      | 16 ++++++++--------
 .../internal/processors/hadoop/HadoopJobId.java  |  2 +-
 .../hadoop/shuffle/HadoopShuffleAck.java         |  2 +-
 .../hadoop/shuffle/HadoopShuffleMessage.java     |  2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index dd68984651583..86742e804e3a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -168,21 +168,26 @@ public GridIoMessageFactory(MessageFactory[] ext) {
         Message msg = null;
 
         switch (type) {
-            case -30:
+            case -39:
                 msg = new HadoopJobId();
 
                 break;
 
-            case -29:
+            case -38:
                 msg = new HadoopShuffleAck();
 
                 break;
 
-            case -28:
+            case -37:
                 msg = new HadoopShuffleMessage();
 
                 break;
 
+            case -36:
+                msg = new GridDhtAtomicSingleUpdateRequest();
+
+                break;
+
             case -27:
                 msg = new GridDhtTxOnePhaseCommitAckRequest();
 
@@ -793,11 +798,6 @@ public GridIoMessageFactory(MessageFactory[] ext) {
 
                 break;
 
-            case -36:
-                msg = new GridDhtAtomicSingleUpdateRequest();
-
-                break;
-
             // [-3..119] [124..127] [-36]- this
             // [120..123] - DR
             // [-4..-22, -30..-35] - SQL
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
index 740ab89a6dad8..761b90427cb0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
@@ -126,7 +126,7 @@ public int localId() {
 
     /** {@inheritDoc} */
     @Override public byte directType() {
-        return -30;
+        return -39;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
index 6dd2c2db82ff0..9a03f99adde83 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
@@ -136,7 +136,7 @@ public HadoopJobId jobId() {
 
     /** {@inheritDoc} */
     @Override public byte directType() {
-        return -29;
+        return -38;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
index 3732bc2e204ec..af54dcccf2e93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
@@ -302,7 +302,7 @@ else if (marker == MARKER_KEY)
 
     /** {@inheritDoc} */
     @Override public byte directType() {
-        return -28;
+        return -37;
     }
 
     /** {@inheritDoc} */

From 58c338051aed89a787c2a575856e949cc44e8b8c Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Tue, 6 Dec 2016 13:23:05 +0300
Subject: [PATCH 046/446] IGNITE-4301: Hadoop: Optimized shuffle so that only
 one ack is needed when running in embedded mode. This closes #1319.

---
 .../communication/GridIoMessageFactory.java   |  12 +
 .../shuffle/HadoopShuffleFinishRequest.java   | 172 +++++++++++++
 .../shuffle/HadoopShuffleFinishResponse.java  | 142 +++++++++++
 .../hadoop/shuffle/HadoopShuffle.java         |  45 ++--
 .../hadoop/shuffle/HadoopShuffleJob.java      | 231 ++++++++++++++++--
 .../shuffle/HadoopShuffleLocalState.java      |  67 +++++
 .../shuffle/HadoopShuffleRemoteState.java     |  64 +++++
 .../child/HadoopChildProcessRunner.java       |   6 +-
 8 files changed, 686 insertions(+), 53 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishRequest.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishResponse.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleLocalState.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 86742e804e3a2..4ffb22087fcba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -119,6 +119,8 @@
 import org.apache.ignite.internal.processors.datastreamer.DataStreamerResponse;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
 import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleAck;
+import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishRequest;
+import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishResponse;
 import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleMessage;
 import org.apache.ignite.internal.processors.igfs.IgfsAckMessage;
 import org.apache.ignite.internal.processors.igfs.IgfsBlockKey;
@@ -168,6 +170,16 @@ public GridIoMessageFactory(MessageFactory[] ext) {
         Message msg = null;
 
         switch (type) {
+            case -41:
+                msg = new HadoopShuffleFinishResponse();
+
+                break;
+
+            case -40:
+                msg = new HadoopShuffleFinishRequest();
+
+                break;
+
             case -39:
                 msg = new HadoopJobId();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishRequest.java
new file mode 100644
index 0000000000000..f568c2e2de451
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishRequest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+
+/**
+ * Shuffle finish request.
+ */
+public class HadoopShuffleFinishRequest implements Message, HadoopMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Job ID. */
+    @GridToStringInclude
+    private HadoopJobId jobId;
+
+    /** Total message count. */
+    private long msgCnt;
+
+    /**
+     * Default constructor.
+     */
+    public HadoopShuffleFinishRequest() {
+        // No-op.
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param jobId Job.
+     * @param msgCnt Message count.
+     */
+    public HadoopShuffleFinishRequest(HadoopJobId jobId, long msgCnt) {
+        this.jobId = jobId;
+        this.msgCnt = msgCnt;
+    }
+
+    /**
+     * @return Job ID.
+     */
+    public HadoopJobId jobId() {
+        return jobId;
+    }
+
+    /**
+     * @return Message count.
+     */
+    public long messageCount() {
+        return msgCnt;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeMessage("jobId", jobId))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeLong("msgCnt", msgCnt))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                jobId = reader.readMessage("jobId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                msgCnt = reader.readLong("msgCnt");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        return reader.afterMessageRead(HadoopShuffleFinishRequest.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte directType() {
+        return -40;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 2;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeExternal(ObjectOutput out) throws IOException {
+        jobId.writeExternal(out);
+
+        out.writeLong(msgCnt);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        jobId = new HadoopJobId();
+
+        jobId.readExternal(in);
+
+        msgCnt = in.readLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HadoopShuffleFinishRequest.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishResponse.java
new file mode 100644
index 0000000000000..4b7c93bf07a35
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishResponse.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+
+/**
+ * Shuffle finish response.
+ */
+public class HadoopShuffleFinishResponse implements Message, HadoopMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Job ID. */
+    @GridToStringInclude
+    private HadoopJobId jobId;
+
+    /**
+     * Default constructor.
+     */
+    public HadoopShuffleFinishResponse() {
+        // No-op.
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param jobId Job.
+     */
+    public HadoopShuffleFinishResponse(HadoopJobId jobId) {
+        this.jobId = jobId;
+    }
+
+    /**
+     * @return Job ID.
+     */
+    public HadoopJobId jobId() {
+        return jobId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeMessage("jobId", jobId))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                jobId = reader.readMessage("jobId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        return reader.afterMessageRead(HadoopShuffleFinishResponse.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte directType() {
+        return -41;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeExternal(ObjectOutput out) throws IOException {
+        jobId.writeExternal(out);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        jobId = new HadoopJobId();
+
+        jobId.readExternal(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HadoopShuffleFinishResponse.class, this);
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
index 4450bf2f23636..82bbd32f0cd5b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -103,7 +103,7 @@ private HadoopShuffleJob<UUID> newJob(HadoopJobId jobId) throws IgniteCheckedExc
         HadoopMapReducePlan plan = ctx.jobTracker().plan(jobId);
 
         HadoopShuffleJob<UUID> job = new HadoopShuffleJob<>(ctx.localNodeId(), log,
-            ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()));
+            ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()), true);
 
         UUID[] rdcAddrs = new UUID[plan.reducers()];
 
@@ -189,37 +189,34 @@ private void startSending(HadoopShuffleJob<UUID> shuffleJob) {
      * @return {@code True}.
      */
     public boolean onMessageReceived(UUID src, HadoopMessage msg) {
-        if (msg instanceof HadoopShuffleMessage) {
-            HadoopShuffleMessage m = (HadoopShuffleMessage)msg;
+        try {
+            if (msg instanceof HadoopShuffleMessage) {
+                HadoopShuffleMessage m = (HadoopShuffleMessage)msg;
 
-            try {
-                job(m.jobId()).onShuffleMessage(m);
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Message handling failed.", e);
+                job(m.jobId()).onShuffleMessage(src, m);
             }
+            else if (msg instanceof HadoopShuffleAck) {
+                HadoopShuffleAck m = (HadoopShuffleAck)msg;
 
-            try {
-                // Reply with ack.
-                send0(src, new HadoopShuffleAck(m.id(), m.jobId()));
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to reply back to shuffle message sender [snd=" + src + ", msg=" + msg + ']', e);
+                job(m.jobId()).onShuffleAck(m);
             }
-        }
-        else if (msg instanceof HadoopShuffleAck) {
-            HadoopShuffleAck m = (HadoopShuffleAck)msg;
+            else if (msg instanceof HadoopShuffleFinishRequest) {
+                HadoopShuffleFinishRequest m = (HadoopShuffleFinishRequest)msg;
 
-            try {
-                job(m.jobId()).onShuffleAck(m);
+                job(m.jobId()).onShuffleFinishRequest(src, m);
             }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Message handling failed.", e);
+            else if (msg instanceof HadoopShuffleFinishResponse) {
+                HadoopShuffleFinishResponse m = (HadoopShuffleFinishResponse)msg;
+
+                job(m.jobId()).onShuffleFinishResponse(src);
             }
+            else
+                throw new IllegalStateException("Unknown message type received to Hadoop shuffle [src=" + src +
+                    ", msg=" + msg + ']');
+        }
+        catch (IgniteCheckedException e) {
+            U.error(log, "Message handling failed.", e);
         }
-        else
-            throw new IllegalStateException("Unknown message type received to Hadoop shuffle [src=" + src +
-                ", msg=" + msg + ']');
 
         return true;
     }
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 3afb55aae8262..0a3a0ae3aa118 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -17,7 +17,9 @@
 
 package org.apache.ignite.internal.processors.hadoop.shuffle;
 
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
@@ -27,6 +29,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.hadoop.HadoopJob;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
 import org.apache.ignite.internal.processors.hadoop.HadoopPartitioner;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
@@ -118,9 +121,21 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
     /** Message size. */
     private final int msgSize;
 
+    /** Local shuffle states. */
+    private volatile HashMap<T, HadoopShuffleLocalState> locShuffleStates = new HashMap<>();
+
+    /** Remote shuffle states. */
+    private volatile HashMap<T, HadoopShuffleRemoteState> rmtShuffleStates = new HashMap<>();
+
+    /** Mutex for internal synchronization. */
+    private final Object mux = new Object();
+
     /** */
     private final long throttle;
 
+    /** Embedded mode flag. */
+    private final boolean embedded;
+
     /**
      * @param locReduceAddr Local reducer address.
      * @param log Logger.
@@ -128,15 +143,17 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
      * @param mem Memory.
      * @param totalReducerCnt Amount of reducers in the Job.
      * @param locReducers Reducers will work on current node.
+     * @param embedded Whether shuffle is running in embedded mode.
      * @throws IgniteCheckedException If error.
      */
     public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUnsafeMemory mem,
-        int totalReducerCnt, int[] locReducers) throws IgniteCheckedException {
+        int totalReducerCnt, int[] locReducers, boolean embedded) throws IgniteCheckedException {
         this.locReduceAddr = locReduceAddr;
         this.totalReducerCnt = totalReducerCnt;
         this.job = job;
         this.mem = mem;
         this.log = log.getLogger(HadoopShuffleJob.class);
+        this.embedded = embedded;
 
         msgSize = get(job.info(), SHUFFLE_MSG_SIZE, DFLT_SHUFFLE_MSG_SIZE);
 
@@ -238,10 +255,11 @@ private HadoopMultimap getOrCreateMap(AtomicReferenceArray<HadoopMultimap> maps,
     }
 
     /**
+     * @param src Source.
      * @param msg Message.
      * @throws IgniteCheckedException Exception.
      */
-    public void onShuffleMessage(HadoopShuffleMessage msg) throws IgniteCheckedException {
+    public void onShuffleMessage(T src, HadoopShuffleMessage msg) throws IgniteCheckedException {
         assert msg.buffer() != null;
         assert msg.offset() > 0;
 
@@ -276,6 +294,15 @@ public void onShuffleMessage(HadoopShuffleMessage msg) throws IgniteCheckedExcep
                 }
             });
         }
+
+        if (embedded) {
+            // No immediate response.
+            if (localShuffleState(src).onShuffleMessage())
+                sendFinishResponse(src, msg.jobId());
+        }
+        else
+            // Response for every message.
+            io.apply(src, new HadoopShuffleAck(msg.id(), msg.jobId()));
     }
 
     /**
@@ -291,6 +318,121 @@ public void onShuffleAck(HadoopShuffleAck ack) {
             log.warning("Received shuffle ack for not registered shuffle id: " + ack);
     }
 
+    /**
+     * Process shuffle finish request.
+     *
+     * @param src Source.
+     * @param msg Shuffle finish message.
+     */
+    public void onShuffleFinishRequest(T src, HadoopShuffleFinishRequest msg) {
+        if (log.isDebugEnabled())
+            log.debug("Received shuffle finish request [jobId=" + job.id() + ", src=" + src + ", req=" + msg + ']');
+
+        HadoopShuffleLocalState state = localShuffleState(src);
+
+        if (state.onShuffleFinishMessage(msg.messageCount()))
+            sendFinishResponse(src, msg.jobId());
+    }
+
+    /**
+     * Process shuffle finish response.
+     *
+     * @param src Source.
+     */
+    public void onShuffleFinishResponse(T src) {
+        if (log.isDebugEnabled())
+            log.debug("Received shuffle finish response [jobId=" + job.id() + ", src=" + src + ']');
+
+        remoteShuffleState(src).onShuffleFinishResponse();
+    }
+
+    /**
+     * Send finish response.
+     *
+     * @param dest Destination.
+     * @param jobId Job ID.
+     */
+    @SuppressWarnings("unchecked")
+    private void sendFinishResponse(T dest, HadoopJobId jobId) {
+        if (log.isDebugEnabled())
+            log.debug("Sent shuffle finish response [jobId=" + jobId + ", dest=" + dest + ']');
+
+        HadoopShuffleFinishResponse msg = new HadoopShuffleFinishResponse(jobId);
+
+        io.apply(dest, msg);
+    }
+
+    /**
+     * Get local shuffle state for node.
+     *
+     * @param src Source
+     * @return Local shuffle state.
+     */
+    private HadoopShuffleLocalState localShuffleState(T src) {
+        HashMap<T, HadoopShuffleLocalState> states = locShuffleStates;
+
+        HadoopShuffleLocalState res = states.get(src);
+
+        if (res == null) {
+            synchronized (mux) {
+                res = locShuffleStates.get(src);
+
+                if (res == null) {
+                    res = new HadoopShuffleLocalState();
+
+                    states = new HashMap<>(locShuffleStates);
+
+                    states.put(src, res);
+
+                    locShuffleStates = states;
+                }
+            }
+        }
+
+        return res;
+    }
+
+    /**
+     * Get remote shuffle state for node.
+     *
+     * @param src Source.
+     * @return Remote shuffle state.
+     */
+    private HadoopShuffleRemoteState remoteShuffleState(T src) {
+        HashMap<T, HadoopShuffleRemoteState> states = rmtShuffleStates;
+
+        HadoopShuffleRemoteState res = states.get(src);
+
+        if (res == null) {
+            synchronized (mux) {
+                res = rmtShuffleStates.get(src);
+
+                if (res == null) {
+                    res = new HadoopShuffleRemoteState();
+
+                    states = new HashMap<>(rmtShuffleStates);
+
+                    states.put(src, res);
+
+                    rmtShuffleStates = states;
+                }
+            }
+        }
+
+        return res;
+    }
+
+    /**
+     * Get all remote shuffle states.
+     *
+     * @return Remote shuffle states.
+     */
+    private HashMap<T, HadoopShuffleRemoteState> remoteShuffleStates() {
+        synchronized (mux) {
+            return new HashMap<>(rmtShuffleStates);
+        }
+    }
+
     /**
      * Unsafe value.
      */
@@ -406,38 +548,50 @@ private boolean tryAdd(long valPtr, int valSize) {
      * @param newBufMinSize Min new buffer size.
      */
     private void send(final int idx, int newBufMinSize) {
-        final GridFutureAdapter<?> fut = new GridFutureAdapter<>();
-
         HadoopShuffleMessage msg = msgs[idx];
 
         final long msgId = msg.id();
 
-        IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>> old = sentMsgs.putIfAbsent(msgId,
-            new IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>>(msg, fut));
+        final GridFutureAdapter<?> fut;
+
+        if (embedded)
+            fut = null;
+        else {
+            fut = new GridFutureAdapter<>();
+
+            IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>> old = sentMsgs.putIfAbsent(msgId,
+                new IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>>(msg, fut));
 
-        assert old == null;
+            assert old == null;
+        }
 
         try {
             io.apply(reduceAddrs[idx], msg);
+
+            if (embedded)
+                remoteShuffleState(reduceAddrs[idx]).onShuffleMessage();
         }
         catch (GridClosureException e) {
-            fut.onDone(U.unwrap(e));
+            if (fut != null)
+                fut.onDone(U.unwrap(e));
         }
 
-        fut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
-            @Override public void apply(IgniteInternalFuture<?> f) {
-                try {
-                    f.get();
+        if (fut != null) {
+            fut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
+                @Override public void apply(IgniteInternalFuture<?> f) {
+                    try {
+                        f.get();
 
-                    // Clean up the future from map only if there was no exception.
-                    // Otherwise flush() should fail.
-                    sentMsgs.remove(msgId);
-                }
-                catch (IgniteCheckedException e) {
-                    log.error("Failed to send message.", e);
+                        // Clean up the future from map only if there was no exception.
+                        // Otherwise flush() should fail.
+                        sentMsgs.remove(msgId);
+                    }
+                    catch (IgniteCheckedException e) {
+                        log.error("Failed to send message.", e);
+                    }
                 }
-            }
-        });
+            });
+        }
 
         msgs[idx] = newBufMinSize == 0 ? null : new HadoopShuffleMessage(job.id(), idx,
             Math.max(msgSize, newBufMinSize));
@@ -513,14 +667,41 @@ public IgniteInternalFuture<?> flush() throws IgniteCheckedException {
 
         GridCompoundFuture fut = new GridCompoundFuture<>();
 
-        for (IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>> tup : sentMsgs.values())
-            fut.add(tup.get2());
+        if (embedded) {
+            boolean sent = false;
 
-        fut.markInitialized();
+            for (Map.Entry<T, HadoopShuffleRemoteState> rmtStateEntry : remoteShuffleStates().entrySet()) {
+                T dest = rmtStateEntry.getKey();
+                HadoopShuffleRemoteState rmtState = rmtStateEntry.getValue();
 
-        if (log.isDebugEnabled())
-            log.debug("Collected futures to compound futures for flush: " + sentMsgs.size());
+                HadoopShuffleFinishRequest req = new HadoopShuffleFinishRequest(job.id(), rmtState.messageCount());
+
+                io.apply(dest, req);
 
+                if (log.isDebugEnabled())
+                    log.debug("Sent shuffle finish request [jobId=" + job.id() + ", dest=" + dest +
+                        ", req=" + req + ']');
+
+                fut.add(rmtState.future());
+
+                sent = true;
+            }
+
+            if (sent)
+                fut.markInitialized();
+            else
+                return new GridFinishedFuture<>();
+        }
+        else {
+            for (IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>> tup : sentMsgs.values())
+                fut.add(tup.get2());
+
+            fut.markInitialized();
+
+            if (log.isDebugEnabled())
+                log.debug("Collected futures to compound futures for flush: " + sentMsgs.size());
+
+        }
         return fut;
     }
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleLocalState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleLocalState.java
new file mode 100644
index 0000000000000..68c0653b2e8b2
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleLocalState.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Local shuffle state.
+ */
+class HadoopShuffleLocalState {
+    /** Message counter. */
+    private final AtomicLong msgCnt = new AtomicLong();
+
+    /** Reply guard. */
+    private final AtomicBoolean replyGuard = new AtomicBoolean();
+
+    /** Total message count.*/
+    private volatile long totalMsgCnt;
+
+    /**
+     * Callback invoked when shuffle message arrived.
+     *
+     * @return Whether to perform reply.
+     */
+    public boolean onShuffleMessage() {
+        long msgCnt0 = msgCnt.incrementAndGet();
+
+        return msgCnt0 == totalMsgCnt && reserve();
+    }
+
+    /**
+     * Callback invoked when shuffle is finished.
+     *
+     * @param totalMsgCnt Message count.
+     * @return Whether to perform reply.
+     */
+    public boolean onShuffleFinishMessage(long totalMsgCnt) {
+        this.totalMsgCnt = totalMsgCnt;
+
+        return msgCnt.get() == totalMsgCnt && reserve();
+    }
+
+    /**
+     * Reserve reply.
+     *
+     * @return {@code True} if reserved.
+     */
+    private boolean reserve() {
+        return replyGuard.compareAndSet(false, true);
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
new file mode 100644
index 0000000000000..5ffaa555c65db
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.lang.IgniteInClosure;
+
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Remote shuffle state.
+ */
+class HadoopShuffleRemoteState<T> {
+    /** Message count. */
+    private final AtomicLong msgCnt = new AtomicLong();
+
+    /** Completion future. */
+    private final GridFutureAdapter fut = new GridFutureAdapter();
+
+    /**
+     * Callback invoked when shuffle message is sent.
+     */
+    public void onShuffleMessage() {
+        msgCnt.incrementAndGet();
+    }
+
+    /**
+     * Callback invoked on shuffle finish response.
+     */
+    public void onShuffleFinishResponse() {
+        fut.onDone();
+    }
+
+    /**
+     * @return Message count.
+     */
+    public long messageCount() {
+        return msgCnt.get();
+    }
+
+    /**
+     * @return Completion future.
+     */
+    public GridFutureAdapter future() {
+        return fut;
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
index 7001b8ca24177..cb08c0061ed03 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
@@ -151,7 +151,7 @@ private void prepareProcess(HadoopPrepareForJobRequest req) {
                 job.initialize(true, nodeDesc.processId());
 
                 shuffleJob = new HadoopShuffleJob<>(comm.localProcessDescriptor(), log, job, mem,
-                    req.totalReducerCount(), req.localReducers());
+                    req.totalReducerCount(), req.localReducers(), false);
 
                 initializeExecutors(req);
 
@@ -432,9 +432,7 @@ else if (msg instanceof HadoopShuffleMessage) {
                         try {
                             HadoopShuffleMessage m = (HadoopShuffleMessage)msg;
 
-                            shuffleJob.onShuffleMessage(m);
-
-                            comm.sendMessage(desc, new HadoopShuffleAck(m.id(), m.jobId()));
+                            shuffleJob.onShuffleMessage(desc, m);
                         }
                         catch (IgniteCheckedException e) {
                             U.error(log, "Failed to process hadoop shuffle message [desc=" + desc + ", msg=" + msg + ']', e);

From a976c42590f786ff977999736be5436e7dac9d87 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Fri, 9 Dec 2016 12:01:40 +0300
Subject: [PATCH 047/446] IGNITE-4270: Hadoop: implemented striped mapper
 output. This closes #1334.

---
 .../communication/GridIoMessageFactory.java   |   6 +
 .../processors/hadoop/HadoopJobProperty.java  |   7 +
 .../hadoop/HadoopMapperAwareTaskOutput.java   |  32 ++
 .../processors/hadoop/HadoopTaskInfo.java     |  43 ++
 .../shuffle/HadoopDirectShuffleMessage.java   | 243 +++++++++++
 .../processors/hadoop/HadoopMapperUtils.java  |  56 +++
 .../hadoop/impl/v2/HadoopV2Context.java       |  11 +
 .../hadoop/impl/v2/HadoopV2MapTask.java       |  10 +
 .../hadoop/jobtracker/HadoopJobTracker.java   |   4 +
 .../hadoop/shuffle/HadoopShuffle.java         |  23 +-
 .../hadoop/shuffle/HadoopShuffleJob.java      | 389 +++++++++++++-----
 .../shuffle/HadoopShuffleRemoteState.java     |   5 +-
 .../shuffle/direct/HadoopDirectDataInput.java | 166 ++++++++
 .../direct/HadoopDirectDataOutput.java        | 221 ++++++++++
 .../direct/HadoopDirectDataOutputContext.java | 100 +++++
 .../direct/HadoopDirectDataOutputState.java   |  54 +++
 .../child/HadoopChildProcessRunner.java       |   2 +-
 .../impl/HadoopMapReduceEmbeddedSelfTest.java |  22 +-
 18 files changed, 1287 insertions(+), 107 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperAwareTaskOutput.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperUtils.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 4ffb22087fcba..504e6830eac39 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -122,6 +122,7 @@
 import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishRequest;
 import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishResponse;
 import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleMessage;
+import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopDirectShuffleMessage;
 import org.apache.ignite.internal.processors.igfs.IgfsAckMessage;
 import org.apache.ignite.internal.processors.igfs.IgfsBlockKey;
 import org.apache.ignite.internal.processors.igfs.IgfsBlocksMessage;
@@ -170,6 +171,11 @@ public GridIoMessageFactory(MessageFactory[] ext) {
         Message msg = null;
 
         switch (type) {
+            case -42:
+                msg = new HadoopDirectShuffleMessage();
+
+                break;
+
             case -41:
                 msg = new HadoopShuffleFinishResponse();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index e713caa626501..1f0ef1b037213 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -90,6 +90,13 @@ public enum HadoopJobProperty {
      */
     SHUFFLE_MSG_SIZE("ignite.shuffle.message.size"),
 
+    /**
+     * Whether to stripe mapper output for remote reducers.
+     * <p>
+     * Defaults to {@code false}.
+     */
+    SHUFFLE_MAPPER_STRIPED_OUTPUT("ignite.shuffle.mapper.striped.output"),
+
     /**
      * Shuffle job throttle in milliseconds. When job is executed with separate shuffle thread, this parameter
      * controls sleep duration between iterations through intermediate reducer maps.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperAwareTaskOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperAwareTaskOutput.java
new file mode 100644
index 0000000000000..1d6637cb1ac21
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperAwareTaskOutput.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Special output type with callback invoked when mapper finished writing data.
+ */
+public interface HadoopMapperAwareTaskOutput extends HadoopTaskOutput {
+    /**
+     * Callback invoked when mapper finished writing data.
+     *
+     * @throws IgniteCheckedException If failed.
+     */
+    public void onMapperFinished() throws IgniteCheckedException;
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java
index b76fb854f8f7c..3509367574143 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java
@@ -46,6 +46,12 @@ public class HadoopTaskInfo implements Externalizable {
     /** */
     private HadoopInputSplit inputSplit;
 
+    /** Whether mapper index is set. */
+    private boolean mapperIdxSet;
+
+    /** Current mapper index. */
+    private int mapperIdx;
+
     /**
      * For {@link Externalizable}.
      */
@@ -78,6 +84,13 @@ public HadoopTaskInfo(HadoopTaskType type, HadoopJobId jobId, int taskNum, int a
         out.writeInt(taskNum);
         out.writeInt(attempt);
         out.writeObject(inputSplit);
+
+        if (mapperIdxSet) {
+            out.writeBoolean(true);
+            out.writeInt(mapperIdx);
+        }
+        else
+            out.writeBoolean(false);
     }
 
     /** {@inheritDoc} */
@@ -87,6 +100,13 @@ public HadoopTaskInfo(HadoopTaskType type, HadoopJobId jobId, int taskNum, int a
         taskNum = in.readInt();
         attempt = in.readInt();
         inputSplit = (HadoopInputSplit)in.readObject();
+
+        if (in.readBoolean()) {
+            mapperIdxSet = true;
+            mapperIdx = in.readInt();
+        }
+        else
+            mapperIdxSet = false;
     }
 
     /**
@@ -117,6 +137,29 @@ public int attempt() {
         return attempt;
     }
 
+    /**
+     * @param mapperIdx Current mapper index.
+     */
+    public void mapperIndex(int mapperIdx) {
+        this.mapperIdx = mapperIdx;
+
+        mapperIdxSet = true;
+    }
+
+    /**
+     * @return Current mapper index or {@code null}
+     */
+    public int mapperIndex() {
+        return mapperIdx;
+    }
+
+    /**
+     * @return {@code True} if mapped index is set.
+     */
+    public boolean hasMapperIndex() {
+        return mapperIdxSet;
+    }
+
     /**
      * @return Input split.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
new file mode 100644
index 0000000000000..e81dc5f2fa9d0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+
+/**
+ * Direct shuffle message.
+ */
+public class HadoopDirectShuffleMessage implements Message, HadoopMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    @GridToStringInclude
+    private HadoopJobId jobId;
+
+    /** */
+    @GridToStringInclude
+    private int reducer;
+
+    /** Count. */
+    private int cnt;
+
+    /** Buffer. */
+    private byte[] buf;
+
+    /** Buffer length (equal or less than buf.length). */
+    @GridDirectTransient
+    private transient int bufLen;
+
+    /**
+     * Default constructor.
+     */
+    public HadoopDirectShuffleMessage() {
+        // No-op.
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param jobId Job ID.
+     * @param reducer Reducer.
+     * @param cnt Count.
+     * @param buf Buffer.
+     * @param bufLen Buffer length.
+     */
+    public HadoopDirectShuffleMessage(HadoopJobId jobId, int reducer, int cnt, byte[] buf, int bufLen) {
+        assert jobId != null;
+
+        this.jobId = jobId;
+        this.reducer = reducer;
+        this.cnt = cnt;
+        this.buf = buf;
+        this.bufLen = bufLen;
+    }
+
+    /**
+     * @return Job ID.
+     */
+    public HadoopJobId jobId() {
+        return jobId;
+    }
+
+    /**
+     * @return Reducer.
+     */
+    public int reducer() {
+        return reducer;
+    }
+
+    /**
+     * @return Count.
+     */
+    public int count() {
+        return cnt;
+    }
+
+    /**
+     * @return Buffer.
+     */
+    public byte[] buffer() {
+        return buf;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeMessage("jobId", jobId))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeInt("reducer", reducer))
+                    return false;
+
+                writer.incrementState();
+
+            case 2:
+                if (!writer.writeInt("cnt", cnt))
+                    return false;
+
+                writer.incrementState();
+
+            case 3:
+                if (!writer.writeByteArray("buf", this.buf, 0, bufLen))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                jobId = reader.readMessage("jobId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                reducer = reader.readInt("reducer");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 2:
+                cnt = reader.readInt("cnt");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 3:
+                this.buf = reader.readByteArray("buf");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                bufLen = this.buf != null ? this.buf.length : 0;
+
+                reader.incrementState();
+
+        }
+
+        return reader.afterMessageRead(HadoopDirectShuffleMessage.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte directType() {
+        return -42;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 4;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeExternal(ObjectOutput out) throws IOException {
+        jobId.writeExternal(out);
+
+        out.writeInt(reducer);
+        out.writeInt(cnt);
+
+        U.writeByteArray(out, buf);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        jobId = new HadoopJobId();
+        jobId.readExternal(in);
+
+        reducer = in.readInt();
+        cnt = in.readInt();
+
+        buf = U.readByteArray(in);
+        bufLen = buf != null ? buf.length : 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HadoopDirectShuffleMessage.class, this);
+    }
+}
\ No newline at end of file
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperUtils.java
new file mode 100644
index 0000000000000..87adcb7ec8cde
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperUtils.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Set of mapper utility methods.
+ */
+public class HadoopMapperUtils {
+    /** Thread-local mapper index. */
+    private static final ThreadLocal<Integer> MAP_IDX = new ThreadLocal<>();
+
+    /**
+     * @return Current mapper index.
+     */
+    public static int mapperIndex() {
+        Integer res = MAP_IDX.get();
+
+        return res != null ? res : -1;
+    }
+
+    /**
+     * @param idx Current mapper index.
+     */
+    public static void mapperIndex(Integer idx) {
+        MAP_IDX.set(idx);
+    }
+
+    /**
+     * Clear mapper index.
+     */
+    public static void clearMapperIndex() {
+        MAP_IDX.remove();
+    }
+
+    /**
+     * Constructor.
+     */
+    private HadoopMapperUtils() {
+        // No-op.
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java
index 90a1bad10a97c..eec0636ce7853 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
 import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperAwareTaskOutput;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskCancelledException;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInput;
@@ -153,6 +154,16 @@ public HadoopV2Context(HadoopV2TaskContext ctx) {
         }
     }
 
+    /**
+     * Callback invoked from mapper thread when map is finished.
+     *
+     * @throws IgniteCheckedException If failed.
+     */
+    public void onMapperFinished() throws IgniteCheckedException {
+        if (output instanceof HadoopMapperAwareTaskOutput)
+            ((HadoopMapperAwareTaskOutput)output).onMapperFinished();
+    }
+
     /** {@inheritDoc} */
     @Override public OutputCommitter getOutputCommitter() {
         throw new UnsupportedOperationException();
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java
index 418df4eb759f4..eb3b935bd3d8b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobInfo;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperUtils;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
 
 /**
@@ -49,6 +50,11 @@ public HadoopV2MapTask(HadoopTaskInfo taskInfo) {
 
         JobContextImpl jobCtx = taskCtx.jobContext();
 
+        if (taskCtx.taskInfo().hasMapperIndex())
+            HadoopMapperUtils.mapperIndex(taskCtx.taskInfo().mapperIndex());
+        else
+            HadoopMapperUtils.clearMapperIndex();
+
         try {
             InputSplit nativeSplit = hadoopContext().getInputSplit();
 
@@ -72,6 +78,8 @@ public HadoopV2MapTask(HadoopTaskInfo taskInfo) {
 
             try {
                 mapper.run(new WrappedMapper().getMapContext(hadoopContext()));
+
+                hadoopContext().onMapperFinished();
             }
             finally {
                 closeWriter();
@@ -92,6 +100,8 @@ public HadoopV2MapTask(HadoopTaskInfo taskInfo) {
             throw new IgniteCheckedException(e);
         }
         finally {
+            HadoopMapperUtils.clearMapperIndex();
+
             if (err != null)
                 abort(outputFormat);
         }
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
index 36782bf2febe0..a7255341e1ebd 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
@@ -1018,6 +1018,8 @@ private Collection<HadoopTaskInfo> mapperTasks(Iterable<HadoopInputSplit> mapper
             if (state == null)
                 state = initState(jobId);
 
+            int mapperIdx = 0;
+
             for (HadoopInputSplit split : mappers) {
                 if (state.addMapper(split)) {
                     if (log.isDebugEnabled())
@@ -1026,6 +1028,8 @@ private Collection<HadoopTaskInfo> mapperTasks(Iterable<HadoopInputSplit> mapper
 
                     HadoopTaskInfo taskInfo = new HadoopTaskInfo(MAP, jobId, meta.taskNumber(split), 0, split);
 
+                    taskInfo.mapperIndex(mapperIdx++);
+
                     if (tasks == null)
                         tasks = new ArrayList<>();
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
index 82bbd32f0cd5b..8ffea8c75075d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.processors.hadoop.HadoopComponent;
 import org.apache.ignite.internal.processors.hadoop.HadoopContext;
+import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
 import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
@@ -39,6 +40,7 @@
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 
+import java.util.Collection;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -102,8 +104,8 @@ public class HadoopShuffle extends HadoopComponent {
     private HadoopShuffleJob<UUID> newJob(HadoopJobId jobId) throws IgniteCheckedException {
         HadoopMapReducePlan plan = ctx.jobTracker().plan(jobId);
 
-        HadoopShuffleJob<UUID> job = new HadoopShuffleJob<>(ctx.localNodeId(), log,
-            ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()), true);
+        HadoopShuffleJob<UUID> job = new HadoopShuffleJob<>(ctx.localNodeId(), log, ctx.jobTracker().job(jobId, null),
+            mem, plan.reducers(), plan.reducers(ctx.localNodeId()), localMappersCount(plan), true);
 
         UUID[] rdcAddrs = new UUID[plan.reducers()];
 
@@ -122,6 +124,18 @@ private HadoopShuffleJob<UUID> newJob(HadoopJobId jobId) throws IgniteCheckedExc
         return job;
     }
 
+    /**
+     * Get number of local mappers.
+     *
+     * @param plan Plan.
+     * @return Number of local mappers.
+     */
+    private int localMappersCount(HadoopMapReducePlan plan) {
+        Collection<HadoopInputSplit> locMappers = plan.mappers(ctx.localNodeId());
+
+        return F.isEmpty(locMappers) ? 0 : locMappers.size();
+    }
+
     /**
      * @param nodeId Node ID to send message to.
      * @param msg Message to send.
@@ -195,6 +209,11 @@ public boolean onMessageReceived(UUID src, HadoopMessage msg) {
 
                 job(m.jobId()).onShuffleMessage(src, m);
             }
+            else if (msg instanceof HadoopDirectShuffleMessage) {
+                HadoopDirectShuffleMessage m = (HadoopDirectShuffleMessage)msg;
+
+                job(m.jobId()).onDirectShuffleMessage(src, m);
+            }
             else if (msg instanceof HadoopShuffleAck) {
                 HadoopShuffleAck m = (HadoopShuffleAck)msg;
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 0a3a0ae3aa118..214a335432df6 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -17,20 +17,16 @@
 
 package org.apache.ignite.internal.processors.hadoop.shuffle;
 
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.hadoop.HadoopJob;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperAwareTaskOutput;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperUtils;
 import org.apache.ignite.internal.processors.hadoop.HadoopPartitioner;
+import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInput;
@@ -41,6 +37,9 @@
 import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopConcurrentHashMultimap;
 import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap;
 import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopSkipList;
+import org.apache.ignite.internal.processors.hadoop.shuffle.direct.HadoopDirectDataInput;
+import org.apache.ignite.internal.processors.hadoop.shuffle.direct.HadoopDirectDataOutputContext;
+import org.apache.ignite.internal.processors.hadoop.shuffle.direct.HadoopDirectDataOutputState;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -55,9 +54,19 @@
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.thread.IgniteThread;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReferenceArray;
 
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.PARTITION_HASHMAP_SIZE;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_JOB_THROTTLE;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MSG_SIZE;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_REDUCER_NO_SORTING;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.get;
@@ -121,6 +130,9 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
     /** Message size. */
     private final int msgSize;
 
+    /** Whether to strip mappers for remote execution. */
+    private final boolean stripeMappers;
+
     /** Local shuffle states. */
     private volatile HashMap<T, HadoopShuffleLocalState> locShuffleStates = new HashMap<>();
 
@@ -143,11 +155,12 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
      * @param mem Memory.
      * @param totalReducerCnt Amount of reducers in the Job.
      * @param locReducers Reducers will work on current node.
+     * @param locMappersCnt Number of mappers running on the given node.
      * @param embedded Whether shuffle is running in embedded mode.
      * @throws IgniteCheckedException If error.
      */
     public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUnsafeMemory mem,
-        int totalReducerCnt, int[] locReducers, boolean embedded) throws IgniteCheckedException {
+        int totalReducerCnt, int[] locReducers, int locMappersCnt, boolean embedded) throws IgniteCheckedException {
         this.locReduceAddr = locReduceAddr;
         this.totalReducerCnt = totalReducerCnt;
         this.job = job;
@@ -155,6 +168,27 @@ public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUn
         this.log = log.getLogger(HadoopShuffleJob.class);
         this.embedded = embedded;
 
+        // No stripes for combiner.
+        boolean stripeMappers0 = get(job.info(), SHUFFLE_MAPPER_STRIPED_OUTPUT, false);
+
+        if (stripeMappers0) {
+            if (job.info().hasCombiner()) {
+                log.info("Striped mapper output is disabled because it cannot be used together with combiner [jobId=" +
+                    job.id() + ']');
+
+                stripeMappers0 = false;
+            }
+
+            if (!embedded) {
+                log.info("Striped mapper output is disabled becuase it cannot be used in external mode [jobId=" +
+                    job.id() + ']');
+
+                stripeMappers0 = false;
+            }
+        }
+
+        stripeMappers = stripeMappers0;
+
         msgSize = get(job.info(), SHUFFLE_MSG_SIZE, DFLT_SHUFFLE_MSG_SIZE);
 
         locReducersCtx = new AtomicReferenceArray<>(totalReducerCnt);
@@ -169,9 +203,20 @@ public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUn
 
         needPartitioner = totalReducerCnt > 1;
 
+        // Size of local map is always equal to total reducer number to allow index-based lookup.
         locMaps = new AtomicReferenceArray<>(totalReducerCnt);
-        rmtMaps = new AtomicReferenceArray<>(totalReducerCnt);
-        msgs = new HadoopShuffleMessage[totalReducerCnt];
+
+        // Size of remote map:
+        // - If there are no local mappers, then we will not send anything, so set to 0;
+        // - If output is not striped, then match it to total reducer count, the same way as for local maps.
+        // - If output is striped, then multiply previous value by number of local mappers.
+        int rmtMapsSize = locMappersCnt == 0 ? 0 : totalReducerCnt;
+
+        if (stripeMappers)
+            rmtMapsSize *= locMappersCnt;
+
+        rmtMaps = new AtomicReferenceArray<>(rmtMapsSize);
+        msgs = new HadoopShuffleMessage[rmtMapsSize];
 
         throttle = get(job.info(), SHUFFLE_JOB_THROTTLE, 0);
     }
@@ -208,24 +253,26 @@ public void startSending(String gridName, IgniteInClosure2X<T, HadoopMessage> io
 
         this.io = io;
 
-        if (!flushed) {
-            snd = new GridWorker(gridName, "hadoop-shuffle-" + job.id(), log) {
-                @Override protected void body() throws InterruptedException {
-                    try {
-                        while (!isCancelled()) {
-                            if (throttle > 0)
-                                Thread.sleep(throttle);
-
-                            collectUpdatesAndSend(false);
+        if (!stripeMappers) {
+            if (!flushed) {
+                snd = new GridWorker(gridName, "hadoop-shuffle-" + job.id(), log) {
+                    @Override protected void body() throws InterruptedException {
+                        try {
+                            while (!isCancelled()) {
+                                if (throttle > 0)
+                                    Thread.sleep(throttle);
+
+                                collectUpdatesAndSend(false);
+                            }
+                        }
+                        catch (IgniteCheckedException e) {
+                            throw new IllegalStateException(e);
                         }
                     }
-                    catch (IgniteCheckedException e) {
-                        throw new IllegalStateException(e);
-                    }
-                }
-            };
+                };
 
-            new IgniteThread(snd).start();
+                new IgniteThread(snd).start();
+            }
         }
 
         ioInitLatch.countDown();
@@ -305,6 +352,46 @@ public void onShuffleMessage(T src, HadoopShuffleMessage msg) throws IgniteCheck
             io.apply(src, new HadoopShuffleAck(msg.id(), msg.jobId()));
     }
 
+    /**
+     * Process shuffle message.
+     *
+     * @param src Source.
+     * @param msg Message.
+     * @throws IgniteCheckedException Exception.
+     */
+    public void onDirectShuffleMessage(T src, HadoopDirectShuffleMessage msg) throws IgniteCheckedException {
+        assert msg.buffer() != null;
+
+        HadoopTaskContext taskCtx = locReducersCtx.get(msg.reducer()).get();
+
+        HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
+
+        perfCntr.onShuffleMessage(msg.reducer(), U.currentTimeMillis());
+
+        HadoopMultimap map = getOrCreateMap(locMaps, msg.reducer());
+
+        HadoopSerialization keySer = taskCtx.keySerialization();
+        HadoopSerialization valSer = taskCtx.valueSerialization();
+
+        // Add data from message to the map.
+        try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
+            HadoopDirectDataInput in = new HadoopDirectDataInput(msg.buffer());
+
+            Object key = null;
+            Object val = null;
+
+            for (int i = 0; i < msg.count(); i++) {
+                key = keySer.read(in, key);
+                val = valSer.read(in, val);
+
+                adder.write(key, val);
+            }
+        }
+
+        if (localShuffleState(src).onShuffleMessage())
+            sendFinishResponse(src, msg.jobId());
+    }
+
     /**
      * @param ack Shuffle ack.
      */
@@ -467,88 +554,149 @@ private UnsafeValue(byte[] buf) {
     }
 
     /**
-     * Sends map updates to remote reducers.
+     * Send updates to remote reducers.
+     *
+     * @param flush Flush flag.
+     * @throws IgniteCheckedException If failed.
      */
     private void collectUpdatesAndSend(boolean flush) throws IgniteCheckedException {
-        for (int i = 0; i < rmtMaps.length(); i++) {
-            HadoopMultimap map = rmtMaps.get(i);
+        for (int i = 0; i < rmtMaps.length(); i++)
+            collectUpdatesAndSend(i, flush);
+    }
+
+    /**
+     * Send updates to concrete remote reducer.
+     *
+     * @param rmtMapIdx Remote map index.
+     * @param flush Flush flag.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void collectUpdatesAndSend(int rmtMapIdx, boolean flush) throws IgniteCheckedException {
+        final int rmtRdcIdx = stripeMappers ? rmtMapIdx % totalReducerCnt : rmtMapIdx;
 
-            if (map == null)
-                continue; // Skip empty map and local node.
+        HadoopMultimap map = rmtMaps.get(rmtMapIdx);
 
-            if (msgs[i] == null)
-                msgs[i] = new HadoopShuffleMessage(job.id(), i, msgSize);
+        if (map == null)
+            return;
 
-            final int idx = i;
+        if (msgs[rmtMapIdx] == null)
+            msgs[rmtMapIdx] = new HadoopShuffleMessage(job.id(), rmtRdcIdx, msgSize);
 
-            map.visit(false, new HadoopMultimap.Visitor() {
-                /** */
-                private long keyPtr;
+        visit(map, rmtMapIdx, rmtRdcIdx);
 
-                /** */
-                private int keySize;
+        if (flush && msgs[rmtMapIdx].offset() != 0)
+            send(rmtMapIdx, rmtRdcIdx, 0);
+    }
 
-                /** */
-                private boolean keyAdded;
+    /**
+     * Flush remote direct context.
+     *
+     * @param rmtMapIdx Remote map index.
+     * @param rmtDirectCtx Remote direct context.
+     * @param reset Whether to perform reset.
+     */
+    private void sendShuffleMessage(int rmtMapIdx, @Nullable HadoopDirectDataOutputContext rmtDirectCtx, boolean reset) {
+        if (rmtDirectCtx == null)
+            return;
 
-                /** {@inheritDoc} */
-                @Override public void onKey(long keyPtr, int keySize) {
-                    this.keyPtr = keyPtr;
-                    this.keySize = keySize;
+        int cnt = rmtDirectCtx.count();
 
-                    keyAdded = false;
-                }
+        if (cnt == 0)
+            return;
 
-                private boolean tryAdd(long valPtr, int valSize) {
-                    HadoopShuffleMessage msg = msgs[idx];
+        int rmtRdcIdx = stripeMappers ? rmtMapIdx % totalReducerCnt : rmtMapIdx;
 
-                    if (!keyAdded) { // Add key and value.
-                        int size = keySize + valSize;
+        HadoopDirectDataOutputState state = rmtDirectCtx.state();
 
-                        if (!msg.available(size, false))
-                            return false;
+        if (reset)
+            rmtDirectCtx.reset();
 
-                        msg.addKey(keyPtr, keySize);
-                        msg.addValue(valPtr, valSize);
+        HadoopDirectShuffleMessage msg = new HadoopDirectShuffleMessage(job.id(), rmtRdcIdx, cnt,
+            state.buffer(), state.bufferLength());
 
-                        keyAdded = true;
+        T nodeId = reduceAddrs[rmtRdcIdx];
 
-                        return true;
-                    }
+        io.apply(nodeId, msg);
+
+        remoteShuffleState(nodeId).onShuffleMessage();
+    }
+
+    /**
+     * Visit output map.
+     *
+     * @param map Map.
+     * @param rmtMapIdx Remote map index.
+     * @param rmtRdcIdx Remote reducer index.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void visit(HadoopMultimap map, final int rmtMapIdx, final int rmtRdcIdx) throws IgniteCheckedException {
+        map.visit(false, new HadoopMultimap.Visitor() {
+            /** */
+            private long keyPtr;
 
-                    if (!msg.available(valSize, true))
+            /** */
+            private int keySize;
+
+            /** */
+            private boolean keyAdded;
+
+            /** {@inheritDoc} */
+            @Override public void onKey(long keyPtr, int keySize) {
+                this.keyPtr = keyPtr;
+                this.keySize = keySize;
+
+                keyAdded = false;
+            }
+
+            private boolean tryAdd(long valPtr, int valSize) {
+                HadoopShuffleMessage msg = msgs[rmtMapIdx];
+
+                if (!keyAdded) { // Add key and value.
+                    int size = keySize + valSize;
+
+                    if (!msg.available(size, false))
                         return false;
 
+                    msg.addKey(keyPtr, keySize);
                     msg.addValue(valPtr, valSize);
 
+                    keyAdded = true;
+
                     return true;
                 }
 
-                /** {@inheritDoc} */
-                @Override public void onValue(long valPtr, int valSize) {
-                    if (tryAdd(valPtr, valSize))
-                        return;
+                if (!msg.available(valSize, true))
+                    return false;
 
-                    send(idx, keySize + valSize);
+                msg.addValue(valPtr, valSize);
 
-                    keyAdded = false;
+                return true;
+            }
 
-                    if (!tryAdd(valPtr, valSize))
-                        throw new IllegalStateException();
-                }
-            });
+            /** {@inheritDoc} */
+            @Override public void onValue(long valPtr, int valSize) {
+                if (tryAdd(valPtr, valSize))
+                    return;
 
-            if (flush && msgs[i].offset() != 0)
-                send(i, 0);
-        }
+                send(rmtMapIdx, rmtRdcIdx, keySize + valSize);
+
+                keyAdded = false;
+
+                if (!tryAdd(valPtr, valSize))
+                    throw new IllegalStateException();
+            }
+        });
     }
 
     /**
-     * @param idx Index of message.
+     * Send message.
+     *
+     * @param rmtMapIdx Remote map index.
+     * @param rmtRdcIdx Remote reducer index.
      * @param newBufMinSize Min new buffer size.
      */
-    private void send(final int idx, int newBufMinSize) {
-        HadoopShuffleMessage msg = msgs[idx];
+    private void send(int rmtMapIdx, int rmtRdcIdx, int newBufMinSize) {
+        HadoopShuffleMessage msg = msgs[rmtMapIdx];
 
         final long msgId = msg.id();
 
@@ -566,10 +714,10 @@ private void send(final int idx, int newBufMinSize) {
         }
 
         try {
-            io.apply(reduceAddrs[idx], msg);
+            io.apply(reduceAddrs[rmtRdcIdx], msg);
 
             if (embedded)
-                remoteShuffleState(reduceAddrs[idx]).onShuffleMessage();
+                remoteShuffleState(reduceAddrs[rmtRdcIdx]).onShuffleMessage();
         }
         catch (GridClosureException e) {
             if (fut != null)
@@ -593,7 +741,7 @@ private void send(final int idx, int newBufMinSize) {
             });
         }
 
-        msgs[idx] = newBufMinSize == 0 ? null : new HadoopShuffleMessage(job.id(), idx,
+        msgs[rmtMapIdx] = newBufMinSize == 0 ? null : new HadoopShuffleMessage(job.id(), rmtRdcIdx,
             Math.max(msgSize, newBufMinSize));
     }
 
@@ -639,31 +787,33 @@ public IgniteInternalFuture<?> flush() throws IgniteCheckedException {
         if (totalReducerCnt == 0)
             return new GridFinishedFuture<>();
 
-        U.await(ioInitLatch);
+        if (!stripeMappers) {
+            U.await(ioInitLatch);
 
-        GridWorker snd0 = snd;
+            GridWorker snd0 = snd;
 
-        if (snd0 != null) {
-            if (log.isDebugEnabled())
-                log.debug("Cancelling sender thread.");
+            if (snd0 != null) {
+                if (log.isDebugEnabled())
+                    log.debug("Cancelling sender thread.");
 
-            snd0.cancel();
+                snd0.cancel();
 
-            try {
-                snd0.join();
+                try {
+                    snd0.join();
 
-                if (log.isDebugEnabled())
-                    log.debug("Finished waiting for sending thread to complete on shuffle job flush: " + job.id());
-            }
-            catch (InterruptedException e) {
-                throw new IgniteInterruptedCheckedException(e);
+                    if (log.isDebugEnabled())
+                        log.debug("Finished waiting for sending thread to complete on shuffle job flush: " + job.id());
+                }
+                catch (InterruptedException e) {
+                    throw new IgniteInterruptedCheckedException(e);
+                }
             }
-        }
 
-        collectUpdatesAndSend(true); // With flush.
+            collectUpdatesAndSend(true); // With flush.
 
-        if (log.isDebugEnabled())
-            log.debug("Finished sending collected updates to remote reducers: " + job.id());
+            if (log.isDebugEnabled())
+                log.debug("Finished sending collected updates to remote reducers: " + job.id());
+        }
 
         GridCompoundFuture fut = new GridCompoundFuture<>();
 
@@ -700,8 +850,8 @@ public IgniteInternalFuture<?> flush() throws IgniteCheckedException {
 
             if (log.isDebugEnabled())
                 log.debug("Collected futures to compound futures for flush: " + sentMsgs.size());
-
         }
+
         return fut;
     }
 
@@ -775,13 +925,17 @@ private boolean isLocalPartition(int part) {
     /**
      * Partitioned output.
      */
-    private class PartitionedOutput implements HadoopTaskOutput {
+    public class PartitionedOutput implements HadoopMapperAwareTaskOutput {
         /** */
         private final HadoopTaskOutput[] locAdders = new HadoopTaskOutput[locMaps.length()];
 
         /** */
         private final HadoopTaskOutput[] rmtAdders = new HadoopTaskOutput[rmtMaps.length()];
 
+        /** Remote direct contexts. */
+        private final HadoopDirectDataOutputContext[] rmtDirectCtxs =
+            new HadoopDirectDataOutputContext[rmtMaps.length()];
+
         /** */
         private HadoopPartitioner partitioner;
 
@@ -819,15 +973,52 @@ private PartitionedOutput(HadoopTaskContext taskCtx) throws IgniteCheckedExcepti
                     locAdders[part] = out = getOrCreateMap(locMaps, part).startAdding(taskCtx);
             }
             else {
-                out = rmtAdders[part];
+                if (stripeMappers) {
+                    int mapperIdx = HadoopMapperUtils.mapperIndex();
 
-                if (out == null)
-                    rmtAdders[part] = out = getOrCreateMap(rmtMaps, part).startAdding(taskCtx);
+                    assert mapperIdx >= 0;
+
+                    int idx = totalReducerCnt * mapperIdx + part;
+
+                    HadoopDirectDataOutputContext rmtDirectCtx = rmtDirectCtxs[idx];
+
+                    if (rmtDirectCtx == null) {
+                        rmtDirectCtx = new HadoopDirectDataOutputContext(msgSize, taskCtx);
+
+                        rmtDirectCtxs[idx] = rmtDirectCtx;
+                    }
+
+                    if (rmtDirectCtx.write(key, val))
+                        sendShuffleMessage(idx, rmtDirectCtx, true);
+
+                    return;
+                }
+                else {
+                    out = rmtAdders[part];
+
+                    if (out == null)
+                        rmtAdders[part] = out = getOrCreateMap(rmtMaps, part).startAdding(taskCtx);
+                }
             }
 
             out.write(key, val);
         }
 
+        /** {@inheritDoc} */
+        @Override public void onMapperFinished() throws IgniteCheckedException {
+            if (stripeMappers) {
+                int mapperIdx = HadoopMapperUtils.mapperIndex();
+
+                assert mapperIdx >= 0;
+
+                for (int i = 0; i < totalReducerCnt; i++) {
+                    int idx = totalReducerCnt * mapperIdx + i;
+
+                    sendShuffleMessage(idx, rmtDirectCtxs[idx], false);
+                }
+            }
+        }
+
         /** {@inheritDoc} */
         @Override public void close() throws IgniteCheckedException {
             for (HadoopTaskOutput adder : locAdders) {
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
index 5ffaa555c65db..43311247e1ead 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
@@ -17,17 +17,14 @@
 
 package org.apache.ignite.internal.processors.hadoop.shuffle;
 
-import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.lang.IgniteInClosure;
 
-import java.util.UUID;
 import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * Remote shuffle state.
  */
-class HadoopShuffleRemoteState<T> {
+class HadoopShuffleRemoteState {
     /** Message count. */
     private final AtomicLong msgCnt = new AtomicLong();
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java
new file mode 100644
index 0000000000000..e3a713aa5fe8b
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.direct;
+
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.apache.ignite.internal.util.GridUnsafe.BYTE_ARR_OFF;
+
+/**
+ * Hadoop data input used for direct communication.
+ */
+public class HadoopDirectDataInput extends InputStream implements DataInput {
+    /** Data buffer. */
+    private final byte[] buf;
+
+    /** Position. */
+    private int pos;
+
+    /**
+     * Constructor.
+     *
+     * @param buf Buffer.
+     */
+    public HadoopDirectDataInput(byte[] buf) {
+        this.buf = buf;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int read() throws IOException {
+        return readByte();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readFully(@NotNull byte[] b) throws IOException {
+        readFully(b, 0, b.length);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
+        System.arraycopy(buf, pos, b, off, len);
+
+        pos += len;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int skipBytes(int n) throws IOException {
+        pos += n;
+
+        return n;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readBoolean() throws IOException {
+        return readByte() == 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte readByte() throws IOException {
+        byte res = GridUnsafe.getByte(buf, BYTE_ARR_OFF + pos);
+
+        pos += 1;
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int readUnsignedByte() throws IOException {
+        return readByte() & 0xff;
+    }
+
+    /** {@inheritDoc} */
+    @Override public short readShort() throws IOException {
+        short res = GridUnsafe.getShort(buf, BYTE_ARR_OFF + pos);
+
+        pos += 2;
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int readUnsignedShort() throws IOException {
+        return readShort() & 0xffff;
+    }
+
+    /** {@inheritDoc} */
+    @Override public char readChar() throws IOException {
+        char res = GridUnsafe.getChar(buf, BYTE_ARR_OFF + pos);
+
+        pos += 2;
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int readInt() throws IOException {
+        int res = GridUnsafe.getInt(buf, BYTE_ARR_OFF + pos);
+
+        pos += 4;
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long readLong() throws IOException {
+        long res = GridUnsafe.getLong(buf, BYTE_ARR_OFF + pos);
+
+        pos += 8;
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public float readFloat() throws IOException {
+        float res = GridUnsafe.getFloat(buf, BYTE_ARR_OFF + pos);
+
+        pos += 4;
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public double readDouble() throws IOException {
+        double res = GridUnsafe.getDouble(buf, BYTE_ARR_OFF + pos);
+
+        pos += 8;
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String readLine() throws IOException {
+        // TODO: Create ticket!
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @NotNull @Override public String readUTF() throws IOException {
+        byte[] bytes = new byte[readShort()];
+
+        if (bytes.length != 0)
+            readFully(bytes);
+
+        return new String(bytes, StandardCharsets.UTF_8);
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java
new file mode 100644
index 0000000000000..151e5528092a4
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.direct;
+
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UTFDataFormatException;
+import java.nio.charset.StandardCharsets;
+
+import static org.apache.ignite.internal.util.GridUnsafe.BYTE_ARR_OFF;
+
+/**
+ * Hadoop data output for direct communication.
+ */
+public class HadoopDirectDataOutput extends OutputStream implements DataOutput {
+    /** Flush size. */
+    private final int flushSize;
+
+    /** Data buffer. */
+    private byte[] buf;
+
+    /** Buffer size. */
+    private int bufSize;
+
+    /** Position. */
+    private int pos;
+
+    /**
+     * Constructor.
+     *
+     * @param flushSize Flush size.
+     */
+    public HadoopDirectDataOutput(int flushSize) {
+        this(flushSize, flushSize);
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param flushSize Flush size.
+     * @param allocSize Allocation size.
+     */
+    public HadoopDirectDataOutput(int flushSize, int allocSize) {
+        this.flushSize = flushSize;
+
+        buf = new byte[allocSize];
+        bufSize = allocSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void write(@NotNull byte[] b) throws IOException {
+        write(b, 0, b.length);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void write(@NotNull byte[] b, int off, int len) throws IOException {
+        int writePos = ensure(len);
+
+        System.arraycopy(b, off, buf, writePos, len);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void write(int val) throws IOException {
+        writeByte(val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeBoolean(boolean val) throws IOException {
+        writeByte(val ? (byte)1 : (byte)0);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeByte(int val) throws IOException {
+        int writePos = ensure(1);
+
+        buf[writePos] = (byte)val;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeShort(int val) throws IOException {
+        int writePos = ensure(2);
+
+        GridUnsafe.putShort(buf, BYTE_ARR_OFF + writePos, (short)val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeChar(int val) throws IOException {
+        int writePos = ensure(2);
+
+        GridUnsafe.putChar(buf, BYTE_ARR_OFF + writePos, (char)val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeInt(int val) throws IOException {
+        int writePos = ensure(4);
+
+        GridUnsafe.putInt(buf, BYTE_ARR_OFF + writePos, val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeLong(long val) throws IOException {
+        int writePos = ensure(8);
+
+        GridUnsafe.putLong(buf, BYTE_ARR_OFF + writePos, val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeFloat(float val) throws IOException {
+        int writePos = ensure(4);
+
+        GridUnsafe.putFloat(buf, BYTE_ARR_OFF + writePos, val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeDouble(double val) throws IOException {
+        int writePos = ensure(8);
+
+        GridUnsafe.putDouble(buf, BYTE_ARR_OFF + writePos, val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeBytes(@NotNull String str) throws IOException {
+        for(int i = 0; i < str.length(); ++i)
+            write((byte)str.charAt(i));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeChars(@NotNull String str) throws IOException {
+        for (int i = 0; i < str.length(); ++i)
+            writeChar(str.charAt(i));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeUTF(@NotNull String str) throws IOException {
+        byte[] bytes = str.getBytes(StandardCharsets.UTF_8);
+
+        int len = bytes.length;
+
+        if (len > 65535)
+            throw new UTFDataFormatException("UTF8 form of string is longer than 65535 bytes: " + str);
+
+        writeShort((short)len);
+        write(bytes);
+    }
+
+    /**
+     * @return Buffer.
+     */
+    public byte[] buffer() {
+        return buf;
+    }
+
+    /**
+     * @return Position.
+     */
+    public int position() {
+        return pos;
+    }
+
+    /**
+     * @return Whether buffer is ready for flush.
+     */
+    public boolean readyForFlush() {
+        return pos >= flushSize;
+    }
+
+    /**
+     * Ensure that the given amount of bytes is available within the stream, then shift the position.
+     *
+     * @param cnt Count.
+     * @return Position
+     */
+    private int ensure(int cnt) {
+        int pos0 = pos;
+
+        if (pos0 + cnt > bufSize)
+            grow(pos0 + cnt);
+
+        pos += cnt;
+
+        return pos0;
+    }
+
+    /**
+     * Grow array up to the given count.
+     *
+     * @param cnt Count.
+     */
+    private void grow(int cnt) {
+        int bufSize0 = (int)(bufSize * 1.1);
+
+        if (bufSize0 < cnt)
+            bufSize0 = cnt;
+
+        byte[] buf0 = new byte[bufSize0];
+
+        System.arraycopy(buf, 0, buf0, 0, pos);
+
+        buf = buf0;
+        bufSize = bufSize0;
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java
new file mode 100644
index 0000000000000..bc70ef3dcebd7
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.direct;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
+import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
+
+/**
+ * Hadoop data output context for direct communication.
+ */
+public class HadoopDirectDataOutputContext {
+    /** Flush size. */
+    private final int flushSize;
+
+    /** Key serialization. */
+    private final HadoopSerialization keySer;
+
+    /** Value serialization. */
+    private final HadoopSerialization valSer;
+
+    /** Data output. */
+    private HadoopDirectDataOutput out;
+
+    /** Number of keys written. */
+    private int cnt;
+
+    /**
+     * Constructor.
+     *
+     * @param flushSize Flush size.
+     * @param taskCtx Task context.
+     * @throws IgniteCheckedException If failed.
+     */
+    public HadoopDirectDataOutputContext(int flushSize, HadoopTaskContext taskCtx)
+        throws IgniteCheckedException {
+        this.flushSize = flushSize;
+
+        keySer = taskCtx.keySerialization();
+        valSer = taskCtx.valueSerialization();
+
+        out = new HadoopDirectDataOutput(flushSize);
+    }
+
+    /**
+     * Write key-value pair.
+     *
+     * @param key Key.
+     * @param val Value.
+     * @return Whether flush is needed.
+     * @throws IgniteCheckedException If failed.
+     */
+    public boolean write(Object key, Object val) throws IgniteCheckedException {
+        keySer.write(out, key);
+        valSer.write(out, val);
+
+        cnt++;
+
+        return out.readyForFlush();
+    }
+
+    /**
+     * @return Key-value pairs count.
+     */
+    public int count() {
+        return cnt;
+    }
+
+    /**
+     * @return State.
+     */
+    public HadoopDirectDataOutputState state() {
+        return new HadoopDirectDataOutputState(out.buffer(), out.position());
+    }
+
+    /**
+     * Reset buffer.
+     */
+    public void reset() {
+        int allocSize = Math.max(flushSize, out.position());
+
+        out = new HadoopDirectDataOutput(flushSize, allocSize);
+        cnt = 0;
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java
new file mode 100644
index 0000000000000..a9c12e38b10fe
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.direct;
+
+/**
+ * Hadoop data output state for direct communication.
+ */
+public class HadoopDirectDataOutputState {
+    /** Buffer. */
+    private final byte[] buf;
+
+    /** Buffer length. */
+    private final int bufLen;
+
+    /**
+     * Constructor.
+     *
+     * @param buf Buffer.
+     * @param bufLen Buffer length.
+     */
+    public HadoopDirectDataOutputState(byte[] buf, int bufLen) {
+        this.buf = buf;
+        this.bufLen = bufLen;
+    }
+
+    /**
+     * @return Buffer.
+     */
+    public byte[] buffer() {
+        return buf;
+    }
+
+    /**
+     * @return Length.
+     */
+    public int bufferLength() {
+        return bufLen;
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
index cb08c0061ed03..3336120d2ebfd 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
@@ -151,7 +151,7 @@ private void prepareProcess(HadoopPrepareForJobRequest req) {
                 job.initialize(true, nodeDesc.processId());
 
                 shuffleJob = new HadoopShuffleJob<>(comm.localProcessDescriptor(), log, job, mem,
-                    req.totalReducerCount(), req.localReducers(), false);
+                    req.totalReducerCount(), req.localReducers(), 0, false);
 
                 initializeExecutors(req);
 
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
index b04deeb8e4017..8897a38148a68 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.igfs.IgfsPath;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty;
 import org.apache.ignite.internal.processors.hadoop.impl.examples.HadoopWordCount1;
 import org.apache.ignite.internal.processors.hadoop.impl.examples.HadoopWordCount2;
 
@@ -54,12 +55,28 @@ public class HadoopMapReduceEmbeddedSelfTest extends HadoopMapReduceTest {
         return cfg;
     }
 
+    /*
+     * @throws Exception If fails.
+     */
+    public void testMultiReducerWholeMapReduceExecution() throws Exception {
+        checkMultiReducerWholeMapReduceExecution(false);
+    }
+
+    /*
+     * @throws Exception If fails.
+     */
+    public void testMultiReducerWholeMapReduceExecutionStriped() throws Exception {
+        checkMultiReducerWholeMapReduceExecution(true);
+    }
+
     /**
      * Tests whole job execution with all phases in old and new versions of API with definition of custom
      * Serialization, Partitioner and IO formats.
+     *
+     * @param striped Whether output should be striped or not.
      * @throws Exception If fails.
      */
-    public void testMultiReducerWholeMapReduceExecution() throws Exception {
+    public void checkMultiReducerWholeMapReduceExecution(boolean striped) throws Exception {
         IgfsPath inDir = new IgfsPath(PATH_INPUT);
 
         igfs.mkdirs(inDir);
@@ -81,6 +98,9 @@ public void testMultiReducerWholeMapReduceExecution() throws Exception {
 
             JobConf jobConf = new JobConf();
 
+            if (striped)
+                jobConf.set(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), "true");
+
             jobConf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, CustomSerialization.class.getName());
 
             //To split into about 6-7 items for v2

From ffe53eb5a59908db3684ce11474cb875c4bf392d Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Mon, 12 Dec 2016 11:29:23 +0300
Subject: [PATCH 048/446] IGNITE-4386: Hadoop: implemented client cleanup on
 protocol close. This closes #1327. This closes #1339.

---
 .../IgniteHadoopClientProtocolProvider.java   |  70 ++----
 .../impl/proto/HadoopClientProtocol.java      |  55 +++--
 .../hadoop/mapreduce/MapReduceClient.java     | 147 +++++++++++
 ...ClientProtocolMultipleServersSelfTest.java |  93 +++----
 .../client/HadoopClientProtocolSelfTest.java  | 228 ++++++++++--------
 5 files changed, 367 insertions(+), 226 deletions(-)
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/mapreduce/MapReduceClient.java

diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java
index 1efe6251d640d..920e8b7641468 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java
@@ -23,24 +23,16 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.concurrent.ConcurrentHashMap;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
 import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
-import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.ConnectorConfiguration;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.client.GridClient;
-import org.apache.ignite.internal.client.GridClientConfiguration;
-import org.apache.ignite.internal.client.GridClientException;
-import org.apache.ignite.internal.client.GridClientFactory;
-import org.apache.ignite.internal.client.marshaller.jdk.GridClientJdkMarshaller;
 import org.apache.ignite.internal.processors.hadoop.impl.proto.HadoopClientProtocol;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.processors.hadoop.mapreduce.MapReduceClient;
 import org.apache.ignite.internal.util.typedef.F;
 
-import static org.apache.ignite.internal.client.GridClientProtocol.TCP;
-
 
 /**
  * Ignite Hadoop client protocol provider.
@@ -50,7 +42,7 @@ public class IgniteHadoopClientProtocolProvider extends ClientProtocolProvider {
     public static final String FRAMEWORK_NAME = "ignite";
 
     /** Clients. */
-    private static final ConcurrentHashMap<String, IgniteInternalFuture<GridClient>> cliMap = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<String, MapReduceClient> cliMap = new ConcurrentHashMap<>();
 
     /** {@inheritDoc} */
     @Override public ClientProtocol create(Configuration conf) throws IOException {
@@ -91,7 +83,12 @@ public class IgniteHadoopClientProtocolProvider extends ClientProtocolProvider {
 
     /** {@inheritDoc} */
     @Override public void close(ClientProtocol cliProto) throws IOException {
-        // No-op.
+        if (cliProto instanceof HadoopClientProtocol) {
+            MapReduceClient cli = ((HadoopClientProtocol)cliProto).client();
+
+            if (cli.release())
+                cliMap.remove(cli.cluster(), cli);
+        }
     }
 
     /**
@@ -102,7 +99,7 @@ public class IgniteHadoopClientProtocolProvider extends ClientProtocolProvider {
      * @return Client protocol.
      * @throws IOException If failed.
      */
-    private static ClientProtocol createProtocol(String addr, Configuration conf) throws IOException {
+    private ClientProtocol createProtocol(String addr, Configuration conf) throws IOException {
         return new HadoopClientProtocol(conf, client(addr, Collections.singletonList(addr)));
     }
 
@@ -114,45 +111,24 @@ private static ClientProtocol createProtocol(String addr, Configuration conf) th
      * @return Client.
      * @throws IOException If failed.
      */
-    private static GridClient client(String clusterName, Collection<String> addrs) throws IOException {
-        try {
-            IgniteInternalFuture<GridClient> fut = cliMap.get(clusterName);
-
-            if (fut == null) {
-                GridFutureAdapter<GridClient> fut0 = new GridFutureAdapter<>();
-
-                IgniteInternalFuture<GridClient> oldFut = cliMap.putIfAbsent(clusterName, fut0);
+    @SuppressWarnings("unchecked")
+    private MapReduceClient client(String clusterName, Collection<String> addrs) throws IOException {
+        while (true) {
+            MapReduceClient cli = cliMap.get(clusterName);
 
-                if (oldFut != null)
-                    return oldFut.get();
-                else {
-                    GridClientConfiguration cliCfg = new GridClientConfiguration();
+            if (cli == null) {
+                cli = new MapReduceClient(clusterName, addrs);
 
-                    cliCfg.setProtocol(TCP);
-                    cliCfg.setServers(addrs);
-                    cliCfg.setMarshaller(new GridClientJdkMarshaller());
-                    cliCfg.setMaxConnectionIdleTime(24 * 60 * 60 * 1000L); // 1 day.
-                    cliCfg.setDaemon(true);
+                MapReduceClient oldCli = cliMap.putIfAbsent(clusterName, cli);
 
-                    try {
-                        GridClient cli = GridClientFactory.start(cliCfg);
-
-                        fut0.onDone(cli);
-
-                        return cli;
-                    }
-                    catch (GridClientException e) {
-                        fut0.onDone(e);
-
-                        throw new IOException("Failed to establish connection with Ignite: " + addrs, e);
-                    }
-                }
+                if (oldCli != null)
+                    cli = oldCli;
             }
+
+            if (cli.acquire())
+                return cli;
             else
-                return fut.get();
-        }
-        catch (IgniteCheckedException e) {
-            throw new IOException("Failed to establish connection with Ignite сдгые: " + addrs, e);
+                cliMap.remove(clusterName, cli);
         }
     }
 }
\ No newline at end of file
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/proto/HadoopClientProtocol.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/proto/HadoopClientProtocol.java
index be2aa093978da..7fc0e77248da3 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/proto/HadoopClientProtocol.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/proto/HadoopClientProtocol.java
@@ -43,7 +43,7 @@
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.token.Token;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.client.GridClient;
+import org.apache.ignite.internal.processors.hadoop.mapreduce.MapReduceClient;
 import org.apache.ignite.internal.client.GridClientException;
 import org.apache.ignite.internal.processors.hadoop.HadoopCommonUtils;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
@@ -78,7 +78,7 @@ public class HadoopClientProtocol implements ClientProtocol {
     private final Configuration conf;
 
     /** Ignite client. */
-    private volatile GridClient cli;
+    private final MapReduceClient cli;
 
     /** Last received version. */
     private long lastVer = -1;
@@ -90,9 +90,10 @@ public class HadoopClientProtocol implements ClientProtocol {
      * Constructor.
      *
      * @param conf Configuration.
-     * @param cli Ignite client.
+     * @param cli Client.
      */
-    public HadoopClientProtocol(Configuration conf, GridClient cli) {
+    public HadoopClientProtocol(Configuration conf, MapReduceClient cli) {
+        assert conf != null;
         assert cli != null;
 
         this.conf = conf;
@@ -104,7 +105,7 @@ public HadoopClientProtocol(Configuration conf, GridClient cli) {
         try {
             conf.setLong(HadoopCommonUtils.REQ_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());
 
-            HadoopJobId jobID = cli.compute().execute(HadoopProtocolNextTaskIdTask.class.getName(), null);
+            HadoopJobId jobID = execute(HadoopProtocolNextTaskIdTask.class);
 
             conf.setLong(HadoopCommonUtils.RESPONSE_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());
 
@@ -121,8 +122,8 @@ public HadoopClientProtocol(Configuration conf, GridClient cli) {
         try {
             conf.setLong(HadoopCommonUtils.JOB_SUBMISSION_START_TS_PROPERTY, U.currentTimeMillis());
 
-            HadoopJobStatus status = cli.compute().execute(HadoopProtocolSubmitJobTask.class.getName(),
-                new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId(), createJobInfo(conf)));
+            HadoopJobStatus status = execute(HadoopProtocolSubmitJobTask.class,
+                jobId.getJtIdentifier(), jobId.getId(), createJobInfo(conf));
 
             if (status == null)
                 throw new IOException("Failed to submit job (null status obtained): " + jobId);
@@ -157,8 +158,7 @@ public HadoopClientProtocol(Configuration conf, GridClient cli) {
     /** {@inheritDoc} */
     @Override public void killJob(JobID jobId) throws IOException, InterruptedException {
         try {
-            cli.compute().execute(HadoopProtocolKillJobTask.class.getName(),
-                new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId()));
+            execute(HadoopProtocolKillJobTask.class, jobId.getJtIdentifier(), jobId.getId());
         }
         catch (GridClientException e) {
             throw new IOException("Failed to kill job: " + jobId, e);
@@ -181,11 +181,12 @@ public HadoopClientProtocol(Configuration conf, GridClient cli) {
         try {
             Long delay = conf.getLong(HadoopJobProperty.JOB_STATUS_POLL_DELAY.propertyName(), -1);
 
-            HadoopProtocolTaskArguments args = delay >= 0 ?
-                new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId(), delay) :
-                new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId());
+            HadoopJobStatus status;
 
-            HadoopJobStatus status = cli.compute().execute(HadoopProtocolJobStatusTask.class.getName(), args);
+            if (delay >= 0)
+                status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId(), delay);
+            else
+                status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId());
 
             if (status == null)
                 throw new IOException("Job tracker doesn't have any information about the job: " + jobId);
@@ -200,8 +201,8 @@ public HadoopClientProtocol(Configuration conf, GridClient cli) {
     /** {@inheritDoc} */
     @Override public Counters getJobCounters(JobID jobId) throws IOException, InterruptedException {
         try {
-            final HadoopCounters counters = cli.compute().execute(HadoopProtocolJobCountersTask.class.getName(),
-                new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId()));
+            final HadoopCounters counters = execute(HadoopProtocolJobCountersTask.class,
+                jobId.getJtIdentifier(), jobId.getId());
 
             if (counters == null)
                 throw new IOException("Job tracker doesn't have any information about the job: " + jobId);
@@ -328,6 +329,21 @@ public HadoopClientProtocol(Configuration conf, GridClient cli) {
         return ProtocolSignature.getProtocolSignature(this, protocol, clientVersion, clientMethodsHash);
     }
 
+    /**
+     * Execute task.
+     *
+     * @param taskCls Task class.
+     * @param args Arguments.
+     * @return Result.
+     * @throws IOException If failed.
+     * @throws GridClientException If failed.
+     */
+    private <T> T execute(Class taskCls, Object... args) throws IOException, GridClientException {
+        HadoopProtocolTaskArguments args0 = args != null ? new HadoopProtocolTaskArguments(args) : null;
+
+        return cli.client().compute().execute(taskCls.getName(), args0);
+    }
+
     /**
      * Process received status update.
      *
@@ -351,4 +367,13 @@ private JobStatus processStatus(HadoopJobStatus status) {
 
         return HadoopUtils.status(lastStatus, conf);
     }
+
+    /**
+     * Gets the GridClient data.
+     *
+     * @return The client data.
+     */
+    public MapReduceClient client() {
+        return cli;
+    }
 }
\ No newline at end of file
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/mapreduce/MapReduceClient.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/mapreduce/MapReduceClient.java
new file mode 100644
index 0000000000000..3d52176e1ebb2
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/mapreduce/MapReduceClient.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.mapreduce;
+
+import org.apache.ignite.internal.client.GridClient;
+import org.apache.ignite.internal.client.GridClientConfiguration;
+import org.apache.ignite.internal.client.GridClientException;
+import org.apache.ignite.internal.client.GridClientFactory;
+import org.apache.ignite.internal.client.marshaller.jdk.GridClientJdkMarshaller;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.ignite.internal.client.GridClientProtocol.TCP;
+
+/**
+ * Client.
+ */
+public class MapReduceClient {
+    /** Cluster name. */
+    private final String cluster;
+
+    /** Addresses. */
+    private final Collection<String> addrs;
+
+    /** Mutex. */
+    private final Object mux = new Object();
+
+    /** Usage counter. */
+    private final AtomicInteger cnt = new AtomicInteger();
+
+    /** Client. */
+    private volatile GridClient cli;
+
+    /**
+     * Constructor.
+     *
+     * @param cluster Cluster name.
+     * @param addrs Addresses.
+     */
+    public MapReduceClient(String cluster, Collection<String> addrs) {
+        this.cluster = cluster;
+        this.addrs = addrs;
+    }
+
+    /**
+     * @return Cluster name..
+     */
+    public String cluster() {
+        return cluster;
+    }
+
+    /**
+     * Gets the client.
+     *
+     * @return The client.
+     */
+    public GridClient client() throws IOException {
+        GridClient cli0 = cli;
+
+        if (cli0 == null) {
+            synchronized (mux) {
+                cli0 = cli;
+
+                if (cli0 == null) {
+                    GridClientConfiguration cliCfg = new GridClientConfiguration();
+
+                    cliCfg.setProtocol(TCP);
+                    cliCfg.setServers(addrs);
+                    cliCfg.setMarshaller(new GridClientJdkMarshaller());
+                    cliCfg.setMaxConnectionIdleTime(24 * 60 * 60 * 1000L); // 1 day.
+                    cliCfg.setDaemon(true);
+
+                    try {
+                        cli0 = GridClientFactory.start(cliCfg);
+
+                        cli = cli0;
+                    }
+                    catch (GridClientException e) {
+                        throw new IOException("Failed to establish connection with Ignite: " + addrs, e);
+                    }
+                }
+            }
+        }
+
+        return cli0;
+    }
+
+    /**
+     * Increments usage count.
+     *
+     * @return {@code True} if succeeded and client can be used.
+     */
+    public boolean acquire() {
+        while (true) {
+            int cur = cnt.get();
+
+            if (cur < 0)
+                return false;
+
+            int next = cur + 1;
+
+            if (cnt.compareAndSet(cur, next))
+                return true;
+        }
+    }
+
+    /**
+     * Decrements the usages of the client and closes it if this is the last usage.
+     *
+     * @return {@code True} if client can be closed safely by the called.
+     */
+    public boolean release() {
+        int cnt0 = cnt.decrementAndGet();
+
+        assert cnt0 >= 0;
+
+        if (cnt0 == 0) {
+            if (cnt.compareAndSet(0, -1)) {
+                GridClient cli0 = cli;
+
+                if (cli0 != null)
+                    cli0.close();
+
+                return true;
+            }
+        }
+
+        return false;
+    }
+}
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolMultipleServersSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolMultipleServersSelfTest.java
index 0805be1b9879b..a4b5e6aac89ac 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolMultipleServersSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolMultipleServersSelfTest.java
@@ -23,8 +23,8 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
@@ -40,13 +40,10 @@
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteFileSystem;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider;
 import org.apache.ignite.igfs.IgfsPath;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.client.GridClient;
 import org.apache.ignite.internal.client.GridServerUnreachableException;
 import org.apache.ignite.internal.processors.hadoop.impl.HadoopAbstractSelfTest;
 import org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils;
@@ -78,35 +75,13 @@ public class HadoopClientProtocolMultipleServersSelfTest extends HadoopAbstractS
         return true;
     }
 
-    /** {@inheritDoc} */
-    @Override protected void beforeTest() throws Exception {
-        super.beforeTest();
-
-        clearClients();
-    }
-
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         stopAllGrids();
 
-        clearClients();
-
         super.afterTest();
     }
 
-    /**
-     * @throws IgniteCheckedException If failed.
-     */
-    private void clearConnectionMap() throws IgniteCheckedException {
-        ConcurrentHashMap<String, IgniteInternalFuture<GridClient>> cliMap =
-            GridTestUtils.getFieldValue(IgniteHadoopClientProtocolProvider.class, "cliMap");
-
-        for(IgniteInternalFuture<GridClient> fut : cliMap.values())
-            fut.get().close();
-
-        cliMap.clear();
-    }
-
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -116,18 +91,6 @@ private void clearConnectionMap() throws IgniteCheckedException {
         return cfg;
     }
 
-    /**
-     *
-     */
-    private void clearClients() {
-        ConcurrentHashMap<String, IgniteInternalFuture<GridClient>> cliMap = GridTestUtils.getFieldValue(
-            IgniteHadoopClientProtocolProvider.class,
-            IgniteHadoopClientProtocolProvider.class,
-            "cliMap");
-
-        cliMap.clear();
-    }
-
     /**
      * @throws Exception If failed.
      */
@@ -154,26 +117,31 @@ private void beforeJob() throws Exception {
     private void checkJobSubmit(Configuration conf) throws Exception {
         final Job job = Job.getInstance(conf);
 
-        job.setJobName(JOB_NAME);
+        try {
+            job.setJobName(JOB_NAME);
 
-        job.setOutputKeyClass(Text.class);
-        job.setOutputValueClass(IntWritable.class);
+            job.setOutputKeyClass(Text.class);
+            job.setOutputValueClass(IntWritable.class);
 
-        job.setInputFormatClass(TextInputFormat.class);
-        job.setOutputFormatClass(OutFormat.class);
+            job.setInputFormatClass(TextInputFormat.class);
+            job.setOutputFormatClass(OutFormat.class);
 
-        job.setMapperClass(TestMapper.class);
-        job.setReducerClass(TestReducer.class);
+            job.setMapperClass(TestMapper.class);
+            job.setReducerClass(TestReducer.class);
 
-        job.setNumReduceTasks(0);
+            job.setNumReduceTasks(0);
 
-        FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
+            FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
 
-        job.submit();
+            job.submit();
 
-        job.waitForCompletion(false);
+            job.waitForCompletion(false);
 
-        assert job.getStatus().getState() == JobStatus.State.SUCCEEDED : job.getStatus().getState();
+            assert job.getStatus().getState() == JobStatus.State.SUCCEEDED : job.getStatus().getState();
+        }
+        finally {
+            job.getCluster().close();
+        }
     }
 
     /**
@@ -197,18 +165,25 @@ public void testMultipleAddresses() throws Exception {
      */
     @SuppressWarnings({"ConstantConditions", "ThrowableResultOfMethodCallIgnored"})
     public void testSingleAddress() throws Exception {
-        // Don't use REST_PORT to test connection fails if the only this port is configured
-        restPort = REST_PORT + 1;
+        try {
+            // Don't use REST_PORT to test connection fails if the only this port is configured
+            restPort = REST_PORT + 1;
 
-        startGrids(gridCount());
+            startGrids(gridCount());
 
-        GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
+            GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
                 @Override public Object call() throws Exception {
-                    checkJobSubmit(configSingleAddress());
-                    return null;
-                }
-            },
-            GridServerUnreachableException.class, "Failed to connect to any of the servers in list");
+                        checkJobSubmit(configSingleAddress());
+                        return null;
+                    }
+                },
+                GridServerUnreachableException.class, "Failed to connect to any of the servers in list");
+        }
+        finally {
+            FileSystem fs = FileSystem.get(configSingleAddress());
+
+            fs.close();
+        }
     }
 
     /**
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
index 1ef7dd0a5cca9..7156a3d693936 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
@@ -50,7 +50,6 @@
 import org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider;
 import org.apache.ignite.igfs.IgfsFile;
 import org.apache.ignite.igfs.IgfsPath;
-import org.apache.ignite.internal.processors.hadoop.HadoopCommonUtils;
 import org.apache.ignite.internal.processors.hadoop.impl.HadoopAbstractSelfTest;
 import org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -115,7 +114,6 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
         stopAllGrids();
 
         super.afterTestsStopped();
-//        IgniteHadoopClientProtocolProvider.cliMap.clear();
     }
 
     /** {@inheritDoc} */
@@ -196,43 +194,48 @@ public void testJobCounters() throws Exception {
 
         final Job job = Job.getInstance(conf);
 
-        job.setOutputKeyClass(Text.class);
-        job.setOutputValueClass(IntWritable.class);
+        try {
+            job.setOutputKeyClass(Text.class);
+            job.setOutputValueClass(IntWritable.class);
 
-        job.setMapperClass(TestCountingMapper.class);
-        job.setReducerClass(TestCountingReducer.class);
-        job.setCombinerClass(TestCountingCombiner.class);
+            job.setMapperClass(TestCountingMapper.class);
+            job.setReducerClass(TestCountingReducer.class);
+            job.setCombinerClass(TestCountingCombiner.class);
 
-        FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
-        FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
+            FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
+            FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
 
-        job.submit();
+            job.submit();
 
-        final Counter cntr = job.getCounters().findCounter(TestCounter.COUNTER1);
+            final Counter cntr = job.getCounters().findCounter(TestCounter.COUNTER1);
 
-        assertEquals(0, cntr.getValue());
+            assertEquals(0, cntr.getValue());
 
-        cntr.increment(10);
+            cntr.increment(10);
 
-        assertEquals(10, cntr.getValue());
+            assertEquals(10, cntr.getValue());
 
-        // Transferring to map phase.
-        setupLockFile.delete();
+            // Transferring to map phase.
+            setupLockFile.delete();
 
-        // Transferring to reduce phase.
-        mapLockFile.delete();
+            // Transferring to reduce phase.
+            mapLockFile.delete();
 
-        job.waitForCompletion(false);
+            job.waitForCompletion(false);
 
-        assertEquals("job must end successfully", JobStatus.State.SUCCEEDED, job.getStatus().getState());
+            assertEquals("job must end successfully", JobStatus.State.SUCCEEDED, job.getStatus().getState());
 
-        final Counters counters = job.getCounters();
+            final Counters counters = job.getCounters();
 
-        assertNotNull("counters cannot be null", counters);
-        assertEquals("wrong counters count", 3, counters.countCounters());
-        assertEquals("wrong counter value", 15, counters.findCounter(TestCounter.COUNTER1).getValue());
-        assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER2).getValue());
-        assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER3).getValue());
+            assertNotNull("counters cannot be null", counters);
+            assertEquals("wrong counters count", 3, counters.countCounters());
+            assertEquals("wrong counter value", 15, counters.findCounter(TestCounter.COUNTER1).getValue());
+            assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER2).getValue());
+            assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER3).getValue());
+        }
+        finally {
+            job.getCluster().close();
+        }
     }
 
     /**
@@ -304,114 +307,119 @@ public void checkJobSubmit(boolean noCombiners, boolean noReducers) throws Excep
 
         final Job job = Job.getInstance(conf);
 
-        job.setJobName(JOB_NAME);
+        try {
+            job.setJobName(JOB_NAME);
 
-        job.setOutputKeyClass(Text.class);
-        job.setOutputValueClass(IntWritable.class);
+            job.setOutputKeyClass(Text.class);
+            job.setOutputValueClass(IntWritable.class);
 
-        job.setMapperClass(TestMapper.class);
-        job.setReducerClass(TestReducer.class);
+            job.setMapperClass(TestMapper.class);
+            job.setReducerClass(TestReducer.class);
 
-        if (!noCombiners)
-            job.setCombinerClass(TestCombiner.class);
+            if (!noCombiners)
+                job.setCombinerClass(TestCombiner.class);
 
-        if (noReducers)
-            job.setNumReduceTasks(0);
+            if (noReducers)
+                job.setNumReduceTasks(0);
 
-        job.setInputFormatClass(TextInputFormat.class);
-        job.setOutputFormatClass(TestOutputFormat.class);
+            job.setInputFormatClass(TextInputFormat.class);
+            job.setOutputFormatClass(TestOutputFormat.class);
 
-        FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
-        FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
+            FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
+            FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
 
-        job.submit();
+            job.submit();
 
-        JobID jobId = job.getJobID();
+            JobID jobId = job.getJobID();
 
-        // Setup phase.
-        JobStatus jobStatus = job.getStatus();
-        checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
-        assert jobStatus.getSetupProgress() >= 0.0f && jobStatus.getSetupProgress() < 1.0f;
-        assert jobStatus.getMapProgress() == 0.0f;
-        assert jobStatus.getReduceProgress() == 0.0f;
+            // Setup phase.
+            JobStatus jobStatus = job.getStatus();
+            checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+            assert jobStatus.getSetupProgress() >= 0.0f && jobStatus.getSetupProgress() < 1.0f;
+            assert jobStatus.getMapProgress() == 0.0f;
+            assert jobStatus.getReduceProgress() == 0.0f;
 
-        U.sleep(2100);
+            U.sleep(2100);
 
-        JobStatus recentJobStatus = job.getStatus();
+            JobStatus recentJobStatus = job.getStatus();
 
-        assert recentJobStatus.getSetupProgress() > jobStatus.getSetupProgress() :
-            "Old=" + jobStatus.getSetupProgress() + ", new=" + recentJobStatus.getSetupProgress();
+            assert recentJobStatus.getSetupProgress() > jobStatus.getSetupProgress() : "Old="
+                + jobStatus.getSetupProgress() + ", new=" + recentJobStatus.getSetupProgress();
 
-        // Transferring to map phase.
-        setupLockFile.delete();
+            // Transferring to map phase.
+            setupLockFile.delete();
 
-        assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override public boolean apply() {
-                try {
-                    return F.eq(1.0f, job.getStatus().getSetupProgress());
+            assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    try {
+                        return F.eq(1.0f, job.getStatus().getSetupProgress());
+                    }
+                    catch (Exception e) {
+                        throw new RuntimeException("Unexpected exception.", e);
+                    }
                 }
-                catch (Exception e) {
-                    throw new RuntimeException("Unexpected exception.", e);
-                }
-            }
-        }, 5000L);
+            }, 5000L);
 
-        // Map phase.
-        jobStatus = job.getStatus();
-        checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
-        assert jobStatus.getSetupProgress() == 1.0f;
-        assert jobStatus.getMapProgress() >= 0.0f && jobStatus.getMapProgress() < 1.0f;
-        assert jobStatus.getReduceProgress() == 0.0f;
+            // Map phase.
+            jobStatus = job.getStatus();
+            checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+            assert jobStatus.getSetupProgress() == 1.0f;
+            assert jobStatus.getMapProgress() >= 0.0f && jobStatus.getMapProgress() < 1.0f;
+            assert jobStatus.getReduceProgress() == 0.0f;
 
-        U.sleep(2100);
+            U.sleep(2100);
 
-        recentJobStatus = job.getStatus();
+            recentJobStatus = job.getStatus();
 
-        assert recentJobStatus.getMapProgress() > jobStatus.getMapProgress() :
-            "Old=" + jobStatus.getMapProgress() + ", new=" + recentJobStatus.getMapProgress();
+            assert recentJobStatus.getMapProgress() > jobStatus.getMapProgress() : "Old=" + jobStatus.getMapProgress()
+                + ", new=" + recentJobStatus.getMapProgress();
 
-        // Transferring to reduce phase.
-        mapLockFile.delete();
+            // Transferring to reduce phase.
+            mapLockFile.delete();
 
-        assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override public boolean apply() {
-                try {
-                    return F.eq(1.0f, job.getStatus().getMapProgress());
-                }
-                catch (Exception e) {
-                    throw new RuntimeException("Unexpected exception.", e);
+            assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    try {
+                        return F.eq(1.0f, job.getStatus().getMapProgress());
+                    }
+                    catch (Exception e) {
+                        throw new RuntimeException("Unexpected exception.", e);
+                    }
                 }
-            }
-        }, 5000L);
+            }, 5000L);
 
-        if (!noReducers) {
-            // Reduce phase.
-            jobStatus = job.getStatus();
-            checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
-            assert jobStatus.getSetupProgress() == 1.0f;
-            assert jobStatus.getMapProgress() == 1.0f;
-            assert jobStatus.getReduceProgress() >= 0.0f && jobStatus.getReduceProgress() < 1.0f;
+            if (!noReducers) {
+                // Reduce phase.
+                jobStatus = job.getStatus();
+                checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+                assert jobStatus.getSetupProgress() == 1.0f;
+                assert jobStatus.getMapProgress() == 1.0f;
+                assert jobStatus.getReduceProgress() >= 0.0f && jobStatus.getReduceProgress() < 1.0f;
 
-            // Ensure that reduces progress increases.
-            U.sleep(2100);
+                // Ensure that reduces progress increases.
+                U.sleep(2100);
 
-            recentJobStatus = job.getStatus();
+                recentJobStatus = job.getStatus();
 
-            assert recentJobStatus.getReduceProgress() > jobStatus.getReduceProgress() :
-                "Old=" + jobStatus.getReduceProgress() + ", new=" + recentJobStatus.getReduceProgress();
+                assert recentJobStatus.getReduceProgress() > jobStatus.getReduceProgress() : "Old="
+                    + jobStatus.getReduceProgress() + ", new=" + recentJobStatus.getReduceProgress();
 
-            reduceLockFile.delete();
-        }
+                reduceLockFile.delete();
+            }
 
-        job.waitForCompletion(false);
+            job.waitForCompletion(false);
 
-        jobStatus = job.getStatus();
-        checkJobStatus(job.getStatus(), jobId, JOB_NAME, JobStatus.State.SUCCEEDED, 1.0f);
-        assert jobStatus.getSetupProgress() == 1.0f;
-        assert jobStatus.getMapProgress() == 1.0f;
-        assert jobStatus.getReduceProgress() == 1.0f;
+            jobStatus = job.getStatus();
+            checkJobStatus(job.getStatus(), jobId, JOB_NAME, JobStatus.State.SUCCEEDED, 1.0f);
+            assert jobStatus.getSetupProgress() == 1.0f;
+            assert jobStatus.getMapProgress() == 1.0f;
+            assert jobStatus.getReduceProgress() == 1.0f;
 
-        dumpIgfs(igfs, new IgfsPath(PATH_OUTPUT));
+            dumpIgfs(igfs, new IgfsPath(PATH_OUTPUT));
+        }
+        finally {
+            job.getCluster().close();
+        }
     }
 
     /**
@@ -517,7 +525,12 @@ public static class TestMapper extends Mapper<Object, Text, Text, IntWritable> {
      * Test Hadoop counters.
      */
     public enum TestCounter {
-        COUNTER1, COUNTER2, COUNTER3
+        /** */
+        COUNTER1,
+        /** */
+        COUNTER2,
+        /** */
+        COUNTER3
     }
 
     /**
@@ -535,6 +548,7 @@ public static class TestCountingMapper extends TestMapper {
      * Test combiner that counts invocations.
      */
     public static class TestCountingCombiner extends TestReducer {
+        /** {@inheritDoc} */
         @Override public void reduce(Text key, Iterable<IntWritable> values,
             Context ctx) throws IOException, InterruptedException {
             ctx.getCounter(TestCounter.COUNTER1).increment(1);
@@ -552,6 +566,7 @@ public static class TestCountingCombiner extends TestReducer {
      * Test reducer that counts invocations.
      */
     public static class TestCountingReducer extends TestReducer {
+        /** {@inheritDoc} */
         @Override public void reduce(Text key, Iterable<IntWritable> values,
             Context ctx) throws IOException, InterruptedException {
             ctx.getCounter(TestCounter.COUNTER1).increment(1);
@@ -566,6 +581,9 @@ public static class TestCombiner extends Reducer<Text, IntWritable, Text, IntWri
         // No-op.
     }
 
+    /**
+     * Test output format.
+     */
     public static class TestOutputFormat<K, V> extends TextOutputFormat<K, V> {
         /** {@inheritDoc} */
         @Override public synchronized OutputCommitter getOutputCommitter(TaskAttemptContext ctx)

From b44baf1e8c42c57fa4e241d5943593fa4ae42f12 Mon Sep 17 00:00:00 2001
From: iveselovskiy <iveselovskiy@gridgain.com>
Date: Mon, 12 Dec 2016 16:52:47 +0300
Subject: [PATCH 049/446] IGNITE-4341: Hadoop: added Terasort to unit tests.
 This closes #1302. This closes #1321.

---
 modules/hadoop/pom.xml                        |   7 +
 .../impl/fs/HadoopFileSystemsUtils.java       |  11 +
 .../impl/v2/HadoopV2JobResourceManager.java   |  25 +-
 .../hadoop/impl/HadoopAbstractSelfTest.java   |  13 +-
 .../impl/HadoopAbstractWordCountTest.java     |   6 +-
 .../hadoop/impl/HadoopFileSystemsTest.java    |   9 +
 .../hadoop/impl/HadoopJobTrackerSelfTest.java |   4 +-
 .../impl/HadoopTaskExecutionSelfTest.java     |   4 +-
 .../hadoop/impl/HadoopTeraSortTest.java       | 376 ++++++++++++++++++
 .../client/HadoopClientProtocolSelfTest.java  |   4 +-
 .../collections/HadoopSkipListSelfTest.java   |  14 +-
 .../HadoopExternalTaskExecutionSelfTest.java  |   2 +
 .../testsuites/IgniteHadoopTestSuite.java     |   3 +
 13 files changed, 450 insertions(+), 28 deletions(-)
 create mode 100644 modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java

diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index d0b0481775d42..db302d7f13278 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -94,6 +94,13 @@
             <artifactId>log4j</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-mapreduce-examples</artifactId>
+            <version>${hadoop.version}</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.gridgain</groupId>
             <artifactId>ignite-shmem</artifactId>
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/fs/HadoopFileSystemsUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/fs/HadoopFileSystemsUtils.java
index 5115cb47c212f..37902f0547726 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/fs/HadoopFileSystemsUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/fs/HadoopFileSystemsUtils.java
@@ -17,7 +17,9 @@
 
 package org.apache.ignite.internal.processors.hadoop.impl.fs;
 
+import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FsConstants;
 import org.jetbrains.annotations.Nullable;
 
@@ -48,4 +50,13 @@ public static void setupFileSystems(Configuration cfg) {
     public static String disableFsCachePropertyName(@Nullable String scheme) {
         return String.format("fs.%s.impl.disable.cache", scheme);
     }
+
+    /**
+     * Clears Hadoop {@link FileSystem} cache.
+     *
+     * @throws IOException On error.
+     */
+    public static void clearFileSystemCache() throws IOException {
+        FileSystem.closeAll();
+    }
 }
\ No newline at end of file
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
index 3984f838aa46f..52e394be96589 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
@@ -123,7 +123,9 @@ public void prepareJobEnvironment(boolean download, File jobLocDir) throws Ignit
 
             JobConf cfg = ctx.getJobConf();
 
-            String mrDir = cfg.get("mapreduce.job.dir");
+            Collection<URL> clsPathUrls = new ArrayList<>();
+
+            String mrDir = cfg.get(MRJobConfig.MAPREDUCE_JOB_DIR);
 
             if (mrDir != null) {
                 stagingDir = new Path(new URI(mrDir));
@@ -144,28 +146,23 @@ public void prepareJobEnvironment(boolean download, File jobLocDir) throws Ignit
 
                 File jarJobFile = new File(jobLocDir, "job.jar");
 
-                Collection<URL> clsPathUrls = new ArrayList<>();
-
                 clsPathUrls.add(jarJobFile.toURI().toURL());
 
                 rsrcSet.add(jarJobFile);
                 rsrcSet.add(new File(jobLocDir, "job.xml"));
-
-                processFiles(jobLocDir, ctx.getCacheFiles(), download, false, null, MRJobConfig.CACHE_LOCALFILES);
-                processFiles(jobLocDir, ctx.getCacheArchives(), download, true, null, MRJobConfig.CACHE_LOCALARCHIVES);
-                processFiles(jobLocDir, ctx.getFileClassPaths(), download, false, clsPathUrls, null);
-                processFiles(jobLocDir, ctx.getArchiveClassPaths(), download, true, clsPathUrls, null);
-
-                if (!clsPathUrls.isEmpty()) {
-                    clsPath = new URL[clsPathUrls.size()];
-
-                    clsPathUrls.toArray(clsPath);
-                }
             }
             else if (!jobLocDir.mkdirs())
                 throw new IgniteCheckedException("Failed to create local job directory: "
                     + jobLocDir.getAbsolutePath());
 
+            processFiles(jobLocDir, ctx.getCacheFiles(), download, false, null, MRJobConfig.CACHE_LOCALFILES);
+            processFiles(jobLocDir, ctx.getCacheArchives(), download, true, null, MRJobConfig.CACHE_LOCALARCHIVES);
+            processFiles(jobLocDir, ctx.getFileClassPaths(), download, false, clsPathUrls, null);
+            processFiles(jobLocDir, ctx.getArchiveClassPaths(), download, true, clsPathUrls, null);
+
+            if (!clsPathUrls.isEmpty())
+                clsPath = clsPathUrls.toArray(new URL[clsPathUrls.size()]);
+
             setLocalFSWorkingDirectory(jobLocDir);
         }
         catch (URISyntaxException | IOException e) {
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractSelfTest.java
index 12351c676424b..5666cbc2d714b 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractSelfTest.java
@@ -83,7 +83,9 @@ public abstract class HadoopAbstractSelfTest extends GridCommonAbstractTest {
     private static String initCp;
 
     /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
+    @Override protected final void beforeTestsStarted() throws Exception {
+        HadoopFileSystemsUtils.clearFileSystemCache();
+
         // Add surefire classpath to regular classpath.
         initCp = System.getProperty("java.class.path");
 
@@ -93,6 +95,15 @@ public abstract class HadoopAbstractSelfTest extends GridCommonAbstractTest {
             System.setProperty("java.class.path", initCp + File.pathSeparatorChar + surefireCp);
 
         super.beforeTestsStarted();
+
+        beforeTestsStarted0();
+    }
+
+    /**
+     * Performs additional initialization in the beginning of test class execution.
+     */
+    protected void beforeTestsStarted0() throws Exception {
+        // noop
     }
 
     /** {@inheritDoc} */
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractWordCountTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractWordCountTest.java
index 3cb8f914db5f5..84e6aee4249f7 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractWordCountTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractWordCountTest.java
@@ -49,9 +49,7 @@ public abstract class HadoopAbstractWordCountTest extends HadoopAbstractSelfTest
     protected IgfsEx igfs;
 
     /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        super.beforeTestsStarted();
-
+    @Override protected void beforeTestsStarted0() throws Exception {
         Configuration cfg = new Configuration();
 
         setupFileSystems(cfg);
@@ -62,6 +60,8 @@ public abstract class HadoopAbstractWordCountTest extends HadoopAbstractSelfTest
 
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
         igfs = (IgfsEx)startGrids(gridCount()).fileSystem(igfsName);
     }
 
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopFileSystemsTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopFileSystemsTest.java
index 252d6cb4af907..76806902b4a77 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopFileSystemsTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopFileSystemsTest.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopFileSystemsUtils;
+import org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopLocalFileSystemV1;
 import org.apache.ignite.testframework.GridTestUtils;
 
 /**
@@ -37,11 +38,15 @@ public class HadoopFileSystemsTest extends HadoopAbstractSelfTest {
 
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
         startGrids(gridCount());
     }
 
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
         stopAllGrids(true);
     }
 
@@ -70,6 +75,10 @@ private void testFileSystem(final URI uri) throws Exception {
         cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP,
             new Path(new Path(uri), "user/" + System.getProperty("user.name")).toString());
 
+        FileSystem fs = FileSystem.get(uri, cfg);
+
+        assertTrue(fs instanceof HadoopLocalFileSystemV1);
+
         final CountDownLatch changeUserPhase = new CountDownLatch(THREAD_COUNT);
         final CountDownLatch changeDirPhase = new CountDownLatch(THREAD_COUNT);
         final CountDownLatch changeAbsDirPhase = new CountDownLatch(THREAD_COUNT);
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
index a3bf49c59c22f..91ad5eca51edb 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
@@ -65,9 +65,7 @@ public class HadoopJobTrackerSelfTest extends HadoopAbstractSelfTest {
     }
 
     /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        super.beforeTestsStarted();
-
+    @Override protected void beforeTestsStarted0() throws Exception {
         startGrids(gridCount());
     }
 
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
index 027f921f0d83b..9d45b03993482 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
@@ -83,9 +83,7 @@ public class HadoopTaskExecutionSelfTest extends HadoopAbstractSelfTest {
     }
 
     /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        super.beforeTestsStarted();
-
+    @Override protected void beforeTestsStarted0() throws Exception {
         startGrids(gridCount());
     }
 
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
new file mode 100644
index 0000000000000..0cc95645772f6
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.URI;
+import java.util.UUID;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.examples.terasort.TeraGen;
+import org.apache.hadoop.examples.terasort.TeraInputFormat;
+import org.apache.hadoop.examples.terasort.TeraOutputFormat;
+import org.apache.hadoop.examples.terasort.TeraSort;
+import org.apache.hadoop.examples.terasort.TeraValidate;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.HadoopConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+
+import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.createJobInfo;
+
+/**
+ * Implements TeraSort Hadoop sample as a unit test.
+ */
+public class HadoopTeraSortTest extends HadoopAbstractSelfTest {
+    /** Copy of Hadoop constant of package-private visibility. */
+    public static final String PARTITION_FILENAME = getPartitionFileNameConstant();
+
+    /**  Out destination dir. */
+    protected final String generateOutDir = getFsBase() + "/tera-generated";
+
+    /** Sort destination dir. */
+    protected final String sortOutDir = getFsBase() + "/tera-sorted";
+
+    /** Validation destination dir. */
+    protected final String validateOutDir = getFsBase() + "/tera-validated";
+
+    /**
+     * Extracts value of Hadoop package-private constant.
+     *
+     * @return TeraInputFormat.PARTITION_FILENAME.
+     */
+    private static String getPartitionFileNameConstant() {
+        try {
+            Field f = TeraInputFormat.class.getDeclaredField("PARTITION_FILENAME");
+
+            f.setAccessible(true);
+
+            return (String)f.get(null);
+        }
+        catch (Exception e) {
+            throw new IgniteException(e);
+        }
+    }
+
+    /**
+     * Gets base directory.
+     * Note that this directory will be completely deleted in the and of the test.
+     * @return The base directory.
+     */
+    protected String getFsBase() {
+        return "file:///tmp/" + getUser() + "/hadoop-terasort-test";
+    }
+
+    /**
+     * @return Full input data size, in bytes.
+     */
+    protected long dataSizeBytes() {
+        return 100_000_000;
+    }
+
+    /**
+     * Desired number of maps in TeraSort job.
+     * @return The number of maps.
+     */
+    protected int numMaps() {
+        return gridCount() * 10;
+    }
+
+    /**
+     * Desired number of reduces in TeraSort job.
+     * @return The number of reduces.
+     */
+    protected int numReduces() {
+        return gridCount() * 8;
+    }
+
+    /**
+     * The user to run Hadoop job on behalf of.
+     * @return The user to run Hadoop job on behalf of.
+     */
+    protected String getUser() {
+        return System.getProperty("user.name");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids(true);
+
+        // Delete files used:
+        getFileSystem().delete(new Path(getFsBase()), true);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected final boolean igfsEnabled() {
+        return false;
+    }
+
+    /**
+     * Does actual test TeraSort job Through Ignite API
+     */
+    protected final void teraSort() throws Exception {
+        System.out.println("TeraSort ===============================================================");
+
+        getFileSystem().delete(new Path(sortOutDir), true);
+
+        final JobConf jobConf = new JobConf();
+
+        jobConf.setUser(getUser());
+
+        jobConf.set("fs.defaultFS", getFsBase());
+
+        log().info("Desired number of reduces: " + numReduces());
+
+        jobConf.set("mapreduce.job.reduces", String.valueOf(numReduces()));
+
+        log().info("Desired number of maps: " + numMaps());
+
+        final long splitSize = dataSizeBytes() / numMaps();
+
+        log().info("Desired split size: " + splitSize);
+
+        // Force the split to be of the desired size:
+        jobConf.set("mapred.min.split.size", String.valueOf(splitSize));
+        jobConf.set("mapred.max.split.size", String.valueOf(splitSize));
+
+        Job job = setupConfig(jobConf);
+
+        HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 1);
+
+        IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
+
+        fut.get();
+    }
+
+    /**
+     * Gets the file system we work upon.
+     * @return The file system.
+     * @throws Exception
+     */
+    FileSystem getFileSystem() throws Exception{
+        return FileSystem.get(new URI(getFsBase()), new Configuration());
+    }
+
+    /**
+     * Represents the data generation stage.
+     * @throws Exception
+     */
+    private void teraGenerate() throws Exception {
+        System.out.println("TeraGenerate ===============================================================");
+
+        getFileSystem().delete(new Path(generateOutDir), true);
+
+        final long numLines = dataSizeBytes() / 100; // TeraGen makes 100 bytes ber line
+
+        if (numLines < 1)
+            throw new IllegalStateException("Data size is too small: " + dataSizeBytes());
+
+        // Generate input data:
+        int res = ToolRunner.run(new Configuration(), new TeraGen(), new String[] {"-Dmapreduce.framework.name=local",
+            String.valueOf(numLines), generateOutDir});
+
+        assertEquals(0, res);
+
+        FileStatus[] fileStatuses = getFileSystem().listStatus(new Path(generateOutDir));
+
+        long sumLen = 0;
+
+        for (FileStatus fs: fileStatuses)
+            sumLen += fs.getLen();
+
+        assertEquals(dataSizeBytes(), sumLen); // Ensure correct size data is generated.
+    }
+
+    /**
+     * Creates Job instance and sets up necessary properties for it.
+     * @param conf The Job config.
+     * @return The job.
+     * @throws Exception On error.
+     */
+    private Job setupConfig(JobConf conf) throws Exception {
+        Job job = Job.getInstance(conf);
+
+        Path inputDir = new Path(generateOutDir);
+        Path outputDir = new Path(sortOutDir);
+
+        boolean useSimplePartitioner = TeraSort.getUseSimplePartitioner(job);
+
+        TeraInputFormat.setInputPaths(job, inputDir);
+        FileOutputFormat.setOutputPath(job, outputDir);
+
+        job.setJobName("TeraSort");
+
+        job.setOutputKeyClass(Text.class);
+        job.setOutputValueClass(Text.class);
+
+        job.setInputFormatClass(TeraInputFormat.class);
+        job.setOutputFormatClass(TeraOutputFormat.class);
+
+        if (useSimplePartitioner)
+            job.setPartitionerClass(TeraSort.SimplePartitioner.class);
+        else {
+            long start = System.currentTimeMillis();
+
+            Path partFile = new Path(outputDir, PARTITION_FILENAME);
+
+            URI partUri = new URI(partFile.toString() + "#" + PARTITION_FILENAME);
+
+            try {
+                TeraInputFormat.writePartitionFile(job, partFile);
+            } catch (Throwable e) {
+                throw new RuntimeException(e);
+            }
+
+            job.addCacheFile(partUri);
+
+            long end = System.currentTimeMillis();
+
+            System.out.println("Spent " + (end - start) + "ms computing partitions. " +
+                "Partition file added to distributed cache: " + partUri);
+
+            job.setPartitionerClass(getTeraSortTotalOrderPartitioner()/*TeraSort.TotalOrderPartitioner.class*/);
+        }
+
+        job.getConfiguration().setInt("dfs.replication", TeraSort.getOutputReplication(job));
+
+        /* TeraOutputFormat.setFinalSync(job, true); */
+        Method m = TeraOutputFormat.class.getDeclaredMethod("setFinalSync", JobContext.class, boolean.class);
+        m.setAccessible(true);
+        m.invoke(null, job, true);
+
+        return job;
+    }
+
+    /**
+     * Extracts package-private TeraSort total order partitioner class.
+     *
+     * @return The class.
+     */
+    @SuppressWarnings("unchecked")
+    private Class<? extends Partitioner> getTeraSortTotalOrderPartitioner() {
+        Class[] classes = TeraSort.class.getDeclaredClasses();
+
+        Class<? extends Partitioner> totalOrderPartitionerCls = null;
+
+        for (Class<?> x: classes) {
+            if ("TotalOrderPartitioner".equals(x.getSimpleName())) {
+                totalOrderPartitionerCls = (Class<? extends Partitioner>)x;
+
+                break;
+            }
+        }
+
+        if (totalOrderPartitionerCls == null)
+            throw new IllegalStateException("Failed to find TeraSort total order partitioner class.");
+
+        return totalOrderPartitionerCls;
+    }
+
+    /**
+     * Implements validation phase of the sample.
+     * @throws Exception
+     */
+    private void teraValidate() throws Exception {
+        System.out.println("TeraValidate ===============================================================");
+
+        getFileSystem().delete(new Path(validateOutDir), true);
+
+        // Generate input data:
+        int res = ToolRunner.run(new Configuration(), new TeraValidate(),
+            new String[] {"-Dmapreduce.framework.name=local", sortOutDir, validateOutDir});
+
+        assertEquals(0, res);
+
+        FileStatus[] fileStatuses = getFileSystem().listStatus(new Path(validateOutDir), new PathFilter() {
+            @Override public boolean accept(Path path) {
+                // Typically name is "part-r-00000":
+                return path.getName().startsWith("part-r-");
+            }
+        });
+
+        // TeraValidate has only 1 reduce, so should be only 1 result file:
+        assertEquals(1, fileStatuses.length);
+
+        // The result file must contain only 1 line with the checksum, like this:
+        // "checksum        7a27e2d0d55de",
+        // typically it has length of 23 bytes.
+        // If sorting was not correct, the result contains list of K-V pairs that are not ordered correctly.
+        // In such case the size of the output will be much larger.
+        long len = fileStatuses[0].getLen();
+
+        assertTrue("TeraValidate length: " + len, len >= 16 && len <= 32);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        getFileSystem().delete(new Path(getFsBase()), true);
+
+        startGrids(gridCount());
+    }
+
+    /**
+     * Runs generate/sort/validate phases of the terasort sample.
+     * @throws Exception
+     */
+    public void testTeraSort() throws Exception {
+        teraGenerate();
+
+        teraSort();
+
+        teraValidate();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration igc = super.getConfiguration(gridName);
+
+        HadoopConfiguration hc = createHadoopConfiguration();
+
+        igc.setHadoopConfiguration(hc);
+
+        return igc;
+    }
+
+    /**
+     * Creates Hadoop configuration for the test.
+     * @return The {@link HadoopConfiguration}.
+     */
+    protected HadoopConfiguration createHadoopConfiguration() {
+        HadoopConfiguration hadoopCfg = new HadoopConfiguration();
+
+        // See org.apache.ignite.configuration.HadoopConfiguration.DFLT_MAX_TASK_QUEUE_SIZE
+        hadoopCfg.setMaxTaskQueueSize(30_000);
+
+        return hadoopCfg;
+    }
+}
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
index 7156a3d693936..44fc46e992c00 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
@@ -99,9 +99,7 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
     }
 
     /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        super.beforeTestsStarted();
-
+    @Override protected void beforeTestsStarted0() throws Exception {
         startGrids(gridCount());
 
         setupLockFile.delete();
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
index 111ea78d8280f..113880329dd77 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
@@ -85,6 +85,9 @@ public void testLevel() {
         }
     }
 
+    /**
+     * @throws Exception On error.
+     */
     public void testMapSimple() throws Exception {
         GridUnsafeMemory mem = new GridUnsafeMemory(0);
 
@@ -139,7 +142,16 @@ public void testMapSimple() throws Exception {
         assertEquals(0, mem.allocatedSize());
     }
 
-    private void check(HadoopMultimap m, Multimap<Integer, Integer> mm, final Multimap<Integer, Integer> vis, HadoopTaskContext taskCtx)
+    /**
+     * Check.
+     * @param m The multimap.
+     * @param mm The multimap storing expectations.
+     * @param vis The multimap to store visitor results.
+     * @param taskCtx The task context.
+     * @throws Exception On error.
+     */
+    private void check(HadoopMultimap m, Multimap<Integer, Integer> mm, final Multimap<Integer, Integer> vis,
+        HadoopTaskContext taskCtx)
         throws Exception {
         final HadoopTaskInput in = m.input(taskCtx);
 
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
index 7c43500209730..5f64ce7772187 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
@@ -57,6 +57,8 @@ public class HadoopExternalTaskExecutionSelfTest extends HadoopAbstractSelfTest
     @Override protected void beforeTest() throws Exception {
         fail("https://issues.apache.org/jira/browse/IGNITE-404");
 
+        super.beforeTest();
+
         startGrids(gridCount());
     }
 
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
index 959bc59f819be..6046cc10c4efd 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
@@ -23,6 +23,7 @@
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.internal.processors.hadoop.HadoopTestClassLoader;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopTeraSortTest;
 import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolEmbeddedSelfTest;
 import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolMultipleServersSelfTest;
 import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolSelfTest;
@@ -123,6 +124,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTest(new TestSuite(ldr.loadClass(KerberosHadoopFileSystemFactorySelfTest.class.getName())));
 
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopTeraSortTest.class.getName())));
+
         suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyTest.class.getName())));
         suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyFullMapReduceTest.class.getName())));
 

From 30b869ddd32db637ee9ea8f13a115dd4bacc52fe Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Wed, 14 Dec 2016 14:35:29 +0300
Subject: [PATCH 050/446] IGNITE-4426: Hadoop: tasks can share the same
 classloader. This closes #1344.

---
 .../processors/hadoop/HadoopClassLoader.java  | 10 +++++++++
 .../processors/hadoop/HadoopJobProperty.java  |  8 +++++++
 .../hadoop/impl/v2/HadoopV2Job.java           | 22 +++++++++++++++++--
 3 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index cd94c8978573d..f6c2fa98ddb4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -100,6 +100,16 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
         PREDEFINED_NATIVE_LIBS.add("MapRClient");
     }
 
+    /**
+     * Classloader name for job.
+     *
+     * @param jobId Job ID.
+     * @return Name.
+     */
+    public static String nameForJob(HadoopJobId jobId) {
+        return "hadoop-job-" + jobId;
+    }
+
     /**
      * Gets name for the task class loader. Task class loader
      * @param info The task info.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index 1f0ef1b037213..9e1dedec4a62a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -56,6 +56,13 @@ public enum HadoopJobProperty {
      */
     JOB_STATUS_POLL_DELAY("ignite.job.status.poll.delay"),
 
+    /**
+     * Whether job classloader can be shared between all tasks.
+     * <p>
+     * Defaults to {@code true}.
+     */
+    JOB_SHARED_CLASSLOADER("ignite.job.shared.classloader"),
+
     /**
      * Size in bytes of single memory page which will be allocated for data structures in shuffle.
      * <p>
@@ -105,6 +112,7 @@ public enum HadoopJobProperty {
      */
     SHUFFLE_JOB_THROTTLE("ignite.shuffle.job.throttle");
 
+
     /** Property name. */
     private final String propName;
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Job.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Job.java
index 36da41028678b..a24e5816cba40 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Job.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Job.java
@@ -41,6 +41,7 @@
 import org.apache.ignite.internal.processors.hadoop.HadoopJob;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobInfo;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskType;
@@ -73,6 +74,7 @@
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ConcurrentMap;
 
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.JOB_SHARED_CLASSLOADER;
 import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.jobLocalDir;
 import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.taskLocalDir;
 import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.transformException;
@@ -121,6 +123,9 @@ public class HadoopV2Job implements HadoopJob {
     /** File system cache map. */
     private final HadoopLazyConcurrentMap<FsCacheKey, FileSystem> fsMap = createHadoopLazyConcurrentMap();
 
+    /** Shared class loader. */
+    private volatile HadoopClassLoader sharedClsLdr;
+
     /** Local node ID */
     private volatile UUID locNodeId;
 
@@ -261,8 +266,8 @@ public HadoopV2Job(HadoopJobId jobId, final HadoopDefaultJobInfo jobInfo, Ignite
                 // If there is no pooled class, then load new one.
                 // Note that the classloader identified by the task it was initially created for,
                 // but later it may be reused for other tasks.
-                HadoopClassLoader ldr = new HadoopClassLoader(rsrcMgr.classPath(),
-                    HadoopClassLoader.nameForTask(info, false), libNames, helper);
+                HadoopClassLoader ldr = sharedClsLdr != null ?
+                    sharedClsLdr : createClassLoader(HadoopClassLoader.nameForTask(info, false));
 
                 cls = (Class<? extends HadoopTaskContext>)ldr.loadClass(HadoopV2TaskContext.class.getName());
 
@@ -312,6 +317,9 @@ public HadoopV2Job(HadoopJobId jobId, final HadoopDefaultJobInfo jobInfo, Ignite
 
         try {
             rsrcMgr.prepareJobEnvironment(!external, jobLocalDir(igniteWorkDirectory(), locNodeId, jobId));
+
+            if (HadoopJobProperty.get(jobInfo, JOB_SHARED_CLASSLOADER, true))
+                sharedClsLdr = createClassLoader(HadoopClassLoader.nameForJob(jobId));
         }
         finally {
             HadoopCommonUtils.restoreContextClassLoader(oldLdr);
@@ -454,4 +462,14 @@ public JobConf jobConf() {
     public FileSystem fileSystem(@Nullable URI uri, Configuration cfg) throws IOException {
         return fileSystemForMrUserWithCaching(uri, cfg, fsMap);
     }
+
+    /**
+     * Create class loader with the given name.
+     *
+     * @param name Name.
+     * @return Class loader.
+     */
+    private HadoopClassLoader createClassLoader(String name) {
+        return new HadoopClassLoader(rsrcMgr.classPath(), name, libNames, helper);
+    }
 }
\ No newline at end of file

From c1ddf21fd627c76a8b7e0d81ad43480b1f1e204d Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Thu, 15 Dec 2016 11:58:28 +0300
Subject: [PATCH 051/446] IGNITE-4277: Hadoop: implemented "partially raw"
 comparator. This closes #1345.

---
 .../processors/hadoop/HadoopClassLoader.java  |   1 +
 .../processors/hadoop/HadoopJobProperty.java  |   6 +-
 .../processors/hadoop/HadoopTaskContext.java  |   8 ++
 .../io/PartiallyOffheapRawComparatorEx.java   |  33 +++++
 .../hadoop/io/PartiallyRawComparator.java     |  33 +++++
 .../apache/ignite/hadoop/io/RawMemory.java    |  86 ++++++++++++
 .../hadoop/io/TextPartiallyRawComparator.java | 115 +++++++++++++++
 .../apache/ignite/hadoop/io/package-info.java |  22 +++
 ...legatingPartiallyOffheapRawComparator.java |  54 ++++++++
 .../hadoop/impl/v2/HadoopV2TaskContext.java   |  21 +++
 .../hadoop/io/OffheapRawMemory.java           | 131 ++++++++++++++++++
 .../shuffle/collections/HadoopSkipList.java   |  14 +-
 .../hadoop/impl/HadoopTeraSortTest.java       |   7 +
 .../collections/HadoopAbstractMapTest.java    |   6 +
 14 files changed, 535 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/io/PartiallyOffheapRawComparatorEx.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/PartiallyRawComparator.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/RawMemory.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/package-info.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2DelegatingPartiallyOffheapRawComparator.java
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/io/OffheapRawMemory.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index f6c2fa98ddb4a..81c1405644707 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -372,6 +372,7 @@ public static boolean loadByCurrentClassloader(String clsName) {
             // We use "contains" instead of "equals" to handle subclasses properly.
             if (clsName.contains("org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem") ||
                 clsName.contains("org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem") ||
+                clsName.contains("org.apache.ignite.hadoop.io.TextPartialRawComparator") ||
                 clsName.contains("org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider"))
                 return true;
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index 9e1dedec4a62a..4122eefe0087f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -63,6 +63,11 @@ public enum HadoopJobProperty {
      */
     JOB_SHARED_CLASSLOADER("ignite.job.shared.classloader"),
 
+    /**
+     * Fully qualified name of partially-raw comparator which should be used on sorting phase.
+     */
+    JOB_PARTIAL_RAW_COMPARATOR("ignite.job.partial.raw.comparator"),
+
     /**
      * Size in bytes of single memory page which will be allocated for data structures in shuffle.
      * <p>
@@ -112,7 +117,6 @@ public enum HadoopJobProperty {
      */
     SHUFFLE_JOB_THROTTLE("ignite.shuffle.job.throttle");
 
-
     /** Property name. */
     private final String propName;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
index ecb9f26db5f1c..dddd0176e32fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounter;
 import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
 
 /**
  * Task context.
@@ -156,6 +157,13 @@ public void output(HadoopTaskOutput out) {
      */
     public abstract Comparator<Object> sortComparator();
 
+    /**
+     * Get semi-raw sorting comparator.
+     *
+     * @return Semi-raw sorting comparator.
+     */
+    public abstract PartiallyOffheapRawComparatorEx<Object> partialRawSortComparator();
+
     /**
      * Gets comparator for grouping on combine or reduce operation.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/io/PartiallyOffheapRawComparatorEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/io/PartiallyOffheapRawComparatorEx.java
new file mode 100644
index 0000000000000..157609e40959e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/io/PartiallyOffheapRawComparatorEx.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.io;
+
+/**
+ * Special version of raw comparator allowing direct access to the underlying memory.
+ */
+public interface PartiallyOffheapRawComparatorEx<T> {
+    /**
+     * Perform compare.
+     *
+     * @param val1 First value.
+     * @param val2Ptr Pointer to the second value data.
+     * @param val2Len Length of the second value data.
+     * @return Result.
+     */
+    int compare(T val1, long val2Ptr, int val2Len);
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/PartiallyRawComparator.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/PartiallyRawComparator.java
new file mode 100644
index 0000000000000..b9a450542106b
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/PartiallyRawComparator.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.io;
+
+/**
+ * Partially raw comparator. Compares one deserialized value with serialized value.
+ */
+public interface PartiallyRawComparator<T> {
+    /**
+     * Do compare.
+     *
+     * @param val1 First value (deserialized).
+     * @param val2Buf Second value (serialized).
+     * @return A negative integer, zero, or a positive integer as this object is less than, equal to, or greater
+     *     than the specified object.
+     */
+    int compare(T val1, RawMemory val2Buf);
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/RawMemory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/RawMemory.java
new file mode 100644
index 0000000000000..8dcaf8374146b
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/RawMemory.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.io;
+
+/**
+ * Memory abstraction for raw comparison.
+ */
+public interface RawMemory {
+    /**
+     * Get byte value at the given index.
+     *
+     * @param idx Index.
+     * @return Value.
+     */
+    byte get(int idx);
+
+    /**
+     * Get short value at the given index.
+     *
+     * @param idx Index.
+     * @return Value.
+     */
+    short getShort(int idx);
+
+    /**
+     * Get char value at the given index.
+     *
+     * @param idx Index.
+     * @return Value.
+     */
+    char getChar(int idx);
+
+    /**
+     * Get int value at the given index.
+     *
+     * @param idx Index.
+     * @return Value.
+     */
+    int getInt(int idx);
+
+    /**
+     * Get long value at the given index.
+     *
+     * @param idx Index.
+     * @return Value.
+     */
+    long getLong(int idx);
+
+    /**
+     * Get float value at the given index.
+     *
+     * @param idx Index.
+     * @return Value.
+     */
+    float getFloat(int idx);
+
+    /**
+     * Get double value at the given index.
+     *
+     * @param idx Index.
+     * @return Value.
+     */
+    double getDouble(int idx);
+
+    /**
+     * Get length.
+     *
+     * @return Length.
+     */
+    int length();
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java
new file mode 100644
index 0000000000000..a2bc3d4c8e249
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.io;
+
+import com.google.common.primitives.Longs;
+import com.google.common.primitives.UnsignedBytes;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.ignite.internal.processors.hadoop.io.OffheapRawMemory;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
+import org.apache.ignite.internal.util.GridUnsafe;
+
+/**
+ * Partial raw comparator for {@link Text} data type.
+ * <p>
+ * Implementation is borrowed from {@code org.apache.hadoop.io.FastByteComparisons} and adopted to Ignite
+ * infrastructure.
+ */
+public class TextPartiallyRawComparator implements PartiallyRawComparator<Text>, PartiallyOffheapRawComparatorEx<Text> {
+    /** {@inheritDoc} */
+    @Override public int compare(Text val1, RawMemory val2Buf) {
+        if (val2Buf instanceof OffheapRawMemory) {
+            OffheapRawMemory val2Buf0 = (OffheapRawMemory)val2Buf;
+
+            return compare(val1, val2Buf0.pointer(), val2Buf0.length());
+        }
+        else
+            throw new UnsupportedOperationException("Text can be compared only with offheap memory.");
+    }
+
+    /** {@inheritDoc} */
+    @Override public int compare(Text val1, long val2Ptr, int val2Len) {
+        int len2 = WritableUtils.decodeVIntSize(GridUnsafe.getByte(val2Ptr));
+
+        return compareBytes(val1.getBytes(), val1.getLength(), val2Ptr + len2, val2Len - len2);
+    }
+
+    /**
+     * Internal comparison routine.
+     *
+     * @param buf1 Bytes 1.
+     * @param len1 Length 1.
+     * @param ptr2 Pointer 2.
+     * @param len2 Length 2.
+     * @return Result.
+     */
+    @SuppressWarnings("SuspiciousNameCombination")
+    private static int compareBytes(byte[] buf1, int len1, long ptr2, int len2) {
+        int minLength = Math.min(len1, len2);
+
+        int minWords = minLength / Longs.BYTES;
+
+        for (int i = 0; i < minWords * Longs.BYTES; i += Longs.BYTES) {
+            long lw = GridUnsafe.getLong(buf1, GridUnsafe.BYTE_ARR_OFF + i);
+            long rw = GridUnsafe.getLong(ptr2 + i);
+
+            long diff = lw ^ rw;
+
+            if (diff != 0) {
+                if (GridUnsafe.BIG_ENDIAN)
+                    return (lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE) ? -1 : 1;
+
+                // Use binary search
+                int n = 0;
+                int y;
+                int x = (int) diff;
+
+                if (x == 0) {
+                    x = (int) (diff >>> 32);
+
+                    n = 32;
+                }
+
+                y = x << 16;
+
+                if (y == 0)
+                    n += 16;
+                else
+                    x = y;
+
+                y = x << 8;
+
+                if (y == 0)
+                    n += 8;
+
+                return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
+            }
+        }
+
+        // The epilogue to cover the last (minLength % 8) elements.
+        for (int i = minWords * Longs.BYTES; i < minLength; i++) {
+            int res = UnsignedBytes.compare(buf1[i], GridUnsafe.getByte(ptr2 + i));
+
+            if (res != 0)
+                return res;
+        }
+
+        return len1 - len2;
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/package-info.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/package-info.java
new file mode 100644
index 0000000000000..0d1f7b9443cbd
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains <b>Hadoop Accelerator</b> API for input-output operations.
+ */
+package org.apache.ignite.hadoop.io;
\ No newline at end of file
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2DelegatingPartiallyOffheapRawComparator.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2DelegatingPartiallyOffheapRawComparator.java
new file mode 100644
index 0000000000000..e6d369e1e52a1
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2DelegatingPartiallyOffheapRawComparator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.v2;
+
+import org.apache.ignite.hadoop.io.PartiallyRawComparator;
+import org.apache.ignite.internal.processors.hadoop.io.OffheapRawMemory;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
+
+/**
+ * Delegating partial raw comparator.
+ */
+public class HadoopV2DelegatingPartiallyOffheapRawComparator<T> implements PartiallyOffheapRawComparatorEx<T> {
+    /** Target comparator. */
+    private final PartiallyRawComparator<T> target;
+
+    /** Memory. */
+    private OffheapRawMemory mem;
+
+    /**
+     * Constructor.
+     *
+     * @param target Target.
+     */
+    public HadoopV2DelegatingPartiallyOffheapRawComparator(PartiallyRawComparator<T> target) {
+        assert target != null;
+
+        this.target = target;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int compare(T val1, long val2Ptr, int val2Len) {
+        if (mem == null)
+            mem = new OffheapRawMemory(val2Ptr, val2Len);
+        else
+            mem.update(val2Ptr, val2Len);
+
+        return target.compare(val1, mem);
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
index d444f2b6be4b8..42bbec5cffd89 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
@@ -38,13 +38,16 @@
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.hadoop.io.PartiallyRawComparator;
 import org.apache.ignite.internal.processors.hadoop.HadoopClassLoader;
 import org.apache.ignite.internal.processors.hadoop.HadoopCommonUtils;
 import org.apache.ignite.internal.processors.hadoop.HadoopExternalSplit;
 import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
 import org.apache.ignite.internal.processors.hadoop.HadoopJob;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty;
 import org.apache.ignite.internal.processors.hadoop.HadoopPartitioner;
 import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
 import org.apache.ignite.internal.processors.hadoop.HadoopSplitWrapper;
@@ -62,6 +65,7 @@
 import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1Partitioner;
 import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1ReduceTask;
 import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1SetupTask;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
 import org.apache.ignite.internal.processors.igfs.IgfsUtils;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.A;
@@ -421,10 +425,27 @@ private HadoopSerialization getSerialization(Class<?> cls, Configuration jobConf
     }
 
     /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
     @Override public Comparator<Object> sortComparator() {
         return (Comparator<Object>)jobCtx.getSortComparator();
     }
 
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override public PartiallyOffheapRawComparatorEx<Object> partialRawSortComparator() {
+        Class cls = jobCtx.getJobConf().getClass(HadoopJobProperty.JOB_PARTIAL_RAW_COMPARATOR.propertyName(), null);
+
+        if (cls == null)
+            return null;
+
+        Object res = ReflectionUtils.newInstance(cls, jobConf());
+
+        if (res instanceof PartiallyOffheapRawComparatorEx)
+            return (PartiallyOffheapRawComparatorEx)res;
+        else
+            return new HadoopV2DelegatingPartiallyOffheapRawComparator<>((PartiallyRawComparator)res);
+    }
+
     /** {@inheritDoc} */
     @Override public Comparator<Object> groupComparator() {
         Comparator<?> res;
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/io/OffheapRawMemory.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/io/OffheapRawMemory.java
new file mode 100644
index 0000000000000..564f92cc56378
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/io/OffheapRawMemory.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.io;
+
+import org.apache.ignite.hadoop.io.RawMemory;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Offheap-based memory.
+ */
+public class OffheapRawMemory implements RawMemory {
+    /** Pointer. */
+    private long ptr;
+
+    /** Length. */
+    private int len;
+
+    /**
+     * Constructor.
+     *
+     * @param ptr Pointer.
+     * @param len Length.
+     */
+    public OffheapRawMemory(long ptr, int len) {
+        update(ptr, len);
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte get(int idx) {
+        ensure(idx, 1);
+
+        return GridUnsafe.getByte(ptr + idx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public short getShort(int idx) {
+        ensure(idx, 2);
+
+        return GridUnsafe.getShort(ptr + idx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public char getChar(int idx) {
+        ensure(idx, 2);
+
+        return GridUnsafe.getChar(ptr + idx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getInt(int idx) {
+        ensure(idx, 4);
+
+        return GridUnsafe.getInt(ptr + idx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getLong(int idx) {
+        ensure(idx, 8);
+
+        return GridUnsafe.getLong(ptr + idx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public float getFloat(int idx) {
+        ensure(idx, 4);
+
+        return GridUnsafe.getFloat(ptr + idx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public double getDouble(int idx) {
+        ensure(idx, 8);
+
+        return GridUnsafe.getDouble(ptr + idx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int length() {
+        return len;
+    }
+
+    /**
+     * @return Raw pointer.
+     */
+    public long pointer() {
+        return ptr;
+    }
+
+    /**
+     * Update pointer and length.
+     *
+     * @param ptr Pointer.
+     * @param len Length.
+     */
+    public void update(long ptr, int len) {
+        this.ptr = ptr;
+        this.len = len;
+    }
+
+    /**
+     * Ensure that the given number of bytes are available for read. Throw an exception otherwise.
+     *
+     * @param idx Index.
+     * @param cnt Count.
+     */
+    private void ensure(int idx, int cnt) {
+        if (idx < 0 || idx + cnt - 1 >= len)
+            throw new IndexOutOfBoundsException("Illegal index [len=" + len + ", idx=" + idx + ']');
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(OffheapRawMemory.class, this);
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java
index 7db88bce818ac..f300a1890afb7 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInput;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.GridRandom;
 import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory;
@@ -279,6 +280,9 @@ private class AdderImpl extends AdderBase {
         /** */
         private final Comparator<Object> cmp;
 
+        /** */
+        private final PartiallyOffheapRawComparatorEx<Object> partialRawCmp;
+
         /** */
         private final Random rnd = new GridRandom();
 
@@ -298,6 +302,7 @@ protected AdderImpl(HadoopTaskContext ctx) throws IgniteCheckedException {
             keyReader = new Reader(keySer);
 
             cmp = ctx.sortComparator();
+            partialRawCmp = ctx.partialRawSortComparator();
         }
 
         /** {@inheritDoc} */
@@ -475,7 +480,14 @@ private long add(Object key, @Nullable Object val) throws IgniteCheckedException
         private int cmp(Object key, long meta) {
             assert meta != 0;
 
-            return cmp.compare(key, keyReader.readKey(meta));
+            if (partialRawCmp != null) {
+                long keyPtr = key(meta);
+                int keySize = keySize(keyPtr);
+
+                return partialRawCmp.compare(key, keyPtr + 4, keySize);
+            }
+            else
+                return cmp.compare(key, keyReader.readKey(meta));
         }
 
         /**
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
index 0cc95645772f6..a016506d77073 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
@@ -41,8 +41,10 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.configuration.HadoopConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.hadoop.io.TextPartiallyRawComparator;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty;
 
 import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.createJobInfo;
 
@@ -161,6 +163,11 @@ protected final void teraSort() throws Exception {
         jobConf.set("mapred.min.split.size", String.valueOf(splitSize));
         jobConf.set("mapred.max.split.size", String.valueOf(splitSize));
 
+        jobConf.setBoolean(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), true);
+
+        jobConf.set(HadoopJobProperty.JOB_PARTIAL_RAW_COMPARATOR.propertyName(),
+            TextPartiallyRawComparator.class.getName());
+
         Job job = setupConfig(jobConf);
 
         HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 1);
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopAbstractMapTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopAbstractMapTest.java
index 9d1fd4f710184..1f8978dfcc6b4 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopAbstractMapTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopAbstractMapTest.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounter;
 import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
 import org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopWritableSerialization;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.Nullable;
 
@@ -83,6 +84,11 @@ protected TaskContext() {
             return ComparableComparator.getInstance();
         }
 
+        /** {@inheritDoc} */
+        @Override public PartiallyOffheapRawComparatorEx<Object> partialRawSortComparator() {
+            return null;
+        }
+
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
         @Override public Comparator<Object> groupComparator() {

From f8ac0f14986169ee25159734d0a97b08976c5751 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Thu, 15 Dec 2016 12:13:10 +0300
Subject: [PATCH 052/446] IGNITE-4277: Hadoop: better property naming for
 "partially raw" comparator.

---
 .../ignite/internal/processors/hadoop/HadoopJobProperty.java  | 4 ++--
 .../processors/hadoop/impl/v2/HadoopV2TaskContext.java        | 2 +-
 .../internal/processors/hadoop/impl/HadoopTeraSortTest.java   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index 4122eefe0087f..9dd430b902ec3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -64,9 +64,9 @@ public enum HadoopJobProperty {
     JOB_SHARED_CLASSLOADER("ignite.job.shared.classloader"),
 
     /**
-     * Fully qualified name of partially-raw comparator which should be used on sorting phase.
+     * Fully qualified name of partially raw comparator which should be used on sorting phase.
      */
-    JOB_PARTIAL_RAW_COMPARATOR("ignite.job.partial.raw.comparator"),
+    JOB_PARTIALLY_RAW_COMPARATOR("ignite.job.partially.raw.comparator"),
 
     /**
      * Size in bytes of single memory page which will be allocated for data structures in shuffle.
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
index 42bbec5cffd89..e9cae1c068a52 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
@@ -433,7 +433,7 @@ private HadoopSerialization getSerialization(Class<?> cls, Configuration jobConf
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public PartiallyOffheapRawComparatorEx<Object> partialRawSortComparator() {
-        Class cls = jobCtx.getJobConf().getClass(HadoopJobProperty.JOB_PARTIAL_RAW_COMPARATOR.propertyName(), null);
+        Class cls = jobCtx.getJobConf().getClass(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(), null);
 
         if (cls == null)
             return null;
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
index a016506d77073..b1fa91f0e4707 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
@@ -165,7 +165,7 @@ protected final void teraSort() throws Exception {
 
         jobConf.setBoolean(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), true);
 
-        jobConf.set(HadoopJobProperty.JOB_PARTIAL_RAW_COMPARATOR.propertyName(),
+        jobConf.set(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(),
             TextPartiallyRawComparator.class.getName());
 
         Job job = setupConfig(jobConf);

From 57eb47f9c057d15739987500a8b3894d1d739d29 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Thu, 15 Dec 2016 12:29:51 +0300
Subject: [PATCH 053/446] IGNITE-4277: Hadoop: better property naming for
 "partially raw" comparator (2).

---
 .../ignite/internal/processors/hadoop/HadoopClassLoader.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 81c1405644707..1209e746076e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -372,7 +372,7 @@ public static boolean loadByCurrentClassloader(String clsName) {
             // We use "contains" instead of "equals" to handle subclasses properly.
             if (clsName.contains("org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem") ||
                 clsName.contains("org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem") ||
-                clsName.contains("org.apache.ignite.hadoop.io.TextPartialRawComparator") ||
+                clsName.contains("org.apache.ignite.hadoop.io.TextPartiallyRawComparator") ||
                 clsName.contains("org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider"))
                 return true;
         }

From 05dd08b993e2d7f88176c051463b178431714f85 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 9 Dec 2016 12:28:47 +0300
Subject: [PATCH 054/446] IGNITE-3220 I/O bottleneck on server/client cluster
 configuration Communications optimizations: - possibility to open separate
 in/out connections - possibility to have multiple connections between nodes -
 implemented NIO sessions balancing between NIO threads - reduced amount of
 work and blocking calls in NIO threads Other: - implemented StripedExecutor
 for cache messages handling - added 'io test' messages for IO performance
 testing

(cherry picked from commit 10ade28)
---
 .../ignite/examples/ExampleNodeStartup.java   |    2 +-
 .../examples/datagrid/CachePutGetExample.java |    2 +-
 .../examples/ScalarJvmCloudExample.scala      |    2 +-
 .../rest/ClientMemcachedProtocolSelfTest.java |    4 +-
 .../rest/protocols/tcp/MockNioSession.java    |   25 +-
 .../apache/ignite/IgniteSystemProperties.java |    3 +
 .../store/CacheLoadOnlyStoreAdapter.java      |    6 +-
 .../configuration/IgniteConfiguration.java    |   50 +-
 .../internal/GridEventConsumeHandler.java     |    2 +-
 .../ignite/internal/GridJobContextImpl.java   |    4 +-
 .../ignite/internal/GridKernalContext.java    |    9 +
 .../internal/GridKernalContextImpl.java       |   16 +-
 .../internal/GridPerformanceSuggestions.java  |    2 +-
 .../org/apache/ignite/internal/GridTopic.java |    5 +-
 .../ignite/internal/IgniteInternalFuture.java |   11 +
 .../apache/ignite/internal/IgniteKernal.java  |   85 +-
 .../apache/ignite/internal/IgnitionEx.java    |   32 +-
 .../GridClientConnectionManagerAdapter.java   |    1 +
 .../client/router/impl/GridTcpRouterImpl.java |    1 +
 .../managers/communication/GridIoManager.java |  207 ++-
 .../managers/communication/GridIoMessage.java |   13 +
 .../communication/GridIoMessageFactory.java   |   12 +-
 .../communication/IgniteIoTestMessage.java    |  235 +++
 .../processors/cache/GridCacheAdapter.java    |   26 +-
 .../processors/cache/GridCacheMessage.java    |    7 +
 .../processors/cache/GridCacheUtils.java      |   35 +
 .../processors/cache/IgniteCacheProxy.java    |    8 +
 .../GridDistributedLockRequest.java           |    5 +
 .../GridDistributedTxFinishResponse.java      |    6 +
 .../GridDistributedUnlockRequest.java         |    5 +
 .../distributed/dht/GridDhtCacheAdapter.java  |    3 +-
 .../distributed/dht/GridDhtLockResponse.java  |    9 +-
 .../dht/atomic/GridDhtAtomicCache.java        |    5 +-
 .../GridDhtAtomicSingleUpdateRequest.java     |    5 +
 .../atomic/GridDhtAtomicUpdateRequest.java    |    5 +
 .../GridNearAtomicFullUpdateRequest.java      |    5 +
 .../GridNearAtomicSingleUpdateRequest.java    |    5 +
 .../distributed/near/GridNearGetRequest.java  |    5 +
 .../local/atomic/GridLocalAtomicCache.java    |    3 +
 .../GridCacheDistributedQueryManager.java     |    2 +-
 .../cache/query/GridCacheQueryRequest.java    |    6 +-
 .../transactions/IgniteTxLocalAdapter.java    |    8 +-
 .../datastreamer/DataStreamProcessor.java     |   22 +-
 .../internal/processors/igfs/IgfsContext.java |    4 +-
 .../processors/igfs/IgfsDataManager.java      |    6 +-
 .../internal/processors/igfs/IgfsImpl.java    |    2 +-
 .../processors/odbc/OdbcProcessor.java        |    1 +
 .../platform/compute/PlatformCompute.java     |    6 +
 .../tcp/GridTcpMemcachedNioListener.java      |   15 +-
 .../protocols/tcp/GridTcpRestNioListener.java |    2 +-
 .../protocols/tcp/GridTcpRestProtocol.java    |    1 +
 .../service/GridServiceProcessor.java         |    6 +-
 .../ignite/internal/util/IgniteUtils.java     |   62 +-
 .../ignite/internal/util/StripedExecutor.java |  667 ++++++++
 .../util/future/GridFinishedFuture.java       |   24 +
 .../util/future/GridFutureAdapter.java        |   15 +-
 .../util/future/GridFutureChainListener.java  |   30 +-
 .../internal/util/ipc/IpcToNioAdapter.java    |    2 +-
 .../nio/GridAbstractCommunicationClient.java  |   12 +-
 .../util/nio/GridCommunicationClient.java     |    9 +-
 .../nio/GridConnectionBytesVerifyFilter.java  |   15 +-
 .../util/nio/GridNioAsyncNotifyFilter.java    |   10 +-
 .../internal/util/nio/GridNioCodecFilter.java |   17 +-
 .../internal/util/nio/GridNioFilter.java      |   16 +-
 .../util/nio/GridNioFilterAdapter.java        |   10 +-
 .../internal/util/nio/GridNioFilterChain.java |   14 +-
 .../internal/util/nio/GridNioFuture.java      |    4 +-
 .../util/nio/GridNioRecoveryDescriptor.java   |  124 +-
 .../internal/util/nio/GridNioServer.java      | 1404 ++++++++++++++---
 .../internal/util/nio/GridNioSession.java     |   25 +-
 .../internal/util/nio/GridNioSessionImpl.java |   65 +-
 .../internal/util/nio/GridNioWorker.java      |   48 +
 .../util/nio/GridSelectorNioSessionImpl.java  |  221 ++-
 .../nio/GridShmemCommunicationClient.java     |    7 +-
 .../nio/GridTcpNioCommunicationClient.java    |   55 +-
 .../util/nio/SessionWriteRequest.java         |   85 +
 .../util/nio/ssl/GridNioSslFilter.java        |   10 +-
 .../util/nio/ssl/GridNioSslHandler.java       |    4 +-
 .../util/tostring/GridToStringBuilder.java    |    2 +-
 .../tcp/TcpCommunicationSpi.java              | 1340 ++++++++++++----
 .../tcp/TcpCommunicationSpiMBean.java         |   40 +
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   14 +-
 .../ignite/stream/socket/SocketStreamer.java  |    1 +
 .../ignite/thread/IgniteThreadFactory.java    |    8 +-
 .../IgniteSlowClientDetectionSelfTest.java    |    1 +
 ...icationBalanceMultipleConnectionsTest.java |   28 +
 .../IgniteCommunicationBalanceTest.java       |  339 ++++
 .../IgniteIoTestMessagesTest.java             |   95 ++
 .../IgniteVariousConnectionNumberTest.java    |  166 ++
 .../CrossCacheTxRandomOperationsTest.java     |   30 +-
 ...AbstractCacheInterceptorRebalanceTest.java |    4 +-
 ...cheOffHeapMultiThreadedUpdateSelfTest.java |    6 +-
 ...tomicMessageRecovery10ConnectionsTest.java |   28 +
 ...essageRecoveryNoPairedConnectionsTest.java |   47 +
 ...heConnectionRecovery10ConnectionsTest.java |   35 +
 .../distributed/IgniteCacheCreatePutTest.java |    2 +-
 ...gniteCacheMessageRecoveryAbstractTest.java |   24 +-
 .../IgniteCacheMessageWriteTimeoutTest.java   |   17 +-
 .../dht/IgniteCacheMultiTxLockSelfTest.java   |    6 +-
 ...NoStripedPoolMultiNodeFullApiSelfTest.java |   35 +
 ...NoStripedPoolMultiNodeFullApiSelfTest.java |   35 +
 .../TxDeadlockDetectionNoHangsTest.java       |    2 +-
 .../TxOptimisticDeadlockDetectionTest.java    |   29 +-
 .../GridServiceProcessorProxySelfTest.java    |    2 +-
 .../future/GridFutureAdapterSelfTest.java     |  122 +-
 .../nio/impl/GridNioFilterChainSelfTest.java  |   32 +-
 .../loadtests/nio/GridNioBenchmarkClient.java |    4 +-
 .../p2p/GridP2PRecursionTaskSelfTest.java     |    2 +-
 .../spi/GridTcpSpiForwardingSelfTest.java     |   18 +-
 .../GridTcpCommunicationSpiAbstractTest.java  |   28 +-
 ...unicationSpiConcurrentConnectSelfTest.java |   82 +-
 ...GridTcpCommunicationSpiConfigSelfTest.java |    5 +-
 ...CommunicationSpiMultithreadedSelfTest.java |   23 +-
 ...cpCommunicationSpiRecoveryAckSelfTest.java |    9 +-
 ...onSpiRecoveryFailureDetectionSelfTest.java |    1 +
 ...ionSpiRecoveryNoPairedConnectionsTest.java |   28 +
 ...idTcpCommunicationSpiRecoverySelfTest.java |   67 +-
 ...mmunicationRecoveryAckClosureSelfTest.java |    9 +-
 .../junits/GridTestKernalContext.java         |    4 +-
 .../IgniteCacheFullApiSelfTestSuite.java      |    6 +
 .../testsuites/IgniteCacheTestSuite.java      |   17 +-
 .../IgniteSpiCommunicationSelfTestSuite.java  |    2 +
 .../hadoop/jobtracker/HadoopJobTracker.java   |    4 +-
 .../HadoopExternalCommunication.java          |    5 +-
 .../communication/HadoopIpcToNioAdapter.java  |    2 +-
 .../communication/HadoopMarshallerFilter.java |    6 +-
 .../ignite/stream/kafka/KafkaStreamer.java    |    2 +-
 .../tools/classgen/ClassesGenerator.java      |    8 +-
 .../yardstick/IgniteBenchmarkUtils.java       |    6 +-
 .../yardstick/cache/CacheEntryEventProbe.java |    2 +-
 .../cache/IgniteIoTestBenchmark.java          |   73 +
 .../io/IgniteIoTestAbstractBenchmark.java     |   61 +
 .../io/IgniteIoTestSendAllBenchmark.java      |   32 +
 .../io/IgniteIoTestSendRandomBenchmark.java   |   35 +
 134 files changed, 5935 insertions(+), 998 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioWorker.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/nio/SessionWriteRequest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessagesTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteVariousConnectionNumberTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecovery10ConnectionsTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecovery10ConnectionsTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest.java
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestAbstractBenchmark.java
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendAllBenchmark.java
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendRandomBenchmark.java

diff --git a/examples/src/main/java/org/apache/ignite/examples/ExampleNodeStartup.java b/examples/src/main/java/org/apache/ignite/examples/ExampleNodeStartup.java
index ad122973967d3..dd8a72be8cf0a 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ExampleNodeStartup.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ExampleNodeStartup.java
@@ -33,4 +33,4 @@ public class ExampleNodeStartup {
     public static void main(String[] args) throws IgniteException {
         Ignition.start("examples/config/example-ignite.xml");
     }
-}
\ No newline at end of file
+}
diff --git a/examples/src/main/java/org/apache/ignite/examples/datagrid/CachePutGetExample.java b/examples/src/main/java/org/apache/ignite/examples/datagrid/CachePutGetExample.java
index 82a76b87b9ab2..b9bae5b0d8834 100644
--- a/examples/src/main/java/org/apache/ignite/examples/datagrid/CachePutGetExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/datagrid/CachePutGetExample.java
@@ -105,4 +105,4 @@ private static void putAllGetAll(IgniteCache<Integer, String> cache) throws Igni
         for (Map.Entry<Integer, String> e : vals.entrySet())
             System.out.println("Got entry [key=" + e.getKey() + ", val=" + e.getValue() + ']');
     }
-}
\ No newline at end of file
+}
diff --git a/examples/src/main/scala/org/apache/ignite/scalar/examples/ScalarJvmCloudExample.scala b/examples/src/main/scala/org/apache/ignite/scalar/examples/ScalarJvmCloudExample.scala
index 1014726549195..814bb2e99611d 100644
--- a/examples/src/main/scala/org/apache/ignite/scalar/examples/ScalarJvmCloudExample.scala
+++ b/examples/src/main/scala/org/apache/ignite/scalar/examples/ScalarJvmCloudExample.scala
@@ -50,7 +50,7 @@ object ScalarJvmCloudExample {
             val pool = Executors.newFixedThreadPool(NODES.size)
 
             // Concurrently startup all nodes.
-            NODES.foreach(name => pool.submit(new Runnable {
+            NODES.foreach(name => pool.execute(new Runnable {
                 @impl def run() {
                     // All defaults.
                     val cfg = new IgniteConfiguration
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/ClientMemcachedProtocolSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/ClientMemcachedProtocolSelfTest.java
index 0f56c73786e27..c03c06ea1750c 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/ClientMemcachedProtocolSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/ClientMemcachedProtocolSelfTest.java
@@ -111,6 +111,8 @@ public void testGetBulk() throws Exception {
 
         Map<String, Object> map = client.getBulk("getKey1", "getKey2");
 
+        info("Map: " + map);
+
         Assert.assertEquals(2, map.size());
 
         Assert.assertEquals("getVal1", map.get("getKey1"));
@@ -443,4 +445,4 @@ private ValueObject(int intVal, String strVal) {
             return res;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/protocols/tcp/MockNioSession.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/protocols/tcp/MockNioSession.java
index c82c73e36784c..9bc4e7f224d6b 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/protocols/tcp/MockNioSession.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/protocols/tcp/MockNioSession.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.rest.protocols.tcp;
 
 import java.net.InetSocketAddress;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
 import org.apache.ignite.internal.util.nio.GridNioFinishedFuture;
 import org.apache.ignite.internal.util.nio.GridNioFuture;
@@ -110,6 +111,11 @@ public MockNioSession(InetSocketAddress locAddr, InetSocketAddress rmtAddr) {
         return new GridNioFinishedFuture<>(true);
     }
 
+    /** {@inheritDoc} */
+    @Override public void sendNoFuture(Object msg) throws IgniteCheckedException {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public GridNioFuture<Object> resumeReads() {
         return null;
@@ -131,12 +137,27 @@ public MockNioSession(InetSocketAddress locAddr, InetSocketAddress rmtAddr) {
     }
 
     /** {@inheritDoc} */
-    @Override public void recoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+    @Override public void outRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void inRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
         // No-op.
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public GridNioRecoveryDescriptor recoveryDescriptor() {
+    @Nullable @Override public GridNioRecoveryDescriptor outRecoveryDescriptor() {
         return null;
     }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public GridNioRecoveryDescriptor inRecoveryDescriptor() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void systemMessage(Object msg) {
+        // No-op.
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index d0c0d5ef96652..9650a31fefe5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -477,6 +477,9 @@ public final class IgniteSystemProperties {
     @Deprecated
     public static final String IGNITE_BINARY_DONT_WRAP_TREE_STRUCTURES = "IGNITE_BINARY_DONT_WRAP_TREE_STRUCTURES";
 
+    /** */
+    public static final String IGNITE_IO_BALANCE_PERIOD = "IGNITE_IO_BALANCE_PERIOD";
+
     /**
      * When set to {@code true} fields are written by BinaryMarshaller in sorted order. Otherwise
      * the natural order is used.
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/CacheLoadOnlyStoreAdapter.java b/modules/core/src/main/java/org/apache/ignite/cache/store/CacheLoadOnlyStoreAdapter.java
index 7494e379621de..d3f381e77d136 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/CacheLoadOnlyStoreAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/CacheLoadOnlyStoreAdapter.java
@@ -153,14 +153,14 @@ public abstract class CacheLoadOnlyStoreAdapter<K, V, I> implements CacheStore<K
                 buf.add(iter.next());
 
                 if (buf.size() == batchSize) {
-                    exec.submit(new Worker(c, buf, args));
+                    exec.execute(new Worker(c, buf, args));
 
                     buf = new ArrayList<>(batchSize);
                 }
             }
 
             if (!buf.isEmpty())
-                exec.submit(new Worker(c, buf, args));
+                exec.execute(new Worker(c, buf, args));
         }
         catch (RejectedExecutionException ignored) {
             // Because of custom RejectedExecutionHandler.
@@ -330,4 +330,4 @@ private class BlockingRejectedExecutionHandler implements RejectedExecutionHandl
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index 75145a302bf8d..dcd8a801aa808 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -146,7 +146,7 @@ public class IgniteConfiguration {
     public static final int AVAILABLE_PROC_CNT = Runtime.getRuntime().availableProcessors();
 
     /** Default core size of public thread pool. */
-    public static final int DFLT_PUBLIC_THREAD_CNT = Math.max(8, AVAILABLE_PROC_CNT) * 2;
+    public static final int DFLT_PUBLIC_THREAD_CNT = Math.max(8, AVAILABLE_PROC_CNT);
 
     /** Default keep alive time for public thread pool. */
     @Deprecated
@@ -236,6 +236,12 @@ public class IgniteConfiguration {
     /** Async Callback pool size. */
     private int callbackPoolSize = DFLT_PUBLIC_THREAD_CNT;
 
+    /**
+     * Use striped pool for internal requests processing when possible
+     * (e.g. cache requests per-partition striping).
+     */
+    private int stripedPoolSize = DFLT_PUBLIC_THREAD_CNT;
+
     /** System pool size. */
     private int sysPoolSize = DFLT_SYSTEM_CORE_THREAD_CNT;
 
@@ -553,6 +559,7 @@ public IgniteConfiguration(IgniteConfiguration cfg) {
         sndRetryDelay = cfg.getNetworkSendRetryDelay();
         sslCtxFactory = cfg.getSslContextFactory();
         storeSesLsnrs = cfg.getCacheStoreSessionListenerFactories();
+        stripedPoolSize = cfg.getStripedPoolSize();
         svcCfgs = cfg.getServiceConfiguration();
         sysPoolSize = cfg.getSystemThreadPoolSize();
         timeSrvPortBase = cfg.getTimeServerPortBase();
@@ -711,6 +718,47 @@ public IgniteConfiguration setGridLogger(IgniteLogger log) {
         return this;
     }
 
+    /**
+     * Returns striped pool size that should be used for cache requests
+     * processing.
+     * <p>
+     * If set to non-positive value then requests get processed in system pool.
+     * <p>
+     * Striped pool is better for typical cache operations.
+     *
+     * @return Positive value if striped pool should be initialized
+     *      with configured number of threads (stripes) and used for requests processing
+     *      or non-positive value to process requests in system pool.
+     *
+     * @see #getPublicThreadPoolSize()
+     * @see #getSystemThreadPoolSize()
+     */
+    public int getStripedPoolSize() {
+        return stripedPoolSize;
+    }
+
+    /**
+     * Sets striped pool size that should be used for cache requests
+     * processing.
+     * <p>
+     * If set to non-positive value then requests get processed in system pool.
+     * <p>
+     * Striped pool is better for typical cache operations.
+     *
+     * @param stripedPoolSize Positive value if striped pool should be initialized
+     *      with passed in number of threads (stripes) and used for requests processing
+     *      or non-positive value to process requests in system pool.
+     * @return {@code this} for chaining.
+     *
+     * @see #getPublicThreadPoolSize()
+     * @see #getSystemThreadPoolSize()
+     */
+    public IgniteConfiguration setStripedPoolSize(int stripedPoolSize) {
+        this.stripedPoolSize = stripedPoolSize;
+
+        return this;
+    }
+
     /**
      * Should return a thread pool size to be used in grid.
      * This executor service will be in charge of processing {@link ComputeJob GridJobs}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
index 715f8a5e48581..68d34ceb005f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
@@ -181,7 +181,7 @@ public GridEventConsumeHandler() {
                         notificationQueue.add(new T3<>(nodeId, routineId, evt));
 
                         if (!notificationInProgress) {
-                            ctx.getSystemExecutorService().submit(new Runnable() {
+                            ctx.getSystemExecutorService().execute(new Runnable() {
                                 @Override public void run() {
                                     if (!ctx.continuous().lockStopping())
                                         return;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobContextImpl.java
index 804d2284ee5f7..dbfa0b17c2ec8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobContextImpl.java
@@ -217,7 +217,7 @@ public void job(GridJobWorker job) {
 
                                 assert execSvc != null;
 
-                                execSvc.submit(new Runnable() {
+                                execSvc.execute(new Runnable() {
                                     @Override public void run() {
                                         callcc0();
                                     }
@@ -300,4 +300,4 @@ private IgniteLogger log() {
     @Override public String toString() {
         return S.toString(GridJobContextImpl.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index ae292232e0930..927944fd908c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.util.IgniteExceptionRegistry;
+import org.apache.ignite.internal.util.StripedExecutor;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.plugin.PluginNotFoundException;
 import org.apache.ignite.plugin.PluginProvider;
@@ -510,6 +511,14 @@ public interface GridKernalContext extends Iterable<GridComponent> {
      */
     public ExecutorService getSystemExecutorService();
 
+    /**
+     * Executor service that is in charge of processing internal system messages
+     * in stripes (dedicated threads).
+     *
+     * @return Thread pool implementation to be used in grid for internal system messages.
+     */
+    public StripedExecutor getStripedExecutorService();
+
     /**
      * Executor service that is in charge of processing internal and Visor
      * {@link org.apache.ignite.compute.ComputeJob GridJobs}.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 94c64481d4ce3..a2ad1b2043368 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -83,6 +83,7 @@
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.util.IgniteExceptionRegistry;
+import org.apache.ignite.internal.util.StripedExecutor;
 import org.apache.ignite.internal.util.spring.IgniteSpringHelper;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -298,6 +299,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     @GridToStringExclude
     protected ExecutorService sysExecSvc;
 
+    /** */
+    @GridToStringExclude
+    protected StripedExecutor stripedExecSvc;
+
     /** */
     @GridToStringExclude
     private ExecutorService p2pExecSvc;
@@ -381,6 +386,7 @@ public GridKernalContextImpl() {
      * @param marshCachePool Marshaller cache pool.
      * @param execSvc Public executor service.
      * @param sysExecSvc System executor service.
+     * @param stripedExecSvc Striped executor.
      * @param p2pExecSvc P2P executor service.
      * @param mgmtExecSvc Management executor service.
      * @param igfsExecSvc IGFS executor service.
@@ -400,6 +406,7 @@ protected GridKernalContextImpl(
         ExecutorService marshCachePool,
         ExecutorService execSvc,
         ExecutorService sysExecSvc,
+        StripedExecutor stripedExecSvc,
         ExecutorService p2pExecSvc,
         ExecutorService mgmtExecSvc,
         ExecutorService igfsExecSvc,
@@ -407,7 +414,8 @@ protected GridKernalContextImpl(
         ExecutorService affExecSvc,
         @Nullable ExecutorService idxExecSvc,
         IgniteStripedThreadPoolExecutor callbackExecSvc,
-        List<PluginProvider> plugins) throws IgniteCheckedException {
+        List<PluginProvider> plugins
+    ) throws IgniteCheckedException {
         assert grid != null;
         assert cfg != null;
         assert gw != null;
@@ -419,6 +427,7 @@ protected GridKernalContextImpl(
         this.marshCachePool = marshCachePool;
         this.execSvc = execSvc;
         this.sysExecSvc = sysExecSvc;
+        this.stripedExecSvc = stripedExecSvc;
         this.p2pExecSvc = p2pExecSvc;
         this.mgmtExecSvc = mgmtExecSvc;
         this.igfsExecSvc = igfsExecSvc;
@@ -947,6 +956,11 @@ protected Object readResolve() throws ObjectStreamException {
         return sysExecSvc;
     }
 
+    /** {@inheritDoc} */
+    @Override public StripedExecutor getStripedExecutorService() {
+        return stripedExecSvc;
+    }
+
     /** {@inheritDoc} */
     @Override public ExecutorService getManagementExecutorService() {
         return mgmtExecSvc;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridPerformanceSuggestions.java b/modules/core/src/main/java/org/apache/ignite/internal/GridPerformanceSuggestions.java
index b040a9732a7a0..5e8e520c277c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridPerformanceSuggestions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridPerformanceSuggestions.java
@@ -89,4 +89,4 @@ public synchronized void logSuggestions(IgniteLogger log, @Nullable String gridN
     @Override public String toString() {
         return S.toString(GridPerformanceSuggestions.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index b5608db32bc8a..24ddcd226ae7d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -97,7 +97,10 @@ public enum GridTopic {
     TOPIC_QUERY,
 
     /** */
-    TOPIC_TX;
+    TOPIC_TX,
+
+    /** */
+    TOPIC_IO_TEST;
 
     /** Enum values. */
     private static final GridTopic[] VALS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteInternalFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteInternalFuture.java
index b80a7554c61c5..789556d9d9625 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteInternalFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteInternalFuture.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal;
 
+import java.util.concurrent.Executor;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import org.apache.ignite.IgniteCheckedException;
@@ -132,6 +133,16 @@ public interface IgniteInternalFuture<R> {
      */
     public <T> IgniteInternalFuture<T> chain(IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb);
 
+    /**
+     * Make a chained future to convert result of this future (when complete) into a new format.
+     * It is guaranteed that done callback will be called only ONCE.
+     *
+     * @param doneCb Done callback that is applied to this future when it finishes to produce chained future result.
+     * @param exec Executor to run callback.
+     * @return Chained future that finishes after this future completes and done callback is called.
+     */
+    public <T> IgniteInternalFuture<T> chain(IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb, Executor exec);
+
     /**
      * @return Error value if future has already been completed with error.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 57aab0030a9a1..7935e069c5244 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -61,10 +61,10 @@
 import org.apache.ignite.IgniteEvents;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteFileSystem;
+import org.apache.ignite.IgniteLock;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.IgniteQueue;
-import org.apache.ignite.IgniteLock;
 import org.apache.ignite.IgniteScheduler;
 import org.apache.ignite.IgniteSemaphore;
 import org.apache.ignite.IgniteServices;
@@ -115,7 +115,6 @@
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
 import org.apache.ignite.internal.processors.hadoop.Hadoop;
-import org.apache.ignite.internal.processors.hadoop.HadoopClassLoader;
 import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.job.GridJobProcessor;
 import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor;
@@ -139,6 +138,7 @@
 import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
+import org.apache.ignite.internal.util.StripedExecutor;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -175,6 +175,7 @@
 import org.apache.ignite.thread.IgniteStripedThreadPoolExecutor;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONFIG_URL;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DAEMON;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_NO_ASCII;
@@ -182,7 +183,6 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_STARVATION_CHECK_INTERVAL;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SUCCESS_FILE;
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.IgniteSystemProperties.getBoolean;
 import static org.apache.ignite.IgniteSystemProperties.snapshot;
 import static org.apache.ignite.internal.GridKernalState.DISCONNECTED;
@@ -199,7 +199,6 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CLIENT_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CONSISTENCY_CHECK_SKIPPED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DAEMON;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_GRID_NAME;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IPS;
@@ -208,11 +207,12 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JVM_ARGS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JVM_PID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LANG_RUNTIME;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM;
@@ -663,6 +663,7 @@ private void notifyLifecycleBeansEx(LifecycleEventType evt) {
      * @param utilityCachePool Utility cache pool.
      * @param execSvc Executor service.
      * @param sysExecSvc System executor service.
+     * @param stripedExecSvc Striped executor.
      * @param p2pExecSvc P2P executor service.
      * @param mgmtExecSvc Management executor service.
      * @param igfsExecSvc IGFS executor service.
@@ -673,11 +674,13 @@ private void notifyLifecycleBeansEx(LifecycleEventType evt) {
      * @throws IgniteCheckedException Thrown in case of any errors.
      */
     @SuppressWarnings({"CatchGenericClass", "unchecked"})
-    public void start(final IgniteConfiguration cfg,
+    public void start(
+        final IgniteConfiguration cfg,
         ExecutorService utilityCachePool,
         ExecutorService marshCachePool,
         final ExecutorService execSvc,
         final ExecutorService sysExecSvc,
+        final StripedExecutor stripedExecSvc,
         ExecutorService p2pExecSvc,
         ExecutorService mgmtExecSvc,
         ExecutorService igfsExecSvc,
@@ -685,7 +688,8 @@ public void start(final IgniteConfiguration cfg,
         ExecutorService affExecSvc,
         @Nullable ExecutorService idxExecSvc,
         IgniteStripedThreadPoolExecutor callbackExecSvc,
-        GridAbsClosure errHnd)
+        GridAbsClosure errHnd
+    )
         throws IgniteCheckedException
     {
         gw.compareAndSet(null, new GridKernalGatewayImpl(cfg.getGridName()));
@@ -785,6 +789,7 @@ public void start(final IgniteConfiguration cfg,
                 marshCachePool,
                 execSvc,
                 sysExecSvc,
+                stripedExecSvc,
                 p2pExecSvc,
                 mgmtExecSvc,
                 igfsExecSvc,
@@ -792,7 +797,8 @@ public void start(final IgniteConfiguration cfg,
                 affExecSvc,
                 idxExecSvc,
                 callbackExecSvc,
-                plugins);
+                plugins
+            );
 
             cfg.getMarshaller().setContext(ctx.marshallerContext());
 
@@ -986,24 +992,51 @@ else if (e instanceof IgniteCheckedException)
 
             starveTask = ctx.timeout().schedule(new Runnable() {
                 /** Last completed task count. */
-                private long lastCompletedCnt;
+                private long lastCompletedCntPub;
+
+                /** Last completed task count. */
+                private long lastCompletedCntSys;
 
                 @Override public void run() {
-                    if (!(execSvc instanceof ThreadPoolExecutor))
-                        return;
+                    if (execSvc instanceof ThreadPoolExecutor) {
+                        ThreadPoolExecutor exec = (ThreadPoolExecutor)execSvc;
+
+                        lastCompletedCntPub = checkPoolStarvation(exec, lastCompletedCntPub, "public");
+                    }
 
-                    ThreadPoolExecutor exec = (ThreadPoolExecutor)execSvc;
+                    if (sysExecSvc instanceof ThreadPoolExecutor) {
+                        ThreadPoolExecutor exec = (ThreadPoolExecutor)sysExecSvc;
 
+                        lastCompletedCntSys = checkPoolStarvation(exec, lastCompletedCntSys, "system");
+                    }
+
+                    if (stripedExecSvc != null)
+                        stripedExecSvc.checkStarvation();
+                }
+
+                /**
+                 * @param exec Thread pool executor to check.
+                 * @param lastCompletedCnt Last completed tasks count.
+                 * @param pool Pool name for message.
+                 * @return Current completed tasks count.
+                 */
+                private long checkPoolStarvation(
+                    ThreadPoolExecutor exec,
+                    long lastCompletedCnt,
+                    String pool
+                ) {
                     long completedCnt = exec.getCompletedTaskCount();
 
                     // If all threads are active and no task has completed since last time and there is
                     // at least one waiting request, then it is possible starvation.
                     if (exec.getPoolSize() == exec.getActiveCount() && completedCnt == lastCompletedCnt &&
                         !exec.getQueue().isEmpty())
-                        LT.warn(log, "Possible thread pool starvation detected (no task completed in last " +
-                            interval + "ms, is executorService pool size large enough?)");
+                        LT.warn(
+                            log,
+                            "Possible thread pool starvation detected (no task completed in last " +
+                                interval + "ms, is " + pool + " thread pool size large enough?)");
 
-                    lastCompletedCnt = completedCnt;
+                    return completedCnt;
                 }
             }, interval, interval);
         }
@@ -1128,6 +1161,8 @@ else if (e instanceof IgniteCheckedException)
             }, longOpDumpTimeout, longOpDumpTimeout);
         }
 
+        ctx.performance().add("Disable assertions (remove '-ea' from JVM options)", !U.assertionsEnabled());
+
         ctx.performance().logSuggestions(log, gridName);
 
         U.quietAndInfo(log, "To start Console Management & Monitoring run ignitevisorcmd.{sh|bat}");
@@ -3509,6 +3544,26 @@ public void dumpDebugInfo() {
         }
     }
 
+    /**
+     * @param node Node.
+     * @param payload Message payload.
+     * @param procFromNioThread If {@code true} message is processed from NIO thread.
+     * @return Response future.
+     */
+    public IgniteInternalFuture sendIoTest(ClusterNode node, byte[] payload, boolean procFromNioThread) {
+        return ctx.io().sendIoTest(node, payload, procFromNioThread);
+    }
+
+    /**
+     * @param nodes Nodes.
+     * @param payload Message payload.
+     * @param procFromNioThread If {@code true} message is processed from NIO thread.
+     * @return Response future.
+     */
+    public IgniteInternalFuture sendIoTest(List<ClusterNode> nodes, byte[] payload, boolean procFromNioThread) {
+        return ctx.io().sendIoTest(nodes, payload, procFromNioThread);
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IgniteKernal.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index b3a9eec840a67..f32a75309166c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.StripedExecutor;
 import org.apache.ignite.internal.util.spring.IgniteSpringHelper;
 import org.apache.ignite.internal.util.typedef.CA;
 import org.apache.ignite.internal.util.typedef.F;
@@ -1459,6 +1460,9 @@ private static final class IgniteNamedInstance {
         /** System executor service. */
         private ThreadPoolExecutor sysExecSvc;
 
+        /** */
+        private StripedExecutor stripedExecSvc;
+
         /** Management executor service. */
         private ThreadPoolExecutor mgmtExecSvc;
 
@@ -1652,8 +1656,6 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
 
             execSvc.allowCoreThreadTimeOut(true);
 
-            // Note that since we use 'LinkedBlockingQueue', number of
-            // maximum threads has no effect.
             validateThreadPoolSize(cfg.getSystemThreadPoolSize(), "system");
 
             sysExecSvc = new IgniteThreadPoolExecutor(
@@ -1666,6 +1668,9 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
 
             sysExecSvc.allowCoreThreadTimeOut(true);
 
+            if (cfg.getStripedPoolSize() > 0)
+                stripedExecSvc = new StripedExecutor(cfg.getStripedPoolSize(), cfg.getGridName(), "sys", log);
+
             // Note that since we use 'LinkedBlockingQueue', number of
             // maximum threads has no effect.
             // Note, that we do not pre-start threads here as management pool may
@@ -1791,13 +1796,26 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 // Init here to make grid available to lifecycle listeners.
                 grid = grid0;
 
-                grid0.start(myCfg, utilityCacheExecSvc, marshCacheExecSvc, execSvc, sysExecSvc, p2pExecSvc, mgmtExecSvc,
-                    igfsExecSvc, restExecSvc, affExecSvc, idxExecSvc, callbackExecSvc,
+                grid0.start(
+                    myCfg,
+                    utilityCacheExecSvc,
+                    marshCacheExecSvc,
+                    execSvc,
+                    sysExecSvc,
+                    stripedExecSvc,
+                    p2pExecSvc,
+                    mgmtExecSvc,
+                    igfsExecSvc,
+                    restExecSvc,
+                    affExecSvc,
+                    idxExecSvc,
+                    callbackExecSvc,
                     new CA() {
                         @Override public void apply() {
                             startLatch.countDown();
                         }
-                    });
+                    }
+                );
 
                 state = STARTED;
 
@@ -2415,6 +2433,10 @@ private void stopExecutors0(IgniteLogger log) {
 
             sysExecSvc = null;
 
+            U.shutdownNow(getClass(), stripedExecSvc, log);
+
+            stripedExecSvc = null;
+
             U.shutdownNow(getClass(), mgmtExecSvc, log);
 
             mgmtExecSvc = null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
index 6ea7c227bddde..12baee0592812 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
@@ -200,6 +200,7 @@ protected GridClientConnectionManagerAdapter(UUID clientId,
                     .socketSendBufferSize(0)
                     .idleTimeout(Long.MAX_VALUE)
                     .gridName(routerClient ? "routerClient" : "gridClient")
+                    .serverName("tcp-client")
                     .daemon(cfg.isDaemon())
                     .build();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
index 06a492924f408..3566830a640a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
@@ -258,6 +258,7 @@ private boolean startTcpServer(InetAddress hostAddr, int port, GridNioServerList
                 .logger(log)
                 .selectorCount(Runtime.getRuntime().availableProcessors())
                 .gridName(gridName)
+                .serverName("router")
                 .tcpNoDelay(tcpNoDelay)
                 .directBuffer(false)
                 .byteOrder(ByteOrder.nativeOrder())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 3df29cf8fb4e5..7ef7bc080fe98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -26,15 +26,17 @@
 import java.util.Map.Entry;
 import java.util.Queue;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
@@ -44,6 +46,7 @@
 import org.apache.ignite.internal.GridTopic;
 import org.apache.ignite.internal.IgniteComponentType;
 import org.apache.ignite.internal.IgniteDeploymentCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.direct.DirectMessageReader;
 import org.apache.ignite.internal.direct.DirectMessageWriter;
 import org.apache.ignite.internal.managers.GridManagerAdapter;
@@ -55,6 +58,8 @@
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet;
 import org.apache.ignite.internal.util.GridSpinReadWriteLock;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridTuple3;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
@@ -83,6 +88,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.GridTopic.TOPIC_COMM_USER;
+import static org.apache.ignite.internal.GridTopic.TOPIC_IO_TEST;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.AFFINITY_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.IDX_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.IGFS_POOL;
@@ -176,6 +182,12 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
     /** Stopping flag. */
     private boolean stopping;
 
+    /** */
+    private final AtomicReference<ConcurrentHashMap<Long, IoTestFuture>> ioTestMap = new AtomicReference<>();
+
+    /** */
+    private final AtomicLong ioTestId = new AtomicLong();
+
     /**
      * @param ctx Grid kernal context.
      */
@@ -297,6 +309,114 @@ public void resetMetrics() {
 
         if (log.isDebugEnabled())
             log.debug(startInfo());
+
+        addMessageListener(GridTopic.TOPIC_IO_TEST, new GridMessageListener() {
+            @Override public void onMessage(UUID nodeId, Object msg) {
+                ClusterNode node = ctx.discovery().node(nodeId);
+
+                if (node == null)
+                    return;
+
+                IgniteIoTestMessage msg0 = (IgniteIoTestMessage)msg;
+
+                if (msg0.request()) {
+                    IgniteIoTestMessage res = new IgniteIoTestMessage(msg0.id(), false, null);
+
+                    res.flags(msg0.flags());
+
+                    try {
+                        send(node, GridTopic.TOPIC_IO_TEST, res, GridIoPolicy.SYSTEM_POOL);
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to send IO test response [msg=" + msg0 + "]", e);
+                    }
+                }
+                else {
+                    IoTestFuture fut = ioTestMap().get(msg0.id());
+
+                    if (fut == null)
+                        U.warn(log, "Failed to find IO test future [msg=" + msg0 + ']');
+                    else
+                        fut.onResponse();
+                }
+            }
+        });
+    }
+
+    /**
+     * @param nodes Nodes.
+     * @param payload Payload.
+     * @param procFromNioThread If {@code true} message is processed from NIO thread.
+     * @return Response future.
+     */
+    public IgniteInternalFuture sendIoTest(List<ClusterNode> nodes, byte[] payload, boolean procFromNioThread) {
+        long id = ioTestId.getAndIncrement();
+
+        IoTestFuture fut = new IoTestFuture(id, nodes.size());
+
+        IgniteIoTestMessage msg = new IgniteIoTestMessage(id, true, payload);
+
+        msg.processFromNioThread(procFromNioThread);
+
+        ioTestMap().put(id, fut);
+
+        for (int i = 0; i < nodes.size(); i++) {
+            ClusterNode node = nodes.get(i);
+
+            try {
+                send(node, GridTopic.TOPIC_IO_TEST, msg, GridIoPolicy.SYSTEM_POOL);
+            }
+            catch (IgniteCheckedException e) {
+                ioTestMap().remove(msg.id());
+
+                return new GridFinishedFuture(e);
+            }
+        }
+
+        return fut;
+    }
+
+    /**
+     * @param node Node.
+     * @param payload Payload.
+     * @param procFromNioThread If {@code true} message is processed from NIO thread.
+     * @return Response future.
+     */
+    public IgniteInternalFuture sendIoTest(ClusterNode node, byte[] payload, boolean procFromNioThread) {
+        long id = ioTestId.getAndIncrement();
+
+        IoTestFuture fut = new IoTestFuture(id, 1);
+
+        IgniteIoTestMessage msg = new IgniteIoTestMessage(id, true, payload);
+
+        msg.processFromNioThread(procFromNioThread);
+
+        ioTestMap().put(id, fut);
+
+        try {
+            send(node, GridTopic.TOPIC_IO_TEST, msg, GridIoPolicy.SYSTEM_POOL);
+        }
+        catch (IgniteCheckedException e) {
+            ioTestMap().remove(msg.id());
+
+            return new GridFinishedFuture(e);
+        }
+
+        return fut;
+    }
+
+    /**
+     * @return IO test futures map.
+     */
+    private ConcurrentHashMap<Long, IoTestFuture> ioTestMap() {
+        ConcurrentHashMap<Long, IoTestFuture> map = ioTestMap.get();
+
+        if (map == null) {
+            if (!ioTestMap.compareAndSet(null, map = new ConcurrentHashMap<>()))
+                map = ioTestMap.get();
+        }
+
+        return map;
     }
 
     /** {@inheritDoc} */
@@ -514,16 +634,6 @@ private void onMessage0(UUID nodeId, GridIoMessage msg, IgniteRunnable msgC) {
                 return;
             }
 
-            // Check discovery.
-            ClusterNode node = ctx.discovery().node(nodeId);
-
-            if (node == null) {
-                if (log.isDebugEnabled())
-                    log.debug("Ignoring message from dead node [senderId=" + nodeId + ", msg=" + msg + ']');
-
-                return; // We can't receive messages from non-discovered ones.
-            }
-
             if (msg.topic() == null) {
                 int topicOrd = msg.topicOrdinal();
 
@@ -678,8 +788,31 @@ private void processRegularMessage(
                     msgC.run();
                 }
             }
+
+            @Override public String toString() {
+                return "Message closure [msg=" + msg + ']';
+            }
         };
 
+        if (msg.topicOrdinal() == TOPIC_IO_TEST.ordinal()) {
+            IgniteIoTestMessage msg0 = (IgniteIoTestMessage)msg.message();
+
+            if (msg0.processFromNioThread()) {
+                c.run();
+
+                return;
+            }
+        }
+
+        if (ctx.config().getStripedPoolSize() > 0 &&
+            plc == GridIoPolicy.SYSTEM_POOL &&
+            msg.partition() != Integer.MIN_VALUE
+            ) {
+            ctx.getStripedExecutorService().execute(msg.partition(), c);
+
+            return;
+        }
+
         try {
             pools.poolForPolicy(plc).execute(c);
         }
@@ -2460,4 +2593,56 @@ public UUID nodeId() {
             return S.toString(DelayedMessage.class, this, super.toString());
         }
     }
+
+    /**
+     *
+     */
+    private class IoTestFuture extends GridFutureAdapter<Object> {
+        /** */
+        private final long id;
+
+        /** */
+        private int cntr;
+
+        /**
+         * @param id ID.
+         * @param cntr Counter.
+         */
+        IoTestFuture(long id, int cntr) {
+            assert cntr > 0 : cntr;
+
+            this.id = id;
+            this.cntr = cntr;
+        }
+
+        /**
+         *
+         */
+        void onResponse() {
+            boolean complete;
+
+            synchronized (this) {
+                complete = --cntr == 0;
+            }
+
+            if (complete)
+                onDone();
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean onDone(@Nullable Object res, @Nullable Throwable err) {
+            if (super.onDone(res, err)) {
+                ioTestMap().remove(id);
+
+                return true;
+            }
+
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(IoTestFuture.class, this);
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessage.java
index b28ced2f35e6f..b1a26e9b434c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessage.java
@@ -20,6 +20,7 @@
 import java.io.Externalizable;
 import java.nio.ByteBuffer;
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.cache.GridCacheMessage;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.plugin.extensions.communication.Message;
@@ -321,6 +322,18 @@ boolean isOrdered() {
         return 7;
     }
 
+    /**
+     * Get single partition for this message (if applicable).
+     *
+     * @return Partition ID.
+     */
+    public int partition() {
+        if (msg instanceof GridCacheMessage)
+            return ((GridCacheMessage)msg).partition();
+        else
+            return Integer.MIN_VALUE;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridIoMessage.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 504e6830eac39..b1fe910f1c572 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -171,6 +171,16 @@ public GridIoMessageFactory(MessageFactory[] ext) {
         Message msg = null;
 
         switch (type) {
+            case -44:
+                msg = new TcpCommunicationSpi.HandshakeMessage2();
+
+                break;
+
+            case -43:
+                msg = new IgniteIoTestMessage();
+
+                break;
+
             case -42:
                 msg = new HadoopDirectShuffleMessage();
 
@@ -816,7 +826,7 @@ public GridIoMessageFactory(MessageFactory[] ext) {
 
                 break;
 
-            // [-3..119] [124..127] [-36]- this
+            // [-3..119] [124..127] [-36..-44]- this
             // [120..123] - DR
             // [-4..-22, -30..-35] - SQL
             default:
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
new file mode 100644
index 0000000000000..77aaa091c4bcb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+import java.nio.ByteBuffer;
+
+/**
+ *
+ */
+public class IgniteIoTestMessage implements Message {
+    /** */
+    private static byte FLAG_PROC_FROM_NIO = 1;
+
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private long id;
+
+    /** */
+    private byte flags;
+
+    /** */
+    private boolean req;
+
+    /** */
+    private byte payload[];
+
+    /**
+     *
+     */
+    IgniteIoTestMessage() {
+        // No-op.
+    }
+
+    /**
+     * @param id Message ID.
+     * @param req Request flag.
+     * @param payload Payload.
+     */
+    IgniteIoTestMessage(long id, boolean req, byte[] payload) {
+        this.id = id;
+        this.req = req;
+        this.payload = payload;
+    }
+
+    /**
+     * @return {@code True} if message should be processed from NIO thread
+     * (otherwise message is submitted to system pool).
+     */
+    boolean processFromNioThread() {
+        return isFlag(FLAG_PROC_FROM_NIO);
+    }
+
+    /**
+     * @param procFromNioThread {@code True} if message should be processed from NIO thread.
+     */
+    void processFromNioThread(boolean procFromNioThread) {
+        setFlag(procFromNioThread, FLAG_PROC_FROM_NIO);
+    }
+
+    /**
+     * @param flags Flags.
+     */
+    public void flags(byte flags) {
+        this.flags = flags;
+    }
+
+    /**
+     * @return Flags.
+     */
+    public byte flags() {
+        return flags;
+    }
+
+    /**
+     * Sets flag mask.
+     *
+     * @param flag Set or clear.
+     * @param mask Mask.
+     */
+    private void setFlag(boolean flag, int mask) {
+        flags = flag ? (byte)(flags | mask) : (byte)(flags & ~mask);
+    }
+
+    /**
+     * Reads flag mask.
+     *
+     * @param mask Mask to read.
+     * @return Flag value.
+     */
+    private boolean isFlag(int mask) {
+        return (flags & mask) != 0;
+    }
+
+    /**
+     * @return {@code true} if this is request.
+     */
+    public boolean request() {
+        return req;
+    }
+
+    /**
+     * @return ID.
+     */
+    public long id() {
+        return id;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeByte("flags", flags))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeLong("id", id))
+                    return false;
+
+                writer.incrementState();
+
+            case 2:
+                if (!writer.writeByteArray("payload", payload))
+                    return false;
+
+                writer.incrementState();
+
+            case 3:
+                if (!writer.writeBoolean("req", req))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                flags = reader.readByte("flags");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                id = reader.readLong("id");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 2:
+                payload = reader.readByteArray("payload");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 3:
+                req = reader.readBoolean("req");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        return reader.afterMessageRead(IgniteIoTestMessage.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte directType() {
+        return -43;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 4;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IgniteIoTestMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 2e24e67cae195..a8d9f1d8b6cbe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -288,6 +288,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
     /** Asynchronous operations limit semaphore. */
     private Semaphore asyncOpsSem;
 
+    /** */
+    protected volatile boolean asyncToggled;
+
     /** {@inheritDoc} */
     @Override public String name() {
         return cacheCfg.getName();
@@ -363,6 +366,18 @@ protected GridCacheAdapter(final GridCacheContext<K, V> ctx, @Nullable GridCache
         aff = new GridCacheAffinityImpl<>(ctx);
     }
 
+    /**
+     * Toggles async flag if someone calls {@code withAsync()}
+     * on proxy and since that we have to properly handle all cache
+     * operations (sync and async) to put them in proper sequence.
+     *
+     * TODO: https://issues.apache.org/jira/browse/IGNITE-4393
+     */
+    void toggleAsync() {
+        if (!asyncToggled)
+            asyncToggled = true;
+    }
+
     /**
      * Prints memory stats.
      */
@@ -1134,7 +1149,7 @@ public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, bool
                 execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
 
                 for (int i = 1; i < jobs.size(); i++)
-                    execSvc.submit(jobs.get(i));
+                    execSvc.execute(jobs.get(i));
             }
 
             try {
@@ -2534,6 +2549,8 @@ private <T> EntryProcessorResult<T> invoke0(
      * @return Put future.
      */
     public IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+        A.notNull(key, "key", val, "val");
+
         final boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
@@ -2554,8 +2571,6 @@ public IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntry
      */
     public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         @Nullable final CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
         if (keyCheck)
             validateCacheKey(key);
 
@@ -4592,6 +4607,9 @@ protected void saveFuture(final FutureHolder holder, IgniteInternalFuture<?> fut
      * @return Failed future if waiting was interrupted.
      */
     @Nullable protected <T> IgniteInternalFuture<T> asyncOpAcquire() {
+        if (!asyncToggled)
+            return null;
+
         try {
             if (asyncOpsSem != null)
                 asyncOpsSem.acquire();
@@ -4610,7 +4628,7 @@ protected void saveFuture(final FutureHolder holder, IgniteInternalFuture<?> fut
      * Releases asynchronous operations permit, if limited.
      */
     protected void asyncOpRelease() {
-        if (asyncOpsSem != null)
+        if (asyncOpsSem != null && asyncToggled)
             asyncOpsSem.release();
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
index 71f99d3c47da0..0646d5ae5e3d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
@@ -128,6 +128,13 @@ public int lookupIndex() {
         return -1;
     }
 
+    /**
+     * @return Partition ID this message is targeted to or {@code -1} if it cannot be determined.
+     */
+    public int partition() {
+        return -1;
+    }
+
     /**
      * If class loading error occurred during unmarshalling and {@link #ignoreClassErrors()} is
      * set to {@code true}, then the error will be passed into this method.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 90e428ca10b80..317820362ff7b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -110,6 +110,41 @@
  * Cache utility methods.
  */
 public class GridCacheUtils {
+    /** Cheat cache ID for debugging and benchmarking purposes. */
+    public static final int cheatCacheId;
+
+    /*
+     *
+     */
+    static {
+        String cheatCache = System.getProperty("CHEAT_CACHE");
+
+        if (cheatCache != null) {
+            cheatCacheId = cheatCache.hashCode();
+
+            if (cheatCacheId == 0)
+                throw new RuntimeException();
+
+            System.out.println(">>> Cheat cache ID [id=" + cheatCacheId + ", name=" + cheatCache + ']');
+        }
+        else
+            cheatCacheId = 0;
+    }
+
+    /**
+     * Quickly checks if passed in cache ID is a "cheat cache ID" set by -DCHEAT_CACHE=user_cache_name
+     * and resolved in static block above.
+     *
+     * FOR DEBUGGING AND TESTING PURPOSES!
+     *
+     * @param id Cache ID to check.
+     * @return {@code True} if this is cheat cache ID.
+     */
+    @Deprecated
+    public static boolean cheatCache(int id) {
+        return cheatCacheId != 0 && id == cheatCacheId;
+    }
+
     /**  Hadoop syste cache name. */
     public static final String SYS_CACHE_HADOOP_MR = "ignite-hadoop-mr-sys-cache";
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
index f87fa1d5d815f..b9e6e828a94fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
@@ -333,6 +333,14 @@ public GridCacheGateway<K, V> gate() {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public IgniteCache<K, V> withAsync() {
+        if (delegate instanceof GridCacheAdapter)
+            ((GridCacheAdapter)delegate).toggleAsync();
+
+        return super.withAsync();
+    }
+
     /** {@inheritDoc} */
     @Override public IgniteCache<K, V> withSkipStore() {
         return skipStore();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
index 9639a9a7288ee..a671296b61f51 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
@@ -310,6 +310,11 @@ public List<KeyCacheObject> keys() {
         return keys;
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return partIds != null && !partIds.isEmpty() ? partIds.get(0) : -1;
+    }
+
     /**
      * @return Max lock wait time.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
index 109d665704dda..c5cf33242b9b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
@@ -85,6 +85,12 @@ public IgniteUuid futureId() {
         return ctx.txFinishMessageLogger();
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        // TODO https://issues.apache.org/jira/browse/IGNITE-4371
+        return Integer.MIN_VALUE;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
         writer.setBuffer(buf);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java
index df6acdd1fbc89..5d70ec1fff110 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java
@@ -89,6 +89,11 @@ public void addKey(KeyCacheObject key, GridCacheContext ctx) throws IgniteChecke
         partIds.add(key.partition());
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return partIds != null && !partIds.isEmpty() ? partIds.get(0) : -1;
+    }
+
     /** {@inheritDoc}
      * @param ctx*/
     @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 35e62670ea2a9..519d0fc640032 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -218,7 +218,8 @@ protected GridDhtCacheAdapter(GridCacheContext<K, V> ctx, GridCacheConcurrentMap
     @Override public void onKernalStart() throws IgniteCheckedException {
         super.onKernalStart();
 
-        preldr.onKernalStart();
+        if (preldr != null)
+            preldr.onKernalStart();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java
index 1e92b54fc59f1..63e33091afcf0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java
@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.GridDirectCollection;
@@ -31,7 +32,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockResponse;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteUuid;
@@ -57,7 +57,7 @@ public class GridDhtLockResponse extends GridDistributedLockResponse {
     /** Invalid partitions. */
     @GridToStringInclude
     @GridDirectCollection(int.class)
-    private Collection<Integer> invalidParts = new GridLeanSet<>();
+    private Collection<Integer> invalidParts;
 
     /** Preload entries. */
     @GridDirectCollection(GridCacheEntryInfo.class)
@@ -127,6 +127,9 @@ public IgniteUuid miniId() {
      * @param part Invalid partition.
      */
     public void addInvalidPartition(int part) {
+        if (invalidParts == null)
+            invalidParts = new HashSet<>();
+
         invalidParts.add(part);
     }
 
@@ -134,7 +137,7 @@ public void addInvalidPartition(int part) {
      * @return Invalid partitions.
      */
     public Collection<Integer> invalidPartitions() {
-        return invalidParts;
+        return invalidParts == null ? Collections.<Integer>emptySet() : invalidParts;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 940c74ed863af..0e60ff49ba79d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -613,8 +613,6 @@ public void near(GridNearAtomicCache<K, V> near) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
         return updateAsync0(
             key,
             val,
@@ -814,6 +812,9 @@ private boolean writeThrough() {
      */
     @SuppressWarnings("unchecked")
     protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>> op) {
+        if (!asyncToggled)
+            return op.apply();
+
         IgniteInternalFuture<T> fail = asyncOpAcquire();
 
         if (fail != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
index a03d948017f91..0af7cf583fdeb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
@@ -234,6 +234,11 @@ public GridDhtAtomicSingleUpdateRequest() {
         return near() ? null : key;
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return partId;
+    }
+
     /** {@inheritDoc} */
     @Override public int partitionId(int idx) {
         assert idx == 0 : idx;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
index f2fbb0e5a9864..1854e52aa79ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
@@ -486,6 +486,11 @@ else if (conflictVers != null)
         return CU.TTL_NOT_CHANGED;
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return partIds != null && !partIds.isEmpty() ? partIds.get(0) : -1;
+    }
+
     /** {@inheritDoc} */
     @Override public long conflictExpireTime(int idx) {
         if (conflictExpireTimes != null) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
index 1b116884c5bc8..87d9225dfe846 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
@@ -610,6 +610,11 @@ else if (conflictVers != null)
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return partIds != null && !partIds.isEmpty() ? partIds.get(0) : -1;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
         writer.setBuffer(buf);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java
index 1c1adddb2c713..c3e9fbe1906f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java
@@ -125,6 +125,11 @@ public GridNearAtomicSingleUpdateRequest() {
         );
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return partId;
+    }
+
     /**
      * @param key Key to add.
      * @param val Optional update value.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
index fa7f367cc4460..4272a4d2c055d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
@@ -244,6 +244,11 @@ public long accessTtl() {
         return accessTtl;
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return partIds != null && !partIds.isEmpty() ? partIds.get(0) : -1;
+    }
+
     /**
      * @param ctx Cache context.
      * @throws IgniteCheckedException If failed.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index a419887698326..bc16ff4fd0b8a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -1585,6 +1585,9 @@ private static CachePartialUpdateCheckedException partialUpdateException() {
      */
     @SuppressWarnings("unchecked")
     protected IgniteInternalFuture asyncOp(final Callable<?> op) {
+        if (!asyncToggled)
+            return ctx.closures().callLocalSafe(op);
+
         IgniteInternalFuture fail = asyncOpAcquire();
 
         if (fail != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
index d34047e4c430a..eb5e214b89474 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
@@ -272,7 +272,7 @@ protected void removeQueryFuture(long reqId) {
                 false,
                 null,
                 req.keyValueFilter(),
-                req.partition(),
+                req.partition() == -1 ? null : req.partition(),
                 req.className(),
                 req.clause(),
                 req.includeMetaData(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
index 60c466226fcf7..9f965d8c0bd04 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
@@ -121,7 +121,7 @@ public class GridCacheQueryRequest extends GridCacheMessage implements GridCache
     private int taskHash;
 
     /** Partition. */
-    private int part;
+    private int part = -1;
 
     /** */
     private AffinityTopologyVersion topVer;
@@ -478,8 +478,8 @@ public int taskHash() {
     /**
      * @return partition.
      */
-    @Nullable public Integer partition() {
-        return part == -1 ? null : part;
+    public int partition() {
+        return part;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 6d21dcfb716fd..393fb1ac4b958 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -391,7 +391,7 @@ protected boolean commitAfterLock() {
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<Void> loadMissing(
         final GridCacheContext cacheCtx,
-        AffinityTopologyVersion topVer,
+        final AffinityTopologyVersion topVer,
         final boolean readThrough,
         boolean async,
         final Collection<KeyCacheObject> keys,
@@ -472,7 +472,7 @@ protected boolean commitAfterLock() {
                             CacheObject cacheVal = cacheCtx.toCacheObject(val);
 
                             while (true) {
-                                GridCacheEntryEx entry = cacheCtx.cache().entryEx(key);
+                                GridCacheEntryEx entry = cacheCtx.cache().entryEx(key, topVer);
 
                                 try {
                                     GridCacheVersion setVer = entry.versionedValue(cacheVal, ver, null);
@@ -1507,7 +1507,7 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
 
                         assert txEntry != null || readCommitted() || skipVals;
 
-                        GridCacheEntryEx e = txEntry == null ? entryEx(cacheCtx, txKey) : txEntry.cached();
+                        GridCacheEntryEx e = txEntry == null ? entryEx(cacheCtx, txKey, topVer) : txEntry.cached();
 
                         if (readCommitted() || skipVals) {
                             cacheCtx.evicts().touch(e, topologyVersion());
@@ -1658,7 +1658,7 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                                             IgniteTxLocalAdapter.this,
                                             /*swap*/cacheCtx.isSwapOrOffheapEnabled(),
                                             /*unmarshal*/true,
-                                            /**update-metrics*/true,
+                                            /*update-metrics*/true,
                                             /*event*/!skipVals,
                                             CU.subjectId(IgniteTxLocalAdapter.this, cctx),
                                             transformClo,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index 32fda87375cb4..fee4dd6616b28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -328,6 +328,8 @@ private void localUpdate(final UUID nodeId,
             if (!allowOverwrite)
                 cctx.topology().readLock();
 
+            GridDhtTopologyFuture topWaitFut = null;
+
             try {
                 GridDhtTopologyFuture fut = cctx.topologyVersionFuture();
 
@@ -352,19 +354,25 @@ else if (allowOverwrite || fut.isDone()) {
 
                     waitFut = allowOverwrite ? null : cctx.mvcc().addDataStreamerFuture(topVer);
                 }
-                else {
-                    fut.listen(new IgniteInClosure<IgniteInternalFuture<AffinityTopologyVersion>>() {
-                        @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> e) {
-                            localUpdate(nodeId, req, updater, topic);
-                        }
-                    });
-                }
+                else
+                    topWaitFut = fut;
             }
             finally {
                 if (!allowOverwrite)
                     cctx.topology().readUnlock();
             }
 
+            if (topWaitFut != null) {
+                // Need call 'listen' after topology read lock is released.
+                topWaitFut.listen(new IgniteInClosure<IgniteInternalFuture<AffinityTopologyVersion>>() {
+                    @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> e) {
+                        localUpdate(nodeId, req, updater, topic);
+                    }
+                });
+
+                return;
+            }
+
             if (job != null) {
                 try {
                     job.call();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
index 3405b5332eed2..4c037b74c5336 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
@@ -226,7 +226,7 @@ public ClusterNode localNode() {
      */
     public void runInIgfsThreadPool(Runnable r) {
         try {
-            igfsSvc.submit(r);
+            igfsSvc.execute(r);
         }
         catch (RejectedExecutionException ignored) {
             // This exception will happen if network speed is too low and data comes faster
@@ -252,4 +252,4 @@ private <T extends IgfsManager> T add(@Nullable T mgr) {
 
         return mgr;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index e534800efd7a6..4490a6854d79b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
+import java.util.concurrent.Executor;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.IgniteException;
@@ -36,6 +37,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
@@ -325,6 +327,8 @@ private IgniteDataStreamer<IgfsBlockKey, byte[]> dataStreamer() {
         IgniteInternalFuture<byte[]> fut = dataCachePrj.getAsync(key);
 
         if (secReader != null) {
+            Executor exec = igfsCtx.kernalContext().pools().poolForPolicy(GridIoPolicy.IGFS_POOL);
+
             fut = fut.chain(new CX1<IgniteInternalFuture<byte[]>, byte[]>() {
                 @Override public byte[] applyx(IgniteInternalFuture<byte[]> fut) throws IgniteCheckedException {
                     byte[] res = fut.get();
@@ -365,7 +369,7 @@ private IgniteDataStreamer<IgfsBlockKey, byte[]> dataStreamer() {
 
                     return res;
                 }
-            });
+            }, exec);
         }
         else
             igfsCtx.metrics().addReadBlocks(1, 0);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index ab4ee8533a051..6b23e8084a46b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -326,7 +326,7 @@ private IgfsFileWorkerBatch newBatch(final IgfsPath path, OutputStream out) thro
                 // Submit it to the thread pool immediately.
                 assert dualPool != null;
 
-                dualPool.submit(batch);
+                dualPool.execute(batch);
 
                 // Spin in case another batch is currently running.
                 while (true) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcProcessor.java
index 9388a8eca682a..7cba9bbf2381f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcProcessor.java
@@ -139,6 +139,7 @@ public OdbcProcessor(GridKernalContext ctx) {
                             .logger(log)
                             .selectorCount(DFLT_SELECTOR_CNT)
                             .gridName(ctx.gridName())
+                            .serverName("odbc")
                             .tcpNoDelay(DFLT_TCP_NODELAY)
                             .directBuffer(DFLT_TCP_DIRECT_BUF)
                             .byteOrder(ByteOrder.nativeOrder())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java
index 8ff15d5513555..5383151b80554 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.platform.compute;
 
+import java.util.concurrent.Executor;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.binary.BinaryObject;
@@ -408,6 +409,11 @@ public ComputeConvertingFuture(ComputeTaskFuture fut) {
             throw new UnsupportedOperationException("Chain operation is not supported.");
         }
 
+        /** {@inheritDoc} */
+        @Override public IgniteInternalFuture chain(IgniteClosure doneCb, Executor exec) {
+            throw new UnsupportedOperationException("Chain operation is not supported.");
+        }
+
         /** {@inheritDoc} */
         @Override public Throwable error() {
             return fut.error();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpMemcachedNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpMemcachedNioListener.java
index b403654efb3fb..71eca652bb7cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpMemcachedNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpMemcachedNioListener.java
@@ -21,7 +21,6 @@
 import java.util.Map;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.rest.GridRestCommand;
 import org.apache.ignite.internal.processors.rest.GridRestProtocolHandler;
@@ -38,8 +37,6 @@
 import org.apache.ignite.internal.util.typedef.C2;
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.marshaller.Marshaller;
-import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.internal.processors.rest.GridRestCommand.ATOMIC_DECREMENT;
@@ -72,24 +69,16 @@ public class GridTcpMemcachedNioListener extends GridNioServerListenerAdapter<Gr
     /** Handler. */
     private final GridRestProtocolHandler hnd;
 
-    /** JDK marshaller. */
-    private final Marshaller jdkMarshaller = new JdkMarshaller();
-
-    /** Context. */
-    private final GridKernalContext ctx;
-
     /**
      * Creates listener which will convert incoming tcp packets to rest requests and forward them to
      * a given rest handler.
      *
      * @param log Logger to use.
      * @param hnd Rest handler.
-     * @param ctx Context.
      */
-    public GridTcpMemcachedNioListener(IgniteLogger log, GridRestProtocolHandler hnd, GridKernalContext ctx) {
+    public GridTcpMemcachedNioListener(IgniteLogger log, GridRestProtocolHandler hnd) {
         this.log = log;
         this.hnd = hnd;
-        this.ctx = ctx;
     }
 
     /** {@inheritDoc} */
@@ -462,4 +451,4 @@ private GridRestRequest createRestRequest(GridMemcachedMessage req, GridRestComm
 
         return new GridTuple3<>(cmd, quiet, retKey);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
index 1c1c6dcd60da2..3ba6d8eae434e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
@@ -145,7 +145,7 @@ public class GridTcpRestNioListener extends GridNioServerListenerAdapter<GridCli
      */
     public GridTcpRestNioListener(IgniteLogger log, GridTcpRestProtocol proto, GridRestProtocolHandler hnd,
         GridKernalContext ctx) {
-        memcachedLsnr = new GridTcpMemcachedNioListener(log, hnd, ctx);
+        memcachedLsnr = new GridTcpMemcachedNioListener(log, hnd);
         redisLsnr = new GridRedisNioListener(log, hnd, ctx);
 
         this.log = log;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
index 6338fcccb58ca..2a002a32dbcb3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
@@ -257,6 +257,7 @@ private boolean startTcpServer(InetAddress hostAddr, int port, GridNioServerList
                 .logger(log)
                 .selectorCount(cfg.getSelectorCount())
                 .gridName(ctx.gridName())
+                .serverName("tcp-rest")
                 .tcpNoDelay(cfg.isNoDelay())
                 .directBuffer(cfg.isDirectBuffer())
                 .byteOrder(ByteOrder.nativeOrder())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6c263638de7c5..aeb3ef483b1ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1131,7 +1131,7 @@ else if (ctxs.size() < assignCnt) {
             // Start service in its own thread.
             final ExecutorService exe = svcCtx.executor();
 
-            exe.submit(new Runnable() {
+            exe.execute(new Runnable() {
                 @Override public void run() {
                     try {
                         svc.execute(svcCtx);
@@ -1394,7 +1394,7 @@ private class ServiceEntriesListener implements CacheEntryUpdatedListener<Object
                 return;
 
             try {
-                depExe.submit(new BusyRunnable() {
+                depExe.execute(new BusyRunnable() {
                     @Override public void run0() {
                         onSystemCacheUpdated(deps);
                     }
@@ -1587,7 +1587,7 @@ private class TopologyListener implements GridLocalEventListener {
                 else
                     topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
 
-                depExe.submit(new BusyRunnable() {
+                depExe.execute(new BusyRunnable() {
                     @Override public void run0() {
                         ClusterNode oldest = CU.oldestAliveCacheServerNode(cache.context().shared(), topVer);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index e1937bbe67810..3dfb3c609ef55 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -153,7 +153,6 @@
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryRawReader;
 import org.apache.ignite.binary.BinaryRawWriter;
 import org.apache.ignite.cluster.ClusterGroupEmptyException;
@@ -506,10 +505,27 @@ public abstract class IgniteUtils {
         }
     };
 
-    /**
-     * Initializes enterprise check.
+    /** */
+    private static final boolean assertionsEnabled;
+
+    /*
+     *
      */
     static {
+        boolean assertionsEnabled0 = true;
+
+        try {
+            assert false;
+
+            assertionsEnabled0 = false;
+        }
+        catch (AssertionError ignored) {
+            assertionsEnabled0 = true;
+        }
+        finally {
+            assertionsEnabled = assertionsEnabled0;
+        }
+
         String osName = System.getProperty("os.name");
 
         String osLow = osName.toLowerCase();
@@ -1283,6 +1299,27 @@ private static Set<Long> getDeadlockedThreadIds(ThreadMXBean mxBean) {
         return deadlockedThreadsIds;
     }
 
+    /**
+     * @param threadId Thread ID.
+     * @param sb Builder.
+     */
+    public static void printStackTrace(long threadId, GridStringBuilder sb) {
+        ThreadMXBean mxBean = ManagementFactory.getThreadMXBean();
+
+        ThreadInfo threadInfo = mxBean.getThreadInfo(threadId, Integer.MAX_VALUE);
+
+        printThreadInfo(threadInfo, sb, Collections.<Long>emptySet());
+    }
+
+    /**
+     * @return {@code true} if there is java level deadlock.
+     */
+    public static boolean deadlockPresent() {
+        ThreadMXBean mxBean = ManagementFactory.getThreadMXBean();
+
+        return !F.isEmpty(mxBean.findDeadlockedThreads());
+    }
+
     /**
      * Prints single thread info to a buffer.
      *
@@ -6140,6 +6177,13 @@ public static void unzip(File zipFile, File toDir, @Nullable IgniteLogger log) t
         }
     }
 
+    /**
+     * @return {@code True} if assertions enabled.
+     */
+    public static boolean assertionsEnabled() {
+        return assertionsEnabled;
+    }
+
     /**
      * Gets OS JDK string.
      *
@@ -8336,6 +8380,18 @@ public static int safeAbs(int i) {
         return i < 0 ? 0 : i;
     }
 
+    /**
+     * Gets absolute value for long. If argument is {@link Long#MIN_VALUE}, then {@code 0} is returned.
+     *
+     * @param i Argument.
+     * @return Absolute value.
+     */
+    public static long safeAbs(long i) {
+        i = Math.abs(i);
+
+        return i < 0 ? 0 : i;
+    }
+
     /**
      * Gets wrapper class for a primitive type.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
new file mode 100644
index 0000000000000..e9ec74b181578
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
@@ -0,0 +1,667 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.LockSupport;
+import org.apache.ignite.IgniteInterruptedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.thread.IgniteThread;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Striped executor.
+ */
+public class StripedExecutor implements ExecutorService {
+    /** Stripes. */
+    private final Stripe[] stripes;
+
+    /** For starvation checks. */
+    private final long[] completedCntrs;
+
+    /** */
+    private final IgniteLogger log;
+
+    /**
+     * Constructor.
+     *
+     * @param cnt Count.
+     */
+    public StripedExecutor(int cnt, String gridName, String poolName, final IgniteLogger log) {
+        A.ensure(cnt > 0, "cnt > 0");
+
+        boolean success = false;
+
+        stripes = new Stripe[cnt];
+
+        completedCntrs = new long[cnt];
+
+        Arrays.fill(completedCntrs, -1);
+
+        this.log = log;
+
+        try {
+            for (int i = 0; i < cnt; i++) {
+                stripes[i] = new StripeConcurrentQueue(
+                    gridName,
+                    poolName,
+                    i,
+                    log);
+
+                stripes[i].start();
+            }
+
+            success = true;
+        }
+        catch (Error | RuntimeException e) {
+            U.error(log, "Failed to initialize striped pool.", e);
+
+            throw e;
+        }
+        finally {
+            if (!success) {
+                for (Stripe stripe : stripes) {
+                    if (stripe != null)
+                        stripe.signalStop();
+                }
+
+                for (Stripe stripe : stripes) {
+                    if (stripe != null)
+                        stripe.awaitStop();
+                }
+            }
+        }
+    }
+
+    /**
+     * Checks starvation in striped pool. Maybe too verbose
+     * but this is needed to faster debug possible issues.
+     */
+    public void checkStarvation() {
+        for (int i = 0; i < stripes.length; i++) {
+            Stripe stripe = stripes[i];
+
+            long completedCnt = stripe.completedCnt;
+
+            boolean active = stripe.active;
+
+            if (completedCntrs[i] != -1 &&
+                completedCntrs[i] == completedCnt &&
+                active) {
+                boolean deadlockPresent = U.deadlockPresent();
+
+                GridStringBuilder sb = new GridStringBuilder();
+
+                sb.a(">>> Possible starvation in striped pool: ")
+                    .a(stripe.thread.getName()).a(U.nl())
+                    .a(stripe.queueToString()).a(U.nl())
+                    .a("deadlock: ").a(deadlockPresent).a(U.nl())
+                    .a("completed: ").a(completedCnt).a(U.nl());
+
+                U.printStackTrace(
+                    stripe.thread.getId(),
+                    sb);
+
+                String msg = sb.toString();
+
+                U.warn(log, msg);
+            }
+
+            if (active || completedCnt > 0)
+                completedCntrs[i] = completedCnt;
+        }
+    }
+
+    /**
+     * @return Stripes count.
+     */
+    public int stripes() {
+        return stripes.length;
+    }
+
+    /**
+     * Execute command.
+     *
+     * @param idx Index.
+     * @param cmd Command.
+     */
+    public void execute(int idx, Runnable cmd) {
+        if (idx == -1)
+            execute(cmd);
+        else {
+            assert idx >= 0 : idx;
+
+            stripes[idx % stripes.length].execute(cmd);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void shutdown() {
+        signalStop();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(@NotNull Runnable cmd) {
+        stripes[ThreadLocalRandom.current().nextInt(stripes.length)].execute(cmd);
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * @return Empty list (always).
+     */
+    @NotNull @Override public List<Runnable> shutdownNow() {
+        signalStop();
+
+        return Collections.emptyList();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean awaitTermination(
+        long timeout,
+        @NotNull TimeUnit unit
+    ) throws InterruptedException {
+        awaitStop();
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isShutdown() {
+        for (Stripe stripe : stripes) {
+            if (stripe != null && stripe.stopping)
+                return true;
+        }
+
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isTerminated() {
+        for (Stripe stripe : stripes) {
+            if (stripe.thread.getState() != Thread.State.TERMINATED)
+                return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Stops executor.
+     */
+    public void stop() {
+        signalStop();
+
+        awaitStop();
+    }
+
+    /**
+     * Signals all stripes.
+     */
+    private void signalStop() {
+        for (Stripe stripe : stripes)
+            stripe.signalStop();
+    }
+
+    /**
+     * @throws IgniteInterruptedException If interrupted.
+     */
+    private void awaitStop() throws IgniteInterruptedException {
+        for (Stripe stripe : stripes)
+            stripe.awaitStop();
+    }
+
+    /**
+     * @return Return total queue size of all stripes.
+     */
+    public int queueSize() {
+        int size = 0;
+
+        for (Stripe stripe : stripes)
+            size += stripe.queueSize();
+
+        return size;
+    }
+
+    /**
+     * @return Completed tasks count.
+     */
+    public long completedTasks() {
+        long cnt = 0;
+
+        for (Stripe stripe : stripes)
+            cnt += stripe.completedCnt;
+
+        return cnt;
+    }
+
+    /**
+     * Operation not supported.
+     */
+    @NotNull @Override public <T> Future<T> submit(
+        @NotNull Runnable task,
+        T res
+    ) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Operation not supported.
+     */
+    @NotNull @Override public Future<?> submit(@NotNull Runnable task) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Operation not supported.
+     */
+    @NotNull @Override public <T> Future<T> submit(@NotNull Callable<T> task) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Operation not supported.
+     */
+    @NotNull @Override public <T> List<Future<T>> invokeAll(@NotNull Collection<? extends Callable<T>> tasks)
+        throws InterruptedException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Operation not supported.
+     */
+    @NotNull @Override public <T> List<Future<T>> invokeAll(
+        @NotNull Collection<? extends Callable<T>> tasks,
+        long timeout,
+        @NotNull TimeUnit unit
+    ) throws InterruptedException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Operation not supported.
+     */
+    @NotNull @Override public <T> T invokeAny(@NotNull Collection<? extends Callable<T>> tasks)
+        throws InterruptedException, ExecutionException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Operation not supported.
+     */
+    @Override public <T> T invokeAny(
+        @NotNull Collection<? extends Callable<T>> tasks,
+        long timeout,
+        @NotNull TimeUnit unit
+    ) throws InterruptedException, ExecutionException, TimeoutException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(StripedExecutor.class, this);
+    }
+
+    /**
+     * Stripe.
+     */
+    private static abstract class Stripe implements Runnable {
+        /** */
+        private final String gridName;
+
+        /** */
+        private final String poolName;
+
+        /** */
+        private final int idx;
+
+        /** */
+        private final IgniteLogger log;
+
+        /** Stopping flag. */
+        private volatile boolean stopping;
+
+        /** */
+        private volatile long completedCnt;
+
+        /** */
+        private volatile boolean active;
+
+        /** Thread executing the loop. */
+        protected Thread thread;
+
+        /**
+         * @param gridName Grid name.
+         * @param poolName Pool name.
+         * @param idx Stripe index.
+         * @param log Logger.
+         */
+        public Stripe(
+            String gridName,
+            String poolName,
+            int idx,
+            IgniteLogger log
+        ) {
+            this.gridName = gridName;
+            this.poolName = poolName;
+            this.idx = idx;
+            this.log = log;
+        }
+
+        /**
+         * Starts the stripe.
+         */
+        void start() {
+            thread = new IgniteThread(gridName, poolName + "-stripe-" + idx, this);
+
+            thread.start();
+        }
+
+        /**
+         * Stop the stripe.
+         */
+        void signalStop() {
+            stopping = true;
+
+            U.interrupt(thread);
+        }
+
+        /**
+         * Await thread stop.
+         */
+        void awaitStop() {
+            try {
+                if (thread != null)
+                    thread.join();
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+
+                throw new IgniteInterruptedException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            while (!stopping) {
+                Runnable cmd;
+
+                try {
+                    cmd = take();
+
+                    if (cmd != null) {
+                        active = true;
+
+                        try {
+                            cmd.run();
+                        }
+                        finally {
+                            active = false;
+                            completedCnt++;
+                        }
+                    }
+                }
+                catch (InterruptedException e) {
+                    stopping = true;
+
+                    Thread.currentThread().interrupt();
+
+                    return;
+                }
+                catch (Throwable e) {
+                    U.error(log, "Failed to execute runnable.", e);
+                }
+            }
+        }
+
+        /**
+         * Execute the command.
+         *
+         * @param cmd Command.
+         */
+        abstract void execute(Runnable cmd);
+
+        /**
+         * @return Next runnable.
+         * @throws InterruptedException If interrupted.
+         */
+        abstract Runnable take() throws InterruptedException;
+
+        /**
+         * @return Queue size.
+         */
+        abstract int queueSize();
+
+        /**
+         * @return Stripe's queue to string presentation.
+         */
+        abstract String queueToString();
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(Stripe.class, this);
+        }
+    }
+
+    /**
+     * Stripe.
+     */
+    private static class StripeConcurrentQueue extends Stripe {
+        /** Queue. */
+        private final Queue<Runnable> queue = new ConcurrentLinkedQueue<>();
+
+        /** */
+        private volatile boolean parked;
+
+        /**
+         * @param gridName Grid name.
+         * @param poolName Pool name.
+         * @param idx Stripe index.
+         * @param log Logger.
+         */
+        public StripeConcurrentQueue(
+            String gridName,
+            String poolName,
+            int idx,
+            IgniteLogger log
+        ) {
+            super(gridName,
+                poolName,
+                idx,
+                log);
+        }
+
+        /** {@inheritDoc} */
+        @Override Runnable take() throws InterruptedException {
+            Runnable r;
+
+            for (int i = 0; i < 2048; i++) {
+                r = queue.poll();
+
+                if (r != null)
+                    return r;
+            }
+
+            parked = true;
+
+            try {
+                for (;;) {
+                    r = queue.poll();
+
+                    if (r != null)
+                        return r;
+
+                    LockSupport.park();
+
+                    if (Thread.interrupted())
+                        throw new InterruptedException();
+                }
+            }
+            finally {
+                parked = false;
+            }
+        }
+
+        /** {@inheritDoc} */
+        void execute(Runnable cmd) {
+            queue.add(cmd);
+
+            if (parked)
+                LockSupport.unpark(thread);
+        }
+
+        /** {@inheritDoc} */
+        @Override String queueToString() {
+            return String.valueOf(queue);
+        }
+
+        /** {@inheritDoc} */
+        @Override int queueSize() {
+            return queue.size();
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(StripeConcurrentQueue.class, this, super.toString());
+        }
+    }
+
+    /**
+     * Stripe.
+     */
+    private static class StripeConcurrentQueueNoPark extends Stripe {
+        /** Queue. */
+        private final Queue<Runnable> queue = new ConcurrentLinkedQueue<>();
+
+        /**
+         * @param gridName Grid name.
+         * @param poolName Pool name.
+         * @param idx Stripe index.
+         * @param log Logger.
+         */
+        public StripeConcurrentQueueNoPark(
+            String gridName,
+            String poolName,
+            int idx,
+            IgniteLogger log
+        ) {
+            super(gridName,
+                poolName,
+                idx,
+                log);
+        }
+
+        /** {@inheritDoc} */
+        @Override Runnable take() {
+            for (;;) {
+                Runnable r = queue.poll();
+
+                if (r != null)
+                    return r;
+            }
+        }
+
+        /** {@inheritDoc} */
+        void execute(Runnable cmd) {
+            queue.add(cmd);
+        }
+
+        /** {@inheritDoc} */
+        @Override int queueSize() {
+            return queue.size();
+        }
+
+        /** {@inheritDoc} */
+        @Override String queueToString() {
+            return String.valueOf(queue);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(StripeConcurrentQueueNoPark.class, this, super.toString());
+        }
+    }
+
+    /**
+     * Stripe.
+     */
+    private static class StripeConcurrentBlockingQueue extends Stripe {
+        /** Queue. */
+        private final BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();
+
+        /**
+         * @param gridName Grid name.
+         * @param poolName Pool name.
+         * @param idx Stripe index.
+         * @param log Logger.
+         */
+        public StripeConcurrentBlockingQueue(
+            String gridName,
+            String poolName,
+            int idx,
+            IgniteLogger log
+        ) {
+            super(gridName,
+                poolName,
+                idx,
+                log);
+        }
+
+        /** {@inheritDoc} */
+        @Override Runnable take() throws InterruptedException {
+            return queue.take();
+        }
+
+        /** {@inheritDoc} */
+        void execute(Runnable cmd) {
+            queue.add(cmd);
+        }
+
+        /** {@inheritDoc} */
+        @Override int queueSize() {
+            return queue.size();
+        }
+
+        /** {@inheritDoc} */
+        @Override String queueToString() {
+            return String.valueOf(queue);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(StripeConcurrentBlockingQueue.class, this, super.toString());
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFinishedFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFinishedFuture.java
index 6baedbdc76b28..dc63adc1d03b7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFinishedFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFinishedFuture.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.util.future;
 
+import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -151,6 +152,29 @@ public GridFinishedFuture(Throwable err) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public <T1> IgniteInternalFuture<T1> chain(final IgniteClosure<? super IgniteInternalFuture<T>, T1> doneCb, Executor exec) {
+        final GridFutureAdapter<T1> fut = new GridFutureAdapter<>();
+
+        exec.execute(new Runnable() {
+            @Override public void run() {
+                try {
+                    fut.onDone(doneCb.apply(GridFinishedFuture.this));
+                }
+                catch (GridClosureException e) {
+                    fut.onDone(e.unwrap());
+                }
+                catch (RuntimeException | Error e) {
+                    fut.onDone(e);
+
+                    throw e;
+                }
+            }
+        });
+
+        return fut;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridFinishedFuture.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
index 2cd534e0905a7..c8d85cda18a77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.util.future;
 
 import java.util.Arrays;
+import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.AbstractQueuedSynchronizer;
 import org.apache.ignite.IgniteCheckedException;
@@ -229,7 +230,13 @@ else if (lsnr instanceof ArrayListener)
 
     /** {@inheritDoc} */
     @Override public <T> IgniteInternalFuture<T> chain(final IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb) {
-        return new ChainFuture<>(this, doneCb);
+        return new ChainFuture<>(this, doneCb, null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public <T> IgniteInternalFuture<T> chain(final IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb,
+        Executor exec) {
+        return new ChainFuture<>(this, doneCb, exec);
     }
 
     /**
@@ -487,15 +494,17 @@ public ChainFuture() {
         /**
          * @param fut Future.
          * @param doneCb Closure.
+         * @param cbExec Optional executor to run callback.
          */
         ChainFuture(
             GridFutureAdapter<R> fut,
-            IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb
+            IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb,
+            @Nullable Executor cbExec
         ) {
             this.fut = fut;
             this.doneCb = doneCb;
 
-            fut.listen(new GridFutureChainListener<>(this, doneCb));
+            fut.listen(new GridFutureChainListener<>(this, doneCb, cbExec));
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureChainListener.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureChainListener.java
index 947b2ad81b45f..367f5d147aa66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureChainListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureChainListener.java
@@ -17,15 +17,17 @@
 
 package org.apache.ignite.internal.util.future;
 
+import java.util.concurrent.Executor;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.lang.GridClosureException;
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Future listener to fill chained future with converted result of the source future.
  */
-public class GridFutureChainListener<T, R> implements IgniteInClosure<IgniteInternalFuture<T>> {
+class GridFutureChainListener<T, R> implements IgniteInClosure<IgniteInternalFuture<T>> {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -35,21 +37,43 @@ public class GridFutureChainListener<T, R> implements IgniteInClosure<IgniteInte
     /** Done callback. */
     private final IgniteClosure<? super IgniteInternalFuture<T>, R> doneCb;
 
+    /** */
+    private Executor cbExec;
+
     /**
      * Constructs chain listener.
+     *
      *  @param fut Target future.
      * @param doneCb Done callback.
+     * @param cbExec Optional executor to run callback.
      */
     public GridFutureChainListener(
         GridFutureAdapter<R> fut,
-        IgniteClosure<? super IgniteInternalFuture<T>, R> doneCb
+        IgniteClosure<? super IgniteInternalFuture<T>, R> doneCb,
+        @Nullable Executor cbExec
     ) {
         this.fut = fut;
         this.doneCb = doneCb;
+        this.cbExec = cbExec;
     }
 
     /** {@inheritDoc} */
-    @Override public void apply(IgniteInternalFuture<T> t) {
+    @Override public void apply(final IgniteInternalFuture<T> t) {
+        if (cbExec != null) {
+            cbExec.execute(new Runnable() {
+                @Override public void run() {
+                    applyCallback(t);
+                }
+            });
+        }
+        else
+            applyCallback(t);
+    }
+
+    /**
+     * @param t Target future.
+     */
+    private void applyCallback(IgniteInternalFuture<T> t) {
         try {
             fut.onDone(doneCb.apply(t));
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/IpcToNioAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/IpcToNioAdapter.java
index 6820dc790077c..d108b56c0db4e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/IpcToNioAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/IpcToNioAdapter.java
@@ -201,7 +201,7 @@ protected HeadFilter() {
         }
 
         /** {@inheritDoc} */
-        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) {
+        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) {
             assert ses == IpcToNioAdapter.this.ses;
 
             return send((Message)msg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridAbstractCommunicationClient.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridAbstractCommunicationClient.java
index 9b014ec65cedc..f2ab93239978a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridAbstractCommunicationClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridAbstractCommunicationClient.java
@@ -35,13 +35,23 @@ public abstract class GridAbstractCommunicationClient implements GridCommunicati
     /** Metrics listener. */
     protected final GridNioMetricsListener metricsLsnr;
 
+    /** */
+    private final int connIdx;
+
     /**
+     * @param connIdx Connection index.
      * @param metricsLsnr Metrics listener.
      */
-    protected GridAbstractCommunicationClient(@Nullable GridNioMetricsListener metricsLsnr) {
+    protected GridAbstractCommunicationClient(int connIdx, @Nullable GridNioMetricsListener metricsLsnr) {
+        this.connIdx = connIdx;
         this.metricsLsnr = metricsLsnr;
     }
 
+    /** {@inheritDoc} */
+    @Override public int connectionIndex() {
+        return connIdx;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean close() {
         return reserves.compareAndSet(0, -1);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridCommunicationClient.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridCommunicationClient.java
index 0de54e99434e3..71b2c24daacd8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridCommunicationClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridCommunicationClient.java
@@ -96,15 +96,20 @@ public interface GridCommunicationClient {
     /**
      * @param nodeId Remote node ID. Provided only for sync clients.
      * @param msg Message to send.
-     * @param closure Ack closure.
+     * @param c Ack closure.
      * @throws IgniteCheckedException If failed.
      * @return {@code True} if should try to resend message.
      */
-    public boolean sendMessage(@Nullable UUID nodeId, Message msg, @Nullable IgniteInClosure<IgniteException> closure)
+    public boolean sendMessage(@Nullable UUID nodeId, Message msg, @Nullable IgniteInClosure<IgniteException> c)
         throws IgniteCheckedException;
 
     /**
      * @return {@code True} if send is asynchronous.
      */
     public boolean async();
+
+    /**
+     * @return Connection index.
+     */
+    public int connectionIndex();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java
index 213fd8dd040a6..7987d3db7d197 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java
@@ -62,13 +62,20 @@ public GridConnectionBytesVerifyFilter(IgniteLogger log) {
     }
 
     /** {@inheritDoc} */
-    @Override public void onExceptionCaught(GridNioSession ses, IgniteCheckedException ex) throws IgniteCheckedException {
+    @Override public void onExceptionCaught(
+        GridNioSession ses,
+        IgniteCheckedException ex
+    ) throws IgniteCheckedException {
         proceedExceptionCaught(ses, ex);
     }
 
     /** {@inheritDoc} */
-    @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
-        return proceedSessionWrite(ses, msg);
+    @Override public GridNioFuture<?> onSessionWrite(
+        GridNioSession ses,
+        Object msg,
+        boolean fut
+    ) throws IgniteCheckedException {
+        return proceedSessionWrite(ses, msg, fut);
     }
 
     /** {@inheritDoc} */
@@ -137,4 +144,4 @@ else if (U.bytesEqual(magicBuf, 0, U.IGNITE_HEADER, 0, U.IGNITE_HEADER.length))
     @Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
         proceedSessionWriteTimeout(ses);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioAsyncNotifyFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioAsyncNotifyFilter.java
index 9925d2edd7b5f..40c87cb699039 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioAsyncNotifyFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioAsyncNotifyFilter.java
@@ -107,8 +107,12 @@ public GridNioAsyncNotifyFilter(String gridName, Executor exec, IgniteLogger log
     }
 
     /** {@inheritDoc} */
-    @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
-        return proceedSessionWrite(ses, msg);
+    @Override public GridNioFuture<?> onSessionWrite(
+        GridNioSession ses,
+        Object msg,
+        boolean fut
+    ) throws IgniteCheckedException {
+        return proceedSessionWrite(ses, msg, fut);
     }
 
     /** {@inheritDoc} */
@@ -139,4 +143,4 @@ private void handleException(GridNioSession ses, IgniteCheckedException ex) {
                 "originalEx=" + ex + ", ex=" + e + ']');
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java
index 7083ccf492f17..343e62501b943 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java
@@ -71,20 +71,27 @@ public GridNioCodecFilter(GridNioParser parser, IgniteLogger log, boolean direct
     }
 
     /** {@inheritDoc} */
-    @Override public void onExceptionCaught(GridNioSession ses, IgniteCheckedException ex) throws IgniteCheckedException {
+    @Override public void onExceptionCaught(
+        GridNioSession ses,
+        IgniteCheckedException ex
+    ) throws IgniteCheckedException {
         proceedExceptionCaught(ses, ex);
     }
 
     /** {@inheritDoc} */
-    @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+    @Override public GridNioFuture<?> onSessionWrite(
+        GridNioSession ses,
+        Object msg,
+        boolean fut
+    ) throws IgniteCheckedException {
         // No encoding needed in direct mode.
         if (directMode)
-            return proceedSessionWrite(ses, msg);
+            return proceedSessionWrite(ses, msg, fut);
 
         try {
             ByteBuffer res = parser.encode(ses, msg);
 
-            return proceedSessionWrite(ses, res);
+            return proceedSessionWrite(ses, res, fut);
         }
         catch (IOException e) {
             throw new GridNioException(e);
@@ -137,4 +144,4 @@ public GridNioCodecFilter(GridNioParser parser, IgniteLogger log, boolean direct
     @Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
         proceedSessionWriteTimeout(ses);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilter.java
index 5f88b1f5b4152..f7928c452b534 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilter.java
@@ -105,10 +105,15 @@ public interface GridNioFilter {
      *
      * @param ses Session instance.
      * @param msg Message to send.
-     * @return Write future.
+     * @param fut {@code True} if write future should be created.
+     * @return Write future or {@code null}.
      * @throws IgniteCheckedException If filter is not in chain or GridNioException occurred in the underlying filter.
      */
-    public GridNioFuture<?> proceedSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException;
+    public GridNioFuture<?> proceedSessionWrite(
+        GridNioSession ses,
+        Object msg,
+        boolean fut
+    ) throws IgniteCheckedException;
 
     /**
      * Forwards session close request to the next logical filter in filter chain.
@@ -149,10 +154,11 @@ public interface GridNioFilter {
      *
      * @param ses Session on which message should be written.
      * @param msg Message being written.
-     * @return Write future.
+     * @param fut {@code True} if write future should be created.
+     * @return Write future or {@code null}.
      * @throws GridNioException If GridNioException occurred while handling event.
      */
-    public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException;
+    public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) throws IgniteCheckedException;
 
     /**
      * Invoked when a new messages received.
@@ -241,4 +247,4 @@ public interface GridNioFilter {
      * @throws IgniteCheckedException If filter is not in chain or GridNioException occurred in the underlying filter.
      */
     public GridNioFuture<?> onResumeReads(GridNioSession ses) throws IgniteCheckedException;
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterAdapter.java
index 18ab1b2f7e796..58ddae5575243 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterAdapter.java
@@ -108,10 +108,14 @@ public String toString() {
     }
 
     /** {@inheritDoc} */
-    @Override public GridNioFuture<?> proceedSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+    @Override public GridNioFuture<?> proceedSessionWrite(
+        GridNioSession ses,
+        Object msg,
+        boolean fut
+    ) throws IgniteCheckedException {
         checkNext();
 
-        return nextFilter.onSessionWrite(ses, msg);
+        return nextFilter.onSessionWrite(ses, msg, fut);
     }
 
     /** {@inheritDoc} */
@@ -180,4 +184,4 @@ private void checkNext() throws GridNioException {
             throw new GridNioException("Failed to proceed with filter call since previous filter is not set " +
                 "(do you use filter outside the filter chain?): " + getClass().getName());
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java
index a3a74e3c235a9..8cc690b144f7b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java
@@ -181,8 +181,12 @@ public String toString() {
      * @return Send future.
      * @throws IgniteCheckedException If IgniteCheckedException occurred while handling event.
      */
-    @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
-        return tail.onSessionWrite(ses, msg);
+    @Override public GridNioFuture<?> onSessionWrite(
+        GridNioSession ses,
+        Object msg,
+        boolean fut
+    ) throws IgniteCheckedException {
+        return tail.onSessionWrite(ses, msg, fut);
     }
 
     /**
@@ -255,9 +259,9 @@ private TailFilter() {
         }
 
         /** {@inheritDoc} */
-        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg)
+        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut)
             throws IgniteCheckedException {
-            return proceedSessionWrite(ses, msg);
+            return proceedSessionWrite(ses, msg, fut);
         }
 
         /** {@inheritDoc} */
@@ -290,4 +294,4 @@ private TailFilter() {
             return proceedResumeReads(ses);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFuture.java
index b02acc836eacc..6c0c9c641ff89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFuture.java
@@ -45,9 +45,9 @@ public interface GridNioFuture<R> extends IgniteInternalFuture<R> {
     /**
      * Sets ack closure which will be applied when ack received.
      *
-     * @param closure Ack closure.
+     * @param c Ack closure.
      */
-    public void ackClosure(IgniteInClosure<IgniteException> closure);
+    public void ackClosure(IgniteInClosure<IgniteException> c);
 
     /**
      * The method will be called when ack received.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
index 35480ac0bc9dd..6258c13287075 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
@@ -35,8 +35,8 @@ public class GridNioRecoveryDescriptor {
     /** Number of acknowledged messages. */
     private long acked;
 
-    /** Unacknowledged message futures. */
-    private final ArrayDeque<GridNioFuture<?>> msgFuts;
+    /** Unacknowledged messages. */
+    private final ArrayDeque<SessionWriteRequest> msgReqs;
 
     /** Number of messages to resend. */
     private int resendCnt;
@@ -77,22 +77,39 @@ public class GridNioRecoveryDescriptor {
     /** Number of descriptor reservations (for info purposes). */
     private int reserveCnt;
 
+    /** */
+    private final boolean pairedConnections;
+
     /**
+     * @param pairedConnections {@code True} if in/out connections pair is used for communication with node.
      * @param queueLimit Maximum size of unacknowledged messages queue.
      * @param node Node.
      * @param log Logger.
      */
-    public GridNioRecoveryDescriptor(int queueLimit, ClusterNode node, IgniteLogger log) {
+    public GridNioRecoveryDescriptor(
+        boolean pairedConnections,
+        int queueLimit,
+        ClusterNode node,
+        IgniteLogger log
+    ) {
         assert !node.isLocal() : node;
         assert queueLimit > 0;
 
-        msgFuts = new ArrayDeque<>(queueLimit);
+        msgReqs = new ArrayDeque<>(queueLimit);
 
+        this.pairedConnections = pairedConnections;
         this.queueLimit = queueLimit;
         this.node = node;
         this.log = log;
     }
 
+    /**
+     * @return {@code True} if in/out connections pair is used for communication with node.
+     */
+    public boolean pairedConnections() {
+        return pairedConnections;
+    }
+
     /**
      * @return Connect count.
      */
@@ -154,19 +171,19 @@ public int queueLimit() {
     }
 
     /**
-     * @param fut NIO future.
+     * @param req Write request.
      * @return {@code False} if queue limit is exceeded.
      */
-    public boolean add(GridNioFuture<?> fut) {
-        assert fut != null;
+    public boolean add(SessionWriteRequest req) {
+        assert req != null;
 
-        if (!fut.skipRecovery()) {
+        if (!req.skipRecovery()) {
             if (resendCnt == 0) {
-                msgFuts.addLast(fut);
+                msgReqs.addLast(req);
 
                 sentCnt++;
 
-                return msgFuts.size() < queueLimit;
+                return msgReqs.size() < queueLimit;
             }
             else
                 resendCnt--;
@@ -181,21 +198,19 @@ public boolean add(GridNioFuture<?> fut) {
     public void ackReceived(long rcvCnt) {
         if (log.isDebugEnabled())
             log.debug("Handle acknowledgment [acked=" + acked + ", rcvCnt=" + rcvCnt +
-                ", msgFuts=" + msgFuts.size() + ']');
+                ", msgReqs=" + msgReqs.size() + ']');
 
         while (acked < rcvCnt) {
-            GridNioFuture<?> fut = msgFuts.pollFirst();
+            SessionWriteRequest req = msgReqs.pollFirst();
 
-            assert fut != null : "Missed message future [rcvCnt=" + rcvCnt +
+            assert req != null : "Missed message [rcvCnt=" + rcvCnt +
                 ", acked=" + acked +
                 ", desc=" + this + ']';
 
-            assert fut.isDone() : fut;
-
-            if (fut.ackClosure() != null)
-                fut.ackClosure().apply(null);
+            if (req.ackClosure() != null)
+                req.ackClosure().apply(null);
 
-            fut.onAckReceived();
+            req.onAckReceived();
 
             acked++;
         }
@@ -214,7 +229,7 @@ public long acked() {
      * @return {@code False} if descriptor is reserved.
      */
     public boolean onNodeLeft() {
-        GridNioFuture<?>[] futs = null;
+        SessionWriteRequest[] reqs = null;
 
         synchronized (this) {
             nodeLeft = true;
@@ -222,24 +237,24 @@ public boolean onNodeLeft() {
             if (reserved)
                 return false;
 
-            if (!msgFuts.isEmpty()) {
-                futs = msgFuts.toArray(new GridNioFuture<?>[msgFuts.size()]);
+            if (!msgReqs.isEmpty()) {
+                reqs = msgReqs.toArray(new SessionWriteRequest[msgReqs.size()]);
 
-                msgFuts.clear();
+                msgReqs.clear();
             }
         }
 
-        if (futs != null)
-            completeOnNodeLeft(futs);
+        if (reqs != null)
+            notifyOnNodeLeft(reqs);
 
         return true;
     }
 
     /**
-     * @return Message futures for unacknowledged messages.
+     * @return Requests for unacknowledged messages.
      */
-    public Deque<GridNioFuture<?>> messagesFutures() {
-        return msgFuts;
+    public Deque<SessionWriteRequest> messagesRequests() {
+        return msgReqs;
     }
 
     /**
@@ -277,14 +292,14 @@ public void onHandshake(long rcvCnt) {
             if (!nodeLeft)
                 ackReceived(rcvCnt);
 
-            resendCnt = msgFuts.size();
+            resendCnt = msgReqs.size();
         }
     }
 
     /**
      *
      */
-    public void connected() {
+    public void onConnected() {
         synchronized (this) {
             assert reserved : this;
             assert !connected : this;
@@ -305,11 +320,38 @@ public void connected() {
         }
     }
 
+    /**
+     * @return Connected flag.
+     */
+    public boolean connected() {
+        synchronized (this) {
+            return connected;
+        }
+    }
+
+    /**
+     * @return Reserved flag.
+     */
+    public boolean reserved() {
+        synchronized (this) {
+            return reserved;
+        }
+    }
+
+    /**
+     * @return Current handshake index.
+     */
+    public Long handshakeIndex() {
+        synchronized (this) {
+            return handshakeReq != null ? handshakeReq.get1() : null;
+        }
+    }
+
     /**
      *
      */
     public void release() {
-        GridNioFuture<?>[] futs = null;
+        SessionWriteRequest[] futs = null;
 
         synchronized (this) {
             connected = false;
@@ -329,15 +371,15 @@ public void release() {
                 notifyAll();
             }
 
-            if (nodeLeft && !msgFuts.isEmpty()) {
-                futs = msgFuts.toArray(new GridNioFuture<?>[msgFuts.size()]);
+            if (nodeLeft && !msgReqs.isEmpty()) {
+                futs = msgReqs.toArray(new SessionWriteRequest[msgReqs.size()]);
 
-                msgFuts.clear();
+                msgReqs.clear();
             }
         }
 
         if (futs != null)
-            completeOnNodeLeft(futs);
+            notifyOnNodeLeft(futs);
     }
 
     /**
@@ -398,16 +440,16 @@ public int reserveCount() {
     }
 
     /**
-     * @param futs Futures to complete.
+     * @param reqs Requests to notify about error.
      */
-    private void completeOnNodeLeft(GridNioFuture<?>[] futs) {
-        for (GridNioFuture<?> msg : futs) {
-            IOException e = new IOException("Failed to send message, node has left: " + node.id());
+    private void notifyOnNodeLeft(SessionWriteRequest[] reqs) {
+        IOException e = new IOException("Failed to send message, node has left: " + node.id());
 
-            ((GridNioFutureImpl)msg).onDone(e);
+        for (SessionWriteRequest req : reqs) {
+            req.onError(e);
 
-            if (msg.ackClosure() != null)
-                msg.ackClosure().apply(new IgniteException(e));
+            if (req.ackClosure() != null)
+                req.ackClosure().apply(new IgniteException(e));
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index c8e2e0be067a8..bc1f173a29af4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -37,6 +37,7 @@
 import java.nio.channels.spi.SelectorProvider;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Deque;
 import java.util.Iterator;
 import java.util.List;
@@ -44,6 +45,8 @@
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
@@ -63,6 +66,7 @@
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
 import org.apache.ignite.plugin.extensions.communication.MessageWriter;
@@ -86,11 +90,14 @@
  *
  */
 public class GridNioServer<T> {
+    /** */
+    public static final String IGNITE_IO_BALANCE_RANDOM_BALANCE = "IGNITE_IO_BALANCE_RANDOM_BALANCER";
+
     /** Default session write timeout. */
     public static final int DFLT_SES_WRITE_TIMEOUT = 5000;
 
     /** Default send queue limit. */
-    public static final int DFLT_SEND_QUEUE_LIMIT = 1024;
+    public static final int DFLT_SEND_QUEUE_LIMIT = 0;
 
     /** Time, which server will wait before retry operation. */
     private static final long ERR_WAIT_TIME = 2000;
@@ -122,6 +129,9 @@ public class GridNioServer<T> {
         }
     }
 
+    /** Defines how many times selector should do {@code selectNow()} before doing {@code select(long)}. */
+    private long selectorSpins;
+
     /** Accept worker thread. */
     @GridToStringExclude
     private final IgniteThread acceptThread;
@@ -145,9 +155,13 @@ public class GridNioServer<T> {
     /** Flag indicating if this server should use direct buffers. */
     private final boolean directBuf;
 
-    /** Index to select which thread will serve next socket channel. Using round-robin balancing. */
+    /** Index to select which thread will serve next incoming socket channel. Using round-robin balancing. */
+    @GridToStringExclude
+    private int readBalanceIdx;
+
+    /** Index to select which thread will serve next out socket channel. Using round-robin balancing. */
     @GridToStringExclude
-    private int balanceIdx;
+    private int writeBalanceIdx = 1;
 
     /** Tcp no delay flag. */
     private final boolean tcpNoDelay;
@@ -204,12 +218,25 @@ public class GridNioServer<T> {
     /** Optional listener to monitor outbound message queue size. */
     private IgniteBiInClosure<GridNioSession, Integer> msgQueueLsnr;
 
+    /** */
+    private final AtomicLong readerMoveCnt = new AtomicLong();
+
+    /** */
+    private final AtomicLong writerMoveCnt = new AtomicLong();
+
+    /** */
+    private final IgniteRunnable balancer;
+
     /**
      * @param addr Address.
      * @param port Port.
      * @param log Log.
      * @param selectorCnt Count of selectors and selecting threads.
      * @param gridName Grid name.
+     * @param srvName Logical server name for threads identification.
+     * @param selectorSpins Defines how many non-blocking {@code selector.selectNow()} should be made before
+     *      falling into {@code selector.select(long)} in NIO server. Long value. Default is {@code 0}.
+     *      Can be set to {@code Long.MAX_VALUE} so selector threads will never block.
      * @param tcpNoDelay If TCP_NODELAY option should be set to accepted sockets.
      * @param directBuf Direct buffer flag.
      * @param order Byte order.
@@ -223,6 +250,7 @@ public class GridNioServer<T> {
      * @param writerFactory Writer factory.
      * @param skipRecoveryPred Skip recovery predicate.
      * @param msgQueueLsnr Message queue size listener.
+     * @param balancing NIO sessions balancing flag.
      * @param filters Filters for this server.
      * @throws IgniteCheckedException If failed.
      */
@@ -232,6 +260,8 @@ private GridNioServer(
         IgniteLogger log,
         int selectorCnt,
         @Nullable String gridName,
+        @Nullable String srvName,
+        long selectorSpins,
         boolean tcpNoDelay,
         boolean directBuf,
         ByteOrder order,
@@ -245,6 +275,7 @@ private GridNioServer(
         GridNioMessageWriterFactory writerFactory,
         IgnitePredicate<Message> skipRecoveryPred,
         IgniteBiInClosure<GridNioSession, Integer> msgQueueLsnr,
+        boolean balancing,
         GridNioFilter... filters
     ) throws IgniteCheckedException {
         if (port != -1)
@@ -268,6 +299,7 @@ private GridNioServer(
         this.sockSndBuf = sockSndBuf;
         this.sndQueueLimit = sndQueueLimit;
         this.msgQueueLsnr = msgQueueLsnr;
+        this.selectorSpins = selectorSpins;
 
         filterChain = new GridNioFilterChain<>(log, lsnr, new HeadFilter(), filters);
 
@@ -299,9 +331,16 @@ private GridNioServer(
         clientThreads = new IgniteThread[selectorCnt];
 
         for (int i = 0; i < selectorCnt; i++) {
+            String threadName;
+
+            if (srvName == null)
+                threadName = "grid-nio-worker-" + i;
+            else
+                threadName = "grid-nio-worker-" + srvName + "-" + i;
+
             AbstractNioClientWorker worker = directMode ?
-                new DirectNioClientWorker(i, gridName, "grid-nio-worker-" + i, log) :
-                new ByteBufferNioClientWorker(i, gridName, "grid-nio-worker-" + i, log);
+                new DirectNioClientWorker(i, gridName, threadName, log) :
+                new ByteBufferNioClientWorker(i, gridName, threadName, log);
 
             clientWorkers.add(worker);
 
@@ -315,6 +354,32 @@ private GridNioServer(
         this.writerFactory = writerFactory;
 
         this.skipRecoveryPred = skipRecoveryPred != null ? skipRecoveryPred : F.<Message>alwaysFalse();
+
+        long balancePeriod = IgniteSystemProperties.getLong(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, 5000);
+
+        IgniteRunnable balancer0 = null;
+
+        if (balancing && balancePeriod > 0) {
+            boolean rndBalance = IgniteSystemProperties.getBoolean(IGNITE_IO_BALANCE_RANDOM_BALANCE, false);
+
+            balancer0 = rndBalance ? new RandomBalancer() : new SizeBasedBalancer(balancePeriod);
+        }
+
+        this.balancer = balancer0;
+    }
+
+    /**
+     * @return Number of reader sessions move.
+     */
+    public long readerMoveCount() {
+        return readerMoveCnt.get();
+    }
+
+    /**
+     * @return Number of reader writer move.
+     */
+    public long writerMoveCount() {
+        return writerMoveCnt.get();
     }
 
     /**
@@ -376,6 +441,13 @@ public InetSocketAddress localAddress() {
         return locAddr;
     }
 
+    /**
+     * @return Selector spins.
+     */
+    public long selectorSpins() {
+        return selectorSpins;
+    }
+
     /**
      * @param ses Session to close.
      * @return Future for operation.
@@ -390,7 +462,7 @@ public GridNioFuture<Boolean> close(GridNioSession ses) {
 
         NioOperationFuture<Boolean> fut = new NioOperationFuture<>(impl, NioOperation.CLOSE);
 
-        clientWorkers.get(impl.selectorIndex()).offer(fut);
+        impl.offerStateChange(fut);
 
         return fut;
     }
@@ -398,61 +470,91 @@ public GridNioFuture<Boolean> close(GridNioSession ses) {
     /**
      * @param ses Session.
      * @param msg Message.
+     * @param createFut {@code True} if future should be created.
      * @return Future for operation.
      */
-    GridNioFuture<?> send(GridNioSession ses, ByteBuffer msg) {
-        assert ses instanceof GridSelectorNioSessionImpl;
+    GridNioFuture<?> send(GridNioSession ses, ByteBuffer msg, boolean createFut) throws IgniteCheckedException {
+        assert ses instanceof GridSelectorNioSessionImpl : ses;
 
         GridSelectorNioSessionImpl impl = (GridSelectorNioSessionImpl)ses;
 
-        NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg);
+        if (createFut) {
+            NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg);
 
-        send0(impl, fut, false);
+            send0(impl, fut, false);
 
-        return fut;
+            return fut;
+        }
+        else {
+            SessionWriteRequest req = new WriteRequestImpl(ses, msg, true);
+
+            send0(impl, req, false);
+
+            return null;
+        }
     }
 
     /**
      * @param ses Session.
      * @param msg Message.
+     * @param createFut {@code True} if future should be created.
      * @return Future for operation.
      */
-    GridNioFuture<?> send(GridNioSession ses, Message msg) {
+    GridNioFuture<?> send(GridNioSession ses, Message msg, boolean createFut) throws IgniteCheckedException {
         assert ses instanceof GridSelectorNioSessionImpl;
 
         GridSelectorNioSessionImpl impl = (GridSelectorNioSessionImpl)ses;
 
-        NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg,
-            skipRecoveryPred.apply(msg));
+        if (createFut) {
+            NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg,
+                skipRecoveryPred.apply(msg));
 
-        send0(impl, fut, false);
+            send0(impl, fut, false);
 
-        return fut;
+            return fut;
+        }
+        else {
+            SessionWriteRequest req = new WriteRequestImpl(ses, msg, skipRecoveryPred.apply(msg));
+
+            send0(impl, req, false);
+
+            return null;
+        }
     }
 
     /**
      * @param ses Session.
-     * @param fut Future.
+     * @param req Request.
      * @param sys System message flag.
+     * @throws IgniteCheckedException If session was closed.
      */
-    private void send0(GridSelectorNioSessionImpl ses, NioOperationFuture<?> fut, boolean sys) {
+    private void send0(GridSelectorNioSessionImpl ses, SessionWriteRequest req, boolean sys) throws IgniteCheckedException {
         assert ses != null;
-        assert fut != null;
+        assert req != null;
 
-        int msgCnt = sys ? ses.offerSystemFuture(fut) : ses.offerFuture(fut);
+        int msgCnt = sys ? ses.offerSystemFuture(req) : ses.offerFuture(req);
 
         IgniteInClosure<IgniteException> ackC;
 
         if (!sys && (ackC = ses.removeMeta(ACK_CLOSURE.ordinal())) != null)
-            fut.ackClosure(ackC);
+            req.ackClosure(ackC);
 
         if (ses.closed()) {
-            if (ses.removeFuture(fut))
-                fut.connectionClosed();
+            if (ses.removeFuture(req)) {
+                IOException err = new IOException("Failed to send message (connection was closed): " + ses);
+
+                req.onError(err);
+
+                if (!(req instanceof GridNioFuture))
+                    throw new IgniteCheckedException(err);
+            }
+        }
+        else if (!ses.procWrite.get() && ses.procWrite.compareAndSet(false, true)) {
+            AbstractNioClientWorker worker = (AbstractNioClientWorker)ses.worker();
+
+            if (worker != null)
+                worker.offer((SessionChangeRequest)req);
         }
-        else if (msgCnt == 1)
-            // Change from 0 to 1 means that worker thread should be waken up.
-            clientWorkers.get(ses.selectorIndex()).offer(fut);
 
         if (msgQueueLsnr != null)
             msgQueueLsnr.apply(ses, msgCnt);
@@ -463,10 +565,10 @@ else if (msgCnt == 1)
      *
      * @param ses Session.
      * @param msg Message.
-     * @return Future.
+     * @throws IgniteCheckedException If session was closed.
      */
-    public GridNioFuture<?> sendSystem(GridNioSession ses, Message msg) {
-        return sendSystem(ses, msg, null);
+    public void sendSystem(GridNioSession ses, Message msg) throws IgniteCheckedException {
+        sendSystem(ses, msg, null);
     }
 
     /**
@@ -475,27 +577,30 @@ public GridNioFuture<?> sendSystem(GridNioSession ses, Message msg) {
      * @param ses Session.
      * @param msg Message.
      * @param lsnr Future listener notified from the session thread.
-     * @return Future.
+     * @throws IgniteCheckedException If session was closed.
      */
-    public GridNioFuture<?> sendSystem(GridNioSession ses,
+    public void sendSystem(GridNioSession ses,
         Message msg,
-        @Nullable IgniteInClosure<? super IgniteInternalFuture<?>> lsnr) {
+        @Nullable IgniteInClosure<? super IgniteInternalFuture<?>> lsnr) throws IgniteCheckedException {
         assert ses instanceof GridSelectorNioSessionImpl;
 
         GridSelectorNioSessionImpl impl = (GridSelectorNioSessionImpl)ses;
 
-        NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg,
-            skipRecoveryPred.apply(msg));
-
         if (lsnr != null) {
+            NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg,
+                skipRecoveryPred.apply(msg));
+
             fut.listen(lsnr);
 
             assert !fut.isDone();
-        }
 
-        send0(impl, fut, true);
+            send0(impl, fut, true);
+        }
+        else {
+            SessionWriteRequest req = new WriteRequestSystemImpl(ses, msg);
 
-        return fut;
+            send0(impl, req, true);
+        }
     }
 
     /**
@@ -504,37 +609,69 @@ public GridNioFuture<?> sendSystem(GridNioSession ses,
     public void resend(GridNioSession ses) {
         assert ses instanceof GridSelectorNioSessionImpl;
 
-        GridNioRecoveryDescriptor recoveryDesc = ses.recoveryDescriptor();
+        GridNioRecoveryDescriptor recoveryDesc = ses.outRecoveryDescriptor();
 
-        if (recoveryDesc != null && !recoveryDesc.messagesFutures().isEmpty()) {
-            Deque<GridNioFuture<?>> futs = recoveryDesc.messagesFutures();
+        if (recoveryDesc != null && !recoveryDesc.messagesRequests().isEmpty()) {
+            Deque<SessionWriteRequest> futs = recoveryDesc.messagesRequests();
 
             if (log.isDebugEnabled())
                 log.debug("Resend messages [rmtNode=" + recoveryDesc.node().id() + ", msgCnt=" + futs.size() + ']');
 
             GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl)ses;
 
-            GridNioFuture<?> fut0 = futs.iterator().next();
+            SessionWriteRequest fut0 = futs.iterator().next();
 
-            for (GridNioFuture<?> fut : futs) {
+            for (SessionWriteRequest fut : futs) {
                 fut.messageThread(true);
 
-                ((NioOperationFuture)fut).resetSession(ses0);
+                fut.resetSession(ses0);
             }
 
             ses0.resend(futs);
 
             // Wake up worker.
-            clientWorkers.get(ses0.selectorIndex()).offer(((NioOperationFuture)fut0));
+            ses0.offerStateChange((GridNioServer.SessionChangeRequest)fut0);
         }
     }
 
+    /**
+     * @return Sessions.
+     */
+    public Collection<? extends GridNioSession> sessions() {
+        return sessions;
+    }
+
+    /**
+     * @return Workers.
+     */
+    public List<AbstractNioClientWorker> workers() {
+        return clientWorkers;
+    }
+
+    /**
+     * @param ses Session.
+     * @param from Move from index.
+     * @param to Move to index.
+     */
+    private void moveSession(GridNioSession ses, int from, int to) {
+        assert from >= 0 && from < clientWorkers.size() : from;
+        assert to >= 0 && to < clientWorkers.size() : to;
+        assert from != to;
+
+        GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl)ses;
+
+        SessionMoveFuture fut = new SessionMoveFuture(ses0, to);
+
+        if (!ses0.offerMove(clientWorkers.get(from), fut))
+            fut.onDone(false);
+    }
+
     /**
      * @param ses Session.
      * @param op Operation.
      * @return Future for operation.
      */
-    GridNioFuture<?> pauseResumeReads(GridNioSession ses, NioOperation op) {
+    private GridNioFuture<?> pauseResumeReads(GridNioSession ses, NioOperation op) {
         assert ses instanceof GridSelectorNioSessionImpl;
         assert op == NioOperation.PAUSE_READ || op == NioOperation.RESUME_READ;
 
@@ -546,7 +683,7 @@ GridNioFuture<?> pauseResumeReads(GridNioSession ses, NioOperation op) {
 
         NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, op);
 
-        clientWorkers.get(impl.selectorIndex()).offer(fut);
+        impl.offerStateChange(fut);
 
         return fut;
     }
@@ -555,6 +692,9 @@ GridNioFuture<?> pauseResumeReads(GridNioSession ses, NioOperation op) {
      *
      */
     public void dumpStats() {
+        U.warn(log, "NIO server statistics [readerSesBalanceCnt=" + readerMoveCnt.get() +
+            ", writerSesBalanceCnt=" + writerMoveCnt.get() + ']');
+
         for (int i = 0; i < clientWorkers.size(); i++)
             clientWorkers.get(i).offer(new NioOperationFuture<Void>(null, NioOperation.DUMP_STATS));
     }
@@ -675,12 +815,35 @@ private Selector createSelector(@Nullable SocketAddress addr) throws IgniteCheck
      * @param req Request to balance.
      */
     private synchronized void offerBalanced(NioOperationFuture req) {
-        clientWorkers.get(balanceIdx).offer(req);
+        assert req.operation() == NioOperation.REGISTER : req;
+        assert req.socketChannel() != null : req;
+
+        int workers = clientWorkers.size();
+
+        int balanceIdx;
+
+        if (workers > 1) {
+            if (req.accepted()) {
+                balanceIdx = readBalanceIdx;
+
+                readBalanceIdx += 2;
+
+                if (readBalanceIdx >= workers)
+                    readBalanceIdx = 0;
+            }
+            else {
+                balanceIdx = writeBalanceIdx;
 
-        balanceIdx++;
+                writeBalanceIdx += 2;
 
-        if (balanceIdx == clientWorkers.size())
+                if (writeBalanceIdx >= workers)
+                    writeBalanceIdx = 1;
+            }
+        }
+        else
             balanceIdx = 0;
+
+        clientWorkers.get(balanceIdx).offer(req);
     }
 
     /** {@inheritDoc} */
@@ -792,21 +955,30 @@ else if (cnt == 0)
 
             while (true) {
                 ByteBuffer buf = ses.removeMeta(BUF_META_KEY);
-                NioOperationFuture<?> req = ses.removeMeta(NIO_OPERATION.ordinal());
+                SessionWriteRequest req = ses.removeMeta(NIO_OPERATION.ordinal());
 
                 // Check if there were any pending data from previous writes.
                 if (buf == null) {
                     assert req == null;
 
-                    req = (NioOperationFuture<?>)ses.pollFuture();
+                    req = ses.pollFuture();
 
                     if (req == null) {
-                        key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+                        if (ses.procWrite.get()) {
+                            ses.procWrite.set(false);
+
+                            if (ses.writeQueue().isEmpty()) {
+                                if ((key.interestOps() & SelectionKey.OP_WRITE) != 0)
+                                    key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+                            }
+                            else
+                                ses.procWrite.set(true);
+                        }
 
                         break;
                     }
 
-                    buf = req.message();
+                    buf = (ByteBuffer)req.message();
                 }
 
                 if (!skipWrite) {
@@ -841,10 +1013,15 @@ else if (cnt == 0)
                     // Message was successfully written.
                     assert req != null;
 
-                    req.onDone();
+                    req.onMessageWritten();
                 }
             }
         }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(ByteBufferNioClientWorker.class, this, super.toString());
+        }
     }
 
     /**
@@ -909,6 +1086,7 @@ protected DirectNioClientWorker(int idx, @Nullable String gridName, String name,
                 metricsLsnr.onBytesReceived(cnt);
 
             ses.bytesReceived(cnt);
+            onRead(cnt);
 
             readBuf.flip();
 
@@ -921,6 +1099,12 @@ protected DirectNioClientWorker(int idx, @Nullable String gridName, String name,
                     readBuf.compact();
                 else
                     readBuf.clear();
+
+                if (ses.hasSystemMessage() && !ses.procWrite.get()) {
+                    ses.procWrite.set(true);
+
+                    registerWrite(ses);
+                }
             }
             catch (IgniteCheckedException e) {
                 close(ses, e);
@@ -993,16 +1177,29 @@ private void processWriteSsl(SelectionKey key) throws IOException {
                 if (ses.meta(WRITE_BUF_LIMIT) != null)
                     buf.limit((int)ses.meta(WRITE_BUF_LIMIT));
 
-                NioOperationFuture<?> req = ses.removeMeta(NIO_OPERATION.ordinal());
+                SessionWriteRequest req = ses.removeMeta(NIO_OPERATION.ordinal());
 
                 while (true) {
                     if (req == null) {
-                        req = (NioOperationFuture<?>)ses.pollFuture();
+                        req = systemMessage(ses);
 
-                        if (req == null && buf.position() == 0) {
-                            key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+                        if (req == null) {
+                            req = ses.pollFuture();
 
-                            break;
+                            if (req == null && buf.position() == 0) {
+                                if (ses.procWrite.get()) {
+                                    ses.procWrite.set(false);
+
+                                    if (ses.writeQueue().isEmpty()) {
+                                        if ((key.interestOps() & SelectionKey.OP_WRITE) != 0)
+                                            key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+                                    }
+                                    else
+                                        ses.procWrite.set(true);
+                                }
+
+                                break;
+                            }
                         }
                     }
 
@@ -1010,7 +1207,7 @@ private void processWriteSsl(SelectionKey key) throws IOException {
                     boolean finished = false;
 
                     if (req != null) {
-                        msg = req.directMessage();
+                        msg = (Message)req.message();
 
                         assert msg != null;
 
@@ -1025,14 +1222,17 @@ private void processWriteSsl(SelectionKey key) throws IOException {
 
                     // Fill up as many messages as possible to write buffer.
                     while (finished) {
-                        req.onDone();
+                        req.onMessageWritten();
 
-                        req = (NioOperationFuture<?>)ses.pollFuture();
+                        req = systemMessage(ses);
+
+                        if (req == null)
+                            req = ses.pollFuture();
 
                         if (req == null)
                             break;
 
-                        msg = req.directMessage();
+                        msg = (Message)req.message();
 
                         assert msg != null;
 
@@ -1129,12 +1329,30 @@ private void writeSslSystem(GridSelectorNioSessionImpl ses, WritableByteChannel
                 ses.bytesSent(cnt);
 
                 if (!buf.hasRemaining())
-                    queue.remove(buf);
+                    queue.poll();
                 else
                     break;
             }
         }
 
+        /**
+         * @param ses Session.
+         * @return System message request.
+         */
+        private SessionWriteRequest systemMessage(GridSelectorNioSessionImpl ses) {
+            if (ses.hasSystemMessage()) {
+                Object msg = ses.systemMessage();
+
+                SessionWriteRequest req = new WriteRequestSystemImpl(ses, msg);
+
+                assert !ses.hasSystemMessage();
+
+                return req;
+            }
+
+            return null;
+        }
+
         /**
          * Processes write-ready event on the key.
          *
@@ -1147,7 +1365,7 @@ private void processWrite0(SelectionKey key) throws IOException {
 
             GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
             ByteBuffer buf = ses.writeBuffer();
-            NioOperationFuture<?> req = ses.removeMeta(NIO_OPERATION.ordinal());
+            SessionWriteRequest req = ses.removeMeta(NIO_OPERATION.ordinal());
 
             MessageWriter writer = ses.meta(MSG_WRITER.ordinal());
 
@@ -1161,12 +1379,25 @@ private void processWrite0(SelectionKey key) throws IOException {
             }
 
             if (req == null) {
-                req = (NioOperationFuture<?>)ses.pollFuture();
+                req = systemMessage(ses);
 
-                if (req == null && buf.position() == 0) {
-                    key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+                if (req == null) {
+                    req = ses.pollFuture();
 
-                    return;
+                    if (req == null && buf.position() == 0) {
+                        if (ses.procWrite.get()) {
+                            ses.procWrite.set(false);
+
+                            if (ses.writeQueue().isEmpty()) {
+                                if ((key.interestOps() & SelectionKey.OP_WRITE) != 0)
+                                    key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+                            }
+                            else
+                                ses.procWrite.set(true);
+                        }
+
+                        return;
+                    }
                 }
             }
 
@@ -1174,9 +1405,9 @@ private void processWrite0(SelectionKey key) throws IOException {
             boolean finished = false;
 
             if (req != null) {
-                msg = req.directMessage();
+                msg = (Message)req.message();
 
-                assert msg != null;
+                assert msg != null : req;
 
                 if (writer != null)
                     writer.setCurrentWriteClass(msg.getClass());
@@ -1189,14 +1420,17 @@ private void processWrite0(SelectionKey key) throws IOException {
 
             // Fill up as many messages as possible to write buffer.
             while (finished) {
-                req.onDone();
+                req.onMessageWritten();
 
-                req = (NioOperationFuture<?>)ses.pollFuture();
+                req = systemMessage(ses);
+
+                if (req == null)
+                    req = ses.pollFuture();
 
                 if (req == null)
                     break;
 
-                msg = req.directMessage();
+                msg = (Message)req.message();
 
                 assert msg != null;
 
@@ -1223,6 +1457,7 @@ private void processWrite0(SelectionKey key) throws IOException {
                     metricsLsnr.onBytesSent(cnt);
 
                 ses.bytesSent(cnt);
+                onWrite(cnt);
             }
             else {
                 // For test purposes only (skipWrite is set to true in tests only).
@@ -1242,14 +1477,19 @@ private void processWrite0(SelectionKey key) throws IOException {
             else
                 buf.clear();
         }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(DirectNioClientWorker.class, this, super.toString());
+        }
     }
 
     /**
      * Thread performing only read operations from the channel.
      */
-    private abstract class AbstractNioClientWorker extends GridWorker {
+    private abstract class AbstractNioClientWorker extends GridWorker implements GridNioWorker {
         /** Queue of change requests on this selector. */
-        private final ConcurrentLinkedQueue<NioOperationFuture> changeReqs = new ConcurrentLinkedQueue<>();
+        private final ConcurrentLinkedQueue<SessionChangeRequest> changeReqs = new ConcurrentLinkedQueue<>();
 
         /** Selector to select read events. */
         private Selector selector;
@@ -1260,6 +1500,25 @@ private abstract class AbstractNioClientWorker extends GridWorker {
         /** Worker index. */
         private final int idx;
 
+        /** */
+        private long bytesRcvd;
+
+        /** */
+        private long bytesSent;
+
+        /** */
+        private volatile long bytesRcvd0;
+
+        /** */
+        private volatile long bytesSent0;
+
+        /** Sessions assigned to this worker. */
+        private final GridConcurrentHashSet<GridSelectorNioSessionImpl> workerSessions =
+            new GridConcurrentHashSet<>();
+
+        /** {@code True} if worker has called or is about to call {@code Selector.select()}. */
+        private volatile boolean select;
+
         /**
          * @param idx Index of this worker in server's array.
          * @param gridName Grid name.
@@ -1322,15 +1581,15 @@ private void createSelector() throws IgniteCheckedException {
             try {
                 SelectedSelectionKeySet selectedKeySet = new SelectedSelectionKeySet();
 
-                Class<?> selectorImplClass =
+                Class<?> selectorImplCls =
                     Class.forName("sun.nio.ch.SelectorImpl", false, U.gridClassLoader());
 
                 // Ensure the current selector implementation is what we can instrument.
-                if (!selectorImplClass.isAssignableFrom(selector.getClass()))
+                if (!selectorImplCls.isAssignableFrom(selector.getClass()))
                     return;
 
-                Field selectedKeysField = selectorImplClass.getDeclaredField("selectedKeys");
-                Field publicSelectedKeysField = selectorImplClass.getDeclaredField("publicSelectedKeys");
+                Field selectedKeysField = selectorImplCls.getDeclaredField("selectedKeys");
+                Field publicSelectedKeysField = selectorImplCls.getDeclaredField("publicSelectedKeys");
 
                 selectedKeysField.setAccessible(true);
                 publicSelectedKeysField.setAccessible(true);
@@ -1357,48 +1616,126 @@ private void createSelector() throws IgniteCheckedException {
          *
          * @param req Change request.
          */
-        private void offer(NioOperationFuture req) {
+        public void offer(SessionChangeRequest req) {
             changeReqs.offer(req);
 
+            if (select)
+                selector.wakeup();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void offer(Collection<SessionChangeRequest> reqs) {
+            for (SessionChangeRequest req : reqs)
+                changeReqs.offer(req);
+
             selector.wakeup();
         }
 
+        /** {@inheritDoc} */
+        @Override public List<SessionChangeRequest> clearSessionRequests(GridNioSession ses) {
+            List<SessionChangeRequest> sesReqs = null;
+
+            for (SessionChangeRequest changeReq : changeReqs) {
+                if (changeReq.session() == ses && !(changeReq instanceof SessionMoveFuture)) {
+                    boolean rmv = changeReqs.remove(changeReq);
+
+                    assert rmv : changeReq;
+
+                    if (sesReqs == null)
+                        sesReqs = new ArrayList<>();
+
+                    sesReqs.add(changeReq);
+                }
+            }
+
+            return sesReqs;
+        }
+
         /**
          * Processes read and write events and registration requests.
          *
          * @throws IgniteCheckedException If IOException occurred or thread was unable to add worker to workers pool.
          */
         @SuppressWarnings("unchecked")
-        private void bodyInternal() throws IgniteCheckedException {
+        private void bodyInternal() throws IgniteCheckedException, InterruptedException {
             try {
                 long lastIdleCheck = U.currentTimeMillis();
 
+                mainLoop:
                 while (!closed && selector.isOpen()) {
-                    NioOperationFuture req;
+                    SessionChangeRequest req0;
 
-                    while ((req = changeReqs.poll()) != null) {
-                        switch (req.operation()) {
+                    while ((req0 = changeReqs.poll()) != null) {
+                        switch (req0.operation()) {
                             case REGISTER: {
-                                register(req);
+                                register((NioOperationFuture)req0);
 
                                 break;
                             }
 
-                            case REQUIRE_WRITE: {
-                                //Just register write key.
-                                SelectionKey key = req.session().key();
+                            case MOVE: {
+                                SessionMoveFuture f = (SessionMoveFuture)req0;
 
-                                if (key.isValid()) {
-                                    key.interestOps(key.interestOps() | SelectionKey.OP_WRITE);
+                                GridSelectorNioSessionImpl ses = f.session();
+
+                                if (idx == f.toIdx) {
+                                    assert f.movedSocketChannel() != null : f;
+
+                                    boolean add = workerSessions.add(ses);
+
+                                    assert add;
 
-                                    // Update timestamp to protected against false write timeout.
-                                    ((GridNioSessionImpl)key.attachment()).bytesSent(0);
+                                    ses.finishMoveSession(this);
+
+                                    if (idx % 2 == 0)
+                                        readerMoveCnt.incrementAndGet();
+                                    else
+                                        writerMoveCnt.incrementAndGet();
+
+                                    SelectionKey key = f.movedSocketChannel().register(selector,
+                                        SelectionKey.OP_READ | SelectionKey.OP_WRITE,
+                                        ses);
+
+                                    ses.key(key);
+
+                                    ses.procWrite.set(true);
+
+                                    f.onDone(true);
+                                }
+                                else {
+                                    assert f.movedSocketChannel() == null : f;
+
+                                    if (workerSessions.remove(ses)) {
+                                        ses.startMoveSession(this);
+
+                                        SelectionKey key = ses.key();
+
+                                        assert key.channel() != null : key;
+
+                                        f.movedSocketChannel((SocketChannel)key.channel());
+
+                                        key.cancel();
+
+                                        clientWorkers.get(f.toIndex()).offer(f);
+                                    }
+                                    else
+                                        f.onDone(false);
                                 }
 
                                 break;
                             }
 
+                            case REQUIRE_WRITE: {
+                                SessionWriteRequest req = (SessionWriteRequest)req0;
+
+                                registerWrite((GridSelectorNioSessionImpl)req.session());
+
+                                break;
+                            }
+
                             case CLOSE: {
+                                NioOperationFuture req = (NioOperationFuture)req0;
+
                                 if (close(req.session(), null))
                                     req.onDone(true);
                                 else
@@ -1408,6 +1745,8 @@ private void bodyInternal() throws IgniteCheckedException {
                             }
 
                             case PAUSE_READ: {
+                                NioOperationFuture req = (NioOperationFuture)req0;
+
                                 SelectionKey key = req.session().key();
 
                                 if (key.isValid()) {
@@ -1426,6 +1765,8 @@ private void bodyInternal() throws IgniteCheckedException {
                             }
 
                             case RESUME_READ: {
+                                NioOperationFuture req = (NioOperationFuture)req0;
+
                                 SelectionKey key = req.session().key();
 
                                 if (key.isValid()) {
@@ -1444,76 +1785,66 @@ private void bodyInternal() throws IgniteCheckedException {
                             }
 
                             case DUMP_STATS: {
-                                StringBuilder sb = new StringBuilder();
+                                NioOperationFuture req = (NioOperationFuture)req0;
 
-                                Set<SelectionKey> keys = selector.keys();
-
-                                sb.append(U.nl())
-                                    .append(">> Selector info [idx=").append(idx)
-                                    .append(", keysCnt=").append(keys.size())
-                                    .append("]").append(U.nl());
-
-                                for (SelectionKey key : keys) {
-                                    GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
-
-                                    MessageWriter writer = ses.meta(MSG_WRITER.ordinal());
-                                    MessageReader reader = ses.meta(GridDirectParser.READER_META_KEY);
-
-                                    sb.append("    Connection info [")
-                                        .append("rmtAddr=").append(ses.remoteAddress())
-                                        .append(", locAddr=").append(ses.localAddress());
-
-                                    GridNioRecoveryDescriptor desc = ses.recoveryDescriptor();
+                                try {
+                                    dumpStats();
+                                }
+                                finally {
+                                    // Complete the request just in case (none should wait on this future).
+                                    req.onDone(true);
+                                }
+                            }
+                        }
+                    }
 
-                                    if (desc != null) {
-                                        sb.append(", msgsSent=").append(desc.sent())
-                                            .append(", msgsAckedByRmt=").append(desc.acked())
-                                            .append(", msgsRcvd=").append(desc.received())
-                                            .append(", descIdHash=").append(System.identityHashCode(desc));
-                                    }
-                                    else
-                                        sb.append(", recoveryDesc=null");
+                    int res = 0;
 
-                                    sb.append(", bytesRcvd=").append(ses.bytesReceived())
-                                        .append(", bytesSent=").append(ses.bytesSent())
-                                        .append(", opQueueSize=").append(ses.writeQueueSize())
-                                        .append(", msgWriter=").append(writer != null ? writer.toString() : "null")
-                                        .append(", msgReader=").append(reader != null ? reader.toString() : "null");
+                    for (long i = 0; i < selectorSpins && res == 0; i++) {
+                        res = selector.selectNow();
 
-                                    int cnt = 0;
+                        if (res > 0) {
+                            // Walk through the ready keys collection and process network events.
+                            if (selectedKeys == null)
+                                processSelectedKeys(selector.selectedKeys());
+                            else
+                                processSelectedKeysOptimized(selectedKeys.flip());
+                        }
 
-                                    for (GridNioFuture<?> fut : ses.writeQueue()) {
-                                        if (cnt == 0)
-                                            sb.append(",\n opQueue=[").append(fut);
-                                        else
-                                            sb.append(',').append(fut);
+                        if (!changeReqs.isEmpty())
+                            continue mainLoop;
 
-                                        if (++cnt == 5) {
-                                            sb.append(']');
+                        // Just in case we do busy selects.
+                        long now = U.currentTimeMillis();
 
-                                            break;
-                                        }
-                                    }
+                        if (now - lastIdleCheck > 2000) {
+                            lastIdleCheck = now;
 
+                            checkIdle(selector.keys());
+                        }
 
-                                    sb.append("]").append(U.nl());
-                                }
+                        if (isCancelled())
+                            return;
+                    }
 
-                                U.warn(log, sb.toString());
+                    // Falling to blocking select.
+                    select = true;
 
-                                // Complete the request just in case (none should wait on this future).
-                                req.onDone(true);
-                            }
+                    try {
+                        if (!changeReqs.isEmpty())
+                            continue;
+
+                        // Wake up every 2 seconds to check if closed.
+                        if (selector.select(2000) > 0) {
+                            // Walk through the ready keys collection and process network events.
+                            if (selectedKeys == null)
+                                processSelectedKeys(selector.selectedKeys());
+                            else
+                                processSelectedKeysOptimized(selectedKeys.flip());
                         }
                     }
-
-                    // Wake up every 2 seconds to check if closed.
-                    if (selector.select(2000) > 0) {
-                        // Walk through the ready keys collection and process network events.
-                        if (selectedKeys == null)
-                            processSelectedKeys(selector.selectedKeys());
-                        else
-                            processSelectedKeysOptimized(selectedKeys.flip());
+                    finally {
+                        select = false;
                     }
 
                     long now = U.currentTimeMillis();
@@ -1553,6 +1884,98 @@ private void bodyInternal() throws IgniteCheckedException {
             }
         }
 
+        /**
+         * @param ses Session.
+         */
+        public final void registerWrite(GridSelectorNioSessionImpl ses) {
+            SelectionKey key = ses.key();
+
+            if (key.isValid()) {
+                if ((key.interestOps() & SelectionKey.OP_WRITE) == 0)
+                    key.interestOps(key.interestOps() | SelectionKey.OP_WRITE);
+
+                // Update timestamp to protected against false write timeout.
+                ses.bytesSent(0);
+            }
+        }
+
+        /**
+         *
+         */
+        private void dumpStats() {
+            StringBuilder sb = new StringBuilder();
+
+            Set<SelectionKey> keys = selector.keys();
+
+            sb.append(U.nl())
+                .append(">> Selector info [idx=").append(idx)
+                .append(", keysCnt=").append(keys.size())
+                .append(", bytesRcvd=").append(bytesRcvd)
+                .append(", bytesRcvd0=").append(bytesRcvd0)
+                .append(", bytesSent=").append(bytesSent)
+                .append(", bytesSent0=").append(bytesSent0)
+                .append("]").append(U.nl());
+
+            for (SelectionKey key : keys) {
+                GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
+
+                MessageWriter writer = ses.meta(MSG_WRITER.ordinal());
+                MessageReader reader = ses.meta(GridDirectParser.READER_META_KEY);
+
+                sb.append("    Connection info [")
+                    .append("in=").append(ses.accepted())
+                    .append(", rmtAddr=").append(ses.remoteAddress())
+                    .append(", locAddr=").append(ses.localAddress());
+
+                GridNioRecoveryDescriptor outDesc = ses.outRecoveryDescriptor();
+
+                if (outDesc != null) {
+                    sb.append(", msgsSent=").append(outDesc.sent())
+                        .append(", msgsAckedByRmt=").append(outDesc.acked())
+                        .append(", descIdHash=").append(System.identityHashCode(outDesc));
+                }
+                else
+                    sb.append(", outRecoveryDesc=null");
+
+                GridNioRecoveryDescriptor inDesc = ses.inRecoveryDescriptor();
+
+                if (inDesc != null) {
+                    sb.append(", msgsRcvd=").append(inDesc.received())
+                        .append(", lastAcked=").append(inDesc.lastAcknowledged())
+                        .append(", descIdHash=").append(System.identityHashCode(inDesc));
+                }
+                else
+                    sb.append(", inRecoveryDesc=null");
+
+                sb.append(", bytesRcvd=").append(ses.bytesReceived())
+                    .append(", bytesRcvd0=").append(ses.bytesReceived0())
+                    .append(", bytesSent=").append(ses.bytesSent())
+                    .append(", bytesSent0=").append(ses.bytesSent0())
+                    .append(", opQueueSize=").append(ses.writeQueueSize())
+                    .append(", msgWriter=").append(writer != null ? writer.toString() : "null")
+                    .append(", msgReader=").append(reader != null ? reader.toString() : "null");
+
+                int cnt = 0;
+
+                for (SessionWriteRequest req : ses.writeQueue()) {
+                    if (cnt == 0)
+                        sb.append(",\n opQueue=[").append(req);
+                    else
+                        sb.append(',').append(req);
+
+                    if (++cnt == 5) {
+                        sb.append(']');
+
+                        break;
+                    }
+                }
+
+                sb.append("]").append(U.nl());
+            }
+
+            U.warn(log, sb.toString());
+        }
+
         /**
          * Processes keys selected by a selector.
          *
@@ -1671,7 +2094,9 @@ private void checkIdle(Iterable<SelectionKey> keys) {
 
                     long idleTimeout0 = idleTimeout;
 
-                    if (!opWrite && now - ses.lastReceiveTime() > idleTimeout0 && now - ses.lastSendScheduleTime() > idleTimeout0) {
+                    if (!opWrite &&
+                        now - ses.lastReceiveTime() > idleTimeout0 &&
+                        now - ses.lastSendScheduleTime() > idleTimeout0) {
                         filterChain.onSessionIdleTimeout(ses);
 
                         // Update timestamp to avoid multiple notifications within one timeout interval.
@@ -1715,7 +2140,7 @@ private void register(NioOperationFuture<GridNioSession> req) {
 
                 final GridSelectorNioSessionImpl ses = new GridSelectorNioSessionImpl(
                     log,
-                    idx,
+                    this,
                     filterChain,
                     (InetSocketAddress)sockCh.getLocalAddress(),
                     (InetSocketAddress)sockCh.getRemoteAddress(),
@@ -1739,6 +2164,7 @@ private void register(NioOperationFuture<GridNioSession> req) {
                     resend(ses);
 
                 sessions.add(ses);
+                workerSessions.add(ses);
 
                 try {
                     filterChain.onSessionOpened(ses);
@@ -1764,7 +2190,7 @@ private void register(NioOperationFuture<GridNioSession> req) {
         }
 
         /**
-         * Closes the ses and all associated resources, then notifies the listener.
+         * Closes the session and all associated resources, then notifies the listener.
          *
          * @param ses Session to be closed.
          * @param e Exception to be passed to the listener, if any.
@@ -1781,12 +2207,10 @@ protected boolean close(final GridSelectorNioSessionImpl ses, @Nullable final Ig
             }
 
             sessions.remove(ses);
+            workerSessions.remove(ses);
 
             SelectionKey key = ses.key();
 
-            // Shutdown input and output so that remote client will see correct socket close.
-            Socket sock = ((SocketChannel)key.channel()).socket();
-
             if (ses.setClosed()) {
                 ses.onClosed();
 
@@ -1798,6 +2222,9 @@ protected boolean close(final GridSelectorNioSessionImpl ses, @Nullable final Ig
                         ((DirectBuffer)ses.readBuffer()).cleaner().clean();
                 }
 
+                // Shutdown input and output so that remote client will see correct socket close.
+                Socket sock = ((SocketChannel)key.channel()).socket();
+
                 try {
                     try {
                         sock.shutdownInput();
@@ -1824,28 +2251,35 @@ protected boolean close(final GridSelectorNioSessionImpl ses, @Nullable final Ig
                 ses.removeMeta(BUF_META_KEY);
 
                 // Since ses is in closed state, no write requests will be added.
-                NioOperationFuture<?> fut = ses.removeMeta(NIO_OPERATION.ordinal());
+                SessionWriteRequest req = ses.removeMeta(NIO_OPERATION.ordinal());
+
+                GridNioRecoveryDescriptor outRecovery = ses.outRecoveryDescriptor();
+                GridNioRecoveryDescriptor inRecovery = ses.inRecoveryDescriptor();
 
-                GridNioRecoveryDescriptor recovery = ses.recoveryDescriptor();
+                IOException err = new IOException("Failed to send message (connection was closed): " + ses);
 
-                if (recovery != null) {
+                if (outRecovery != null || inRecovery != null) {
                     try {
                         // Poll will update recovery data.
-                        while ((fut = (NioOperationFuture<?>)ses.pollFuture()) != null) {
-                            if (fut.skipRecovery())
-                                fut.connectionClosed();
+                        while ((req = ses.pollFuture()) != null) {
+                            if (req.skipRecovery())
+                                req.onError(err);
                         }
                     }
                     finally {
-                        recovery.release();
+                        if (outRecovery != null)
+                            outRecovery.release();
+
+                        if (inRecovery != null && inRecovery != outRecovery)
+                            inRecovery.release();
                     }
                 }
                 else {
-                    if (fut != null)
-                        fut.connectionClosed();
+                    if (req != null)
+                        req.onError(err);
 
-                    while ((fut = (NioOperationFuture<?>)ses.pollFuture()) != null)
-                        fut.connectionClosed();
+                    while ((req = ses.pollFuture()) != null)
+                        req.onError(err);
                 }
 
                 try {
@@ -1876,12 +2310,44 @@ protected boolean close(final GridSelectorNioSessionImpl ses, @Nullable final Ig
          * @throws IOException If write failed.
          */
         protected abstract void processWrite(SelectionKey key) throws IOException;
-    }
 
-    /**
-     * Gets outbound messages queue size.
-     *
-     * @return Write queue size.
+        /**
+         * @param cnt
+         */
+        final void onRead(int cnt) {
+            bytesRcvd += cnt;
+            bytesRcvd0 += cnt;
+        }
+
+        /**
+         * @param cnt
+         */
+        final void onWrite(int cnt) {
+            bytesSent += cnt;
+            bytesSent0 += cnt;
+        }
+
+        /**
+         *
+         */
+        final void reset0() {
+            bytesSent0 = 0;
+            bytesRcvd0 = 0;
+
+            for (GridSelectorNioSessionImpl ses : workerSessions)
+                ses.reset0();
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(AbstractNioClientWorker.class, this, super.toString());
+        }
+    }
+
+    /**
+     * Gets outbound messages queue size.
+     *
+     * @return Write queue size.
      */
     public int outboundMessagesQueueSize() {
         int res = 0;
@@ -1952,6 +2418,9 @@ private void accept() throws IgniteCheckedException {
                     if (selector.select(2000) > 0)
                         // Walk through the ready keys collection and process date requests.
                         processSelectedKeys(selector.selectedKeys());
+
+                    if (balancer != null)
+                        balancer.run();
                 }
             }
             // Ignore this exception as thread interruption is equal to 'close' call.
@@ -2048,10 +2517,13 @@ private void addRegistrationReq(SocketChannel sockCh) {
     /**
      * Asynchronous operation that may be requested on selector.
      */
-    private enum NioOperation {
+    enum NioOperation {
         /** Register read key selection. */
         REGISTER,
 
+        /** Move session between workers. */
+        MOVE,
+
         /** Register write key selection. */
         REQUIRE_WRITE,
 
@@ -2068,10 +2540,194 @@ private enum NioOperation {
         DUMP_STATS
     }
 
+    /**
+     *
+     */
+    private static final class WriteRequestSystemImpl implements SessionWriteRequest, SessionChangeRequest {
+        /** */
+        private final Object msg;
+
+        /** */
+        private final GridNioSession ses;
+
+        /**
+         * @param ses Session.
+         * @param msg Message.
+         */
+        WriteRequestSystemImpl(GridNioSession ses, Object msg) {
+            this.ses = ses;
+            this.msg = msg;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void messageThread(boolean msgThread) {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean messageThread() {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean skipRecovery() {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void ackClosure(IgniteInClosure<IgniteException> c) {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onAckReceived() {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteInClosure<IgniteException> ackClosure() {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onError(Exception e) {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object message() {
+            return msg;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onMessageWritten() {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void resetSession(GridNioSession ses) {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public GridNioSession session() {
+            return ses;
+        }
+
+        /** {@inheritDoc} */
+        @Override public NioOperation operation() {
+            return NioOperation.REQUIRE_WRITE;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(WriteRequestSystemImpl.class, this);
+        }
+    }
+
+    /**
+     *
+     */
+    private static final class WriteRequestImpl implements SessionWriteRequest, SessionChangeRequest {
+        /** */
+        private GridNioSession ses;
+
+        /** */
+        private final Object msg;
+
+        /** */
+        private boolean msgThread;
+
+        /** */
+        private final boolean skipRecovery;
+
+        /** */
+        private IgniteInClosure<IgniteException> ackC;
+
+        /**
+         * @param ses Session.
+         * @param msg Message.
+         * @param skipRecovery Skip recovery flag.
+         */
+        WriteRequestImpl(GridNioSession ses, Object msg, boolean skipRecovery) {
+            this.ses = ses;
+            this.msg = msg;
+            this.skipRecovery = skipRecovery;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void messageThread(boolean msgThread) {
+            this.msgThread = msgThread;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean messageThread() {
+            return msgThread;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean skipRecovery() {
+            return skipRecovery;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void ackClosure(IgniteInClosure<IgniteException> c) {
+            ackC = c;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onAckReceived() {
+            assert msg instanceof Message;
+
+            ((Message)msg).onAckReceived();
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteInClosure<IgniteException> ackClosure() {
+            return ackC;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onError(Exception e) {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object message() {
+            return msg;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onMessageWritten() {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void resetSession(GridNioSession ses) {
+            this.ses = ses;
+        }
+
+        /** {@inheritDoc} */
+        @Override public GridNioSession session() {
+            return ses;
+        }
+
+        /** {@inheritDoc} */
+        @Override public NioOperation operation() {
+            return NioOperation.REQUIRE_WRITE;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(WriteRequestImpl.class, this);
+        }
+    }
+
     /**
      * Class for requesting write and session close operations.
      */
-    private static class NioOperationFuture<R> extends GridNioFutureImpl<R> {
+    private static class NioOperationFuture<R> extends GridNioFutureImpl<R> implements SessionWriteRequest,
+        SessionChangeRequest {
         /** */
         private static final long serialVersionUID = 0L;
 
@@ -2087,11 +2743,7 @@ private static class NioOperationFuture<R> extends GridNioFutureImpl<R> {
         private NioOperation op;
 
         /** Message. */
-        @GridToStringExclude
-        private ByteBuffer msg;
-
-        /** Direct message. */
-        private Message commMsg;
+        private Object msg;
 
         /** */
         @GridToStringExclude
@@ -2153,8 +2805,7 @@ private static class NioOperationFuture<R> extends GridNioFutureImpl<R> {
          * @param op Requested operation.
          * @param msg Message.
          */
-        NioOperationFuture(GridSelectorNioSessionImpl ses, NioOperation op,
-            ByteBuffer msg) {
+        NioOperationFuture(GridSelectorNioSessionImpl ses, NioOperation op, Object msg) {
             assert ses != null;
             assert op != null;
             assert op != NioOperation.REGISTER;
@@ -2182,51 +2833,36 @@ private static class NioOperationFuture<R> extends GridNioFutureImpl<R> {
 
             this.ses = ses;
             this.op = op;
-            this.commMsg = commMsg;
+            this.msg = commMsg;
             this.skipRecovery = skipRecovery;
         }
 
-        /**
-         * @return Requested change operation.
-         */
-        private NioOperation operation() {
+        /** {@inheritDoc} */
+        public NioOperation operation() {
             return op;
         }
 
-        /**
-         * @return Message.
-         */
-        private ByteBuffer message() {
+        /** {@inheritDoc} */
+        public Object message() {
             return msg;
         }
 
-        /**
-         * @return Direct message.
-         */
-        private Message directMessage() {
-            return commMsg;
-        }
-
-        /**
-         * @param ses New session instance.
-         */
-        private void resetSession(GridSelectorNioSessionImpl ses) {
-            assert commMsg != null;
+        /** {@inheritDoc} */
+        public void resetSession(GridNioSession ses) {
+            assert msg instanceof Message : msg;
 
-            this.ses = ses;
+            this.ses = (GridSelectorNioSessionImpl)ses;
         }
 
         /**
          * @return Socket channel for register request.
          */
-        private SocketChannel socketChannel() {
+        SocketChannel socketChannel() {
             return sockCh;
         }
 
-        /**
-         * @return Session for this change request.
-         */
-        private GridSelectorNioSessionImpl session() {
+        /** {@inheritDoc} */
+        public GridSelectorNioSessionImpl session() {
             return ses;
         }
 
@@ -2244,21 +2880,21 @@ boolean accepted() {
             return meta;
         }
 
-        /**
-         * Applicable to write futures only. Fails future with corresponding IOException.
-         */
-        private void connectionClosed() {
-            assert op == NioOperation.REQUIRE_WRITE;
-            assert ses != null;
-
-            onDone(new IOException("Failed to send message (connection was closed): " + ses));
+        /** {@inheritDoc} */
+        @Override public void onError(Exception e) {
+            onDone(e);
         }
 
         /** {@inheritDoc} */
         @Override public void onAckReceived() {
-            assert commMsg != null;
+            assert msg instanceof Message : msg;
 
-            commMsg.onAckReceived();
+            ((Message)msg).onAckReceived();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onMessageWritten() {
+            onDone();
         }
 
         /** {@inheritDoc} */
@@ -2272,6 +2908,59 @@ private void connectionClosed() {
         }
     }
 
+    /**
+     *
+     */
+    private static class SessionMoveFuture extends NioOperationFuture<Boolean> {
+        /** */
+        private final int toIdx;
+
+        /** */
+        @GridToStringExclude
+        private SocketChannel movedSockCh;
+
+        /**
+         * @param ses Session.
+         * @param toIdx Target worker index.
+         */
+        SessionMoveFuture(
+            GridSelectorNioSessionImpl ses,
+            int toIdx
+        ) {
+            super(ses, NioOperation.MOVE);
+
+            this.toIdx = toIdx;
+        }
+
+        /**
+         * @return Target worker index.
+         */
+        int toIndex() {
+            return toIdx;
+        }
+
+        /**
+         * @return Moved session socket channel.
+         */
+        SocketChannel movedSocketChannel() {
+            return movedSockCh;
+        }
+
+        /**
+         * @param movedSockCh Moved session socket channel.
+         */
+        void movedSocketChannel(SocketChannel movedSockCh) {
+            assert movedSockCh != null;
+
+            this.movedSockCh = movedSockCh;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(SessionMoveFuture.class, this, super.toString());
+        }
+    }
+
     /**
      * Filter forwarding messages from chain's head to this server.
      */
@@ -2302,7 +2991,7 @@ protected HeadFilter() {
         }
 
         /** {@inheritDoc} */
-        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) {
+        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) throws IgniteCheckedException {
             if (directMode) {
                 boolean sslSys = sslFilter != null && msg instanceof ByteBuffer;
 
@@ -2313,18 +3002,18 @@ protected HeadFilter() {
 
                     queue.offer((ByteBuffer)msg);
 
-                    SelectionKey key = ((GridSelectorNioSessionImpl)ses).key();
+                    GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl)ses;
 
-                    if (key.isValid())
-                        key.interestOps(key.interestOps() | SelectionKey.OP_WRITE);
+                    if (!ses0.procWrite.get() && ses0.procWrite.compareAndSet(false, true))
+                        ses0.worker().registerWrite(ses0);
 
                     return null;
                 }
                 else
-                    return send(ses, (Message)msg);
+                    return send(ses, (Message)msg, fut);
             }
             else
-                return send(ses, (ByteBuffer)msg);
+                return send(ses, (ByteBuffer)msg, fut);
         }
 
         /** {@inheritDoc} */
@@ -2429,6 +3118,15 @@ public static class Builder<T> {
         /** Message queue size listener. */
         private IgniteBiInClosure<GridNioSession, Integer> msgQueueLsnr;
 
+        /** Name for threads identification. */
+        private String srvName;
+
+        /** */
+        private long selectorSpins;
+
+        /** NIO sessions balancing flag. */
+        private boolean balancing;
+
         /**
          * Finishes building the instance.
          *
@@ -2442,6 +3140,8 @@ public GridNioServer<T> build() throws IgniteCheckedException {
                 log,
                 selectorCnt,
                 gridName,
+                srvName,
+                selectorSpins,
                 tcpNoDelay,
                 directBuf,
                 byteOrder,
@@ -2455,6 +3155,7 @@ public GridNioServer<T> build() throws IgniteCheckedException {
                 writerFactory,
                 skipRecoveryPred,
                 msgQueueLsnr,
+                balancing,
                 filters != null ? Arrays.copyOf(filters, filters.length) : EMPTY_FILTERS
             );
 
@@ -2467,6 +3168,16 @@ public GridNioServer<T> build() throws IgniteCheckedException {
             return ret;
         }
 
+        /**
+         * @param balancing NIO sessions balancing flag.
+         * @return This for chaining.
+         */
+        public Builder<T> balancing(boolean balancing) {
+            this.balancing = balancing;
+
+            return this;
+        }
+
         /**
          * @param addr Local address.
          * @return This for chaining.
@@ -2518,6 +3229,28 @@ public Builder<T> gridName(@Nullable String gridName) {
             return this;
         }
 
+        /**
+         * @param srvName Logical server name for threads identification.
+         * @return This for chaining.
+         */
+        public Builder<T> serverName(@Nullable String srvName) {
+            this.srvName = srvName;
+
+            return this;
+        }
+
+        /**
+         * @param selectorSpins Defines how many non-blocking {@code selector.selectNow()} should be made before
+         *      falling into {@code selector.select(long)} in NIO server. Long value. Default is {@code 0}.
+         *      Can be set to {@code Long.MAX_VALUE} so selector threads will never block.
+         * @return This for chaining.
+         */
+        public Builder<T> selectorSpins(long selectorSpins) {
+            this.selectorSpins = selectorSpins;
+
+            return this;
+        }
+
         /**
          * @param tcpNoDelay If TCP_NODELAY option should be set to accepted sockets.
          * @return This for chaining.
@@ -2678,4 +3411,225 @@ public Builder<T> messageQueueSizeListener(IgniteBiInClosure<GridNioSession, Int
             return this;
         }
     }
+
+    /**
+     *
+     */
+    private class SizeBasedBalancer implements IgniteRunnable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private long lastBalance;
+
+        /** */
+        private final long balancePeriod;
+
+        /**
+         * @param balancePeriod Period.
+         */
+        SizeBasedBalancer(long balancePeriod) {
+            this.balancePeriod = balancePeriod;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            long now = U.currentTimeMillis();
+
+            if (lastBalance + balancePeriod < now) {
+                lastBalance = now;
+
+                long maxRcvd0 = -1, minRcvd0 = -1, maxSent0 = -1, minSent0 = -1;
+                int maxRcvdIdx = -1, minRcvdIdx = -1, maxSentIdx = -1, minSentIdx = -1;
+
+                for (int i = 0; i < clientWorkers.size(); i++) {
+                    GridNioServer.AbstractNioClientWorker worker = clientWorkers.get(i);
+
+                    int sesCnt = worker.workerSessions.size();
+
+                    if (i % 2 == 0) {
+                        // Reader.
+                        long bytesRcvd0 = worker.bytesRcvd0;
+
+                        if ((maxRcvd0 == -1 || bytesRcvd0 > maxRcvd0) && bytesRcvd0 > 0 && sesCnt > 1) {
+                            maxRcvd0 = bytesRcvd0;
+                            maxRcvdIdx = i;
+                        }
+
+                        if (minRcvd0 == -1 || bytesRcvd0 < minRcvd0) {
+                            minRcvd0 = bytesRcvd0;
+                            minRcvdIdx = i;
+                        }
+                    }
+                    else {
+                        // Writer.
+                        long bytesSent0 = worker.bytesSent0;
+
+                        if ((maxSent0 == -1 || bytesSent0 > maxSent0) && bytesSent0 > 0 && sesCnt > 1) {
+                            maxSent0 = bytesSent0;
+                            maxSentIdx = i;
+                        }
+
+                        if (minSent0 == -1 || bytesSent0 < minSent0) {
+                            minSent0 = bytesSent0;
+                            minSentIdx = i;
+                        }
+                    }
+                }
+
+                if (log.isDebugEnabled())
+                    log.debug("Balancing data [minSent0=" + minSent0 + ", minSentIdx=" + minSentIdx +
+                        ", maxSent0=" + maxSent0 + ", maxSentIdx=" + maxSentIdx +
+                        ", minRcvd0=" + minRcvd0 + ", minRcvdIdx=" + minRcvdIdx +
+                        ", maxRcvd0=" + maxRcvd0 + ", maxRcvdIdx=" + maxRcvdIdx + ']');
+
+                if (maxSent0 != -1 && minSent0 != -1) {
+                    GridSelectorNioSessionImpl ses = null;
+
+                    long sentDiff = maxSent0 - minSent0;
+                    long delta = sentDiff;
+                    double threshold = sentDiff * 0.9;
+
+                    GridConcurrentHashSet<GridSelectorNioSessionImpl> sessions =
+                        clientWorkers.get(maxSentIdx).workerSessions;
+
+                    for (GridSelectorNioSessionImpl ses0 : sessions) {
+                        long bytesSent0 = ses0.bytesSent0();
+
+                        if (bytesSent0 < threshold &&
+                            (ses == null || delta > U.safeAbs(bytesSent0 - sentDiff / 2))) {
+                            ses = ses0;
+                            delta = U.safeAbs(bytesSent0 - sentDiff / 2);
+                        }
+                    }
+
+                    if (ses != null) {
+                        if (log.isDebugEnabled())
+                            log.debug("Will move session to less loaded writer [ses=" + ses +
+                                ", from=" + maxSentIdx + ", to=" + minSentIdx + ']');
+
+                        moveSession(ses, maxSentIdx, minSentIdx);
+                    }
+                    else {
+                        if (log.isDebugEnabled())
+                            log.debug("Unable to find session to move for writers.");
+                    }
+                }
+
+                if (maxRcvd0 != -1 && minRcvd0 != -1) {
+                    GridSelectorNioSessionImpl ses = null;
+
+                    long rcvdDiff = maxRcvd0 - minRcvd0;
+                    long delta = rcvdDiff;
+                    double threshold = rcvdDiff * 0.9;
+
+                    GridConcurrentHashSet<GridSelectorNioSessionImpl> sessions =
+                        clientWorkers.get(maxRcvdIdx).workerSessions;
+
+                    for (GridSelectorNioSessionImpl ses0 : sessions) {
+                        long bytesRcvd0 = ses0.bytesReceived0();
+
+                        if (bytesRcvd0 < threshold &&
+                            (ses == null || delta > U.safeAbs(bytesRcvd0 - rcvdDiff / 2))) {
+                            ses = ses0;
+                            delta = U.safeAbs(bytesRcvd0 - rcvdDiff / 2);
+                        }
+                    }
+
+                    if (ses != null) {
+                        if (log.isDebugEnabled())
+                            log.debug("Will move session to less loaded reader [ses=" + ses +
+                                ", from=" + maxRcvdIdx + ", to=" + minRcvdIdx + ']');
+
+                        moveSession(ses, maxRcvdIdx, minRcvdIdx);
+                    }
+                    else {
+                        if (log.isDebugEnabled())
+                            log.debug("Unable to find session to move for readers.");
+                    }
+                }
+
+                for (int i = 0; i < clientWorkers.size(); i++) {
+                    GridNioServer.AbstractNioClientWorker worker = clientWorkers.get(i);
+
+                    worker.reset0();
+                }
+            }
+        }
+    }
+
+    /**
+     * For tests only.
+     */
+    @SuppressWarnings("unchecked")
+    private class RandomBalancer implements IgniteRunnable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+            int w1 = rnd.nextInt(clientWorkers.size());
+
+            if (clientWorkers.get(w1).workerSessions.isEmpty())
+                return;
+
+            int w2 = rnd.nextInt(clientWorkers.size());
+
+            while (w2 == w1)
+                w2 = rnd.nextInt(clientWorkers.size());
+
+            GridNioSession ses = randomSession(clientWorkers.get(w1));
+
+            if (ses != null) {
+                log.info("Move session [from=" + w1 +
+                    ", to=" + w2 +
+                    ", ses=" + ses + ']');
+
+                moveSession(ses, w1, w2);
+            }
+        }
+
+        /**
+         * @param worker Worker.
+         * @return NIO session.
+         */
+        private GridNioSession randomSession(GridNioServer.AbstractNioClientWorker worker) {
+            Collection<GridNioSession> sessions = worker.workerSessions;
+
+            int size = sessions.size();
+
+            if (size == 0)
+                return null;
+
+            int idx = ThreadLocalRandom.current().nextInt(size);
+
+            Iterator<GridNioSession> it = sessions.iterator();
+
+            int cnt = 0;
+
+            while (it.hasNext()) {
+                GridNioSession ses = it.next();
+
+                if (cnt == idx)
+                    return ses;
+            }
+
+            return null;
+        }
+
+    }
+
+    /**
+     *
+     */
+    interface SessionChangeRequest {
+        GridNioSession session();
+
+        /**
+         * @return Requested change operation.
+         */
+        NioOperation operation();
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSession.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSession.java
index e4a722585da69..c1b60abc4c1b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSession.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSession.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.util.nio;
 
 import java.net.InetSocketAddress;
+import org.apache.ignite.IgniteCheckedException;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -104,6 +105,11 @@ public interface GridNioSession {
      */
     public GridNioFuture<?> send(Object msg);
 
+    /**
+     * @param msg Message to be sent.
+     */
+    public void sendNoFuture(Object msg) throws IgniteCheckedException;
+
     /**
      * Gets metadata associated with specified key.
      *
@@ -158,10 +164,25 @@ public interface GridNioSession {
     /**
      * @param recoveryDesc Recovery descriptor.
      */
-    public void recoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc);
+    public void outRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc);
+
+    /**
+     * @param recoveryDesc Recovery descriptor.
+     */
+    public void inRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc);
 
     /**
      * @return Recovery descriptor if recovery is supported, {@code null otherwise.}
      */
-    @Nullable public GridNioRecoveryDescriptor recoveryDescriptor();
+    @Nullable public GridNioRecoveryDescriptor outRecoveryDescriptor();
+
+    /**
+     * @return Recovery descriptor if recovery is supported, {@code null otherwise.}
+     */
+    @Nullable public GridNioRecoveryDescriptor inRecoveryDescriptor();
+
+    /**
+     * @param msg System message to send.
+     */
+    public void systemMessage(Object msg);
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
index 0bcfe6434a5d5..7424531248c65 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
@@ -51,6 +51,12 @@ public class GridNioSessionImpl implements GridNioSession {
     /** Received bytes counter. */
     private volatile long bytesRcvd;
 
+    /** Sent bytes since last NIO sessions balancing. */
+    private volatile long bytesSent0;
+
+    /** Received bytes since last NIO sessions balancing. */
+    private volatile long bytesRcvd0;
+
     /** Last send schedule timestamp. */
     private volatile long sndSchedTime;
 
@@ -99,7 +105,7 @@ public GridNioSessionImpl(
         try {
             resetSendScheduleTime();
 
-            return chain().onSessionWrite(this, msg);
+            return chain().onSessionWrite(this, msg, true);
         }
         catch (IgniteCheckedException e) {
             close();
@@ -108,6 +114,18 @@ public GridNioSessionImpl(
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void sendNoFuture(Object msg) throws IgniteCheckedException {
+        try {
+            chain().onSessionWrite(this, msg, false);
+        }
+        catch (IgniteCheckedException e) {
+            close();
+
+            throw e;
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public GridNioFuture<?> resumeReads() {
         try {
@@ -163,6 +181,28 @@ public GridNioSessionImpl(
         return bytesRcvd;
     }
 
+    /**
+     * @return Sent bytes since last NIO sessions balancing.
+     */
+    public long bytesSent0() {
+        return bytesSent0;
+    }
+
+    /**
+     * @return Received bytes since last NIO sessions balancing.
+     */
+    public long bytesReceived0() {
+        return bytesRcvd0;
+    }
+
+    /**
+     *
+     */
+    public void reset0() {
+        bytesSent0 = 0;
+        bytesRcvd0 = 0;
+    }
+
     /** {@inheritDoc} */
     @Override public long createTime() {
         return createTime;
@@ -240,6 +280,7 @@ protected <T> GridNioFilterChain<T> chain() {
      */
     public void bytesSent(int cnt) {
         bytesSent += cnt;
+        bytesSent0 += cnt;
 
         lastSndTime = U.currentTimeMillis();
     }
@@ -253,6 +294,7 @@ public void bytesSent(int cnt) {
      */
     public void bytesReceived(int cnt) {
         bytesRcvd += cnt;
+        bytesRcvd0 += cnt;
 
         lastRcvTime = U.currentTimeMillis();
     }
@@ -296,17 +338,32 @@ public void readsPaused(boolean readsPaused) {
     }
 
     /** {@inheritDoc} */
-    @Override public void recoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+    @Override public void outRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public GridNioRecoveryDescriptor outRecoveryDescriptor() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void inRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
         throw new UnsupportedOperationException();
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public GridNioRecoveryDescriptor recoveryDescriptor() {
+    @Nullable @Override public GridNioRecoveryDescriptor inRecoveryDescriptor() {
         return null;
     }
 
+    /** {@inheritDoc} */
+    @Override public void systemMessage(Object msg) {
+        throw new UnsupportedOperationException();
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridNioSessionImpl.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioWorker.java
new file mode 100644
index 0000000000000..62985ff5d8d75
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioWorker.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.nio;
+
+import java.util.Collection;
+import java.util.List;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+interface GridNioWorker {
+    /**
+     * @param req Change request.
+     */
+    public void offer(GridNioServer.SessionChangeRequest req);
+
+    /**
+     * @param reqs Change requests.
+     */
+    public void offer(Collection<GridNioServer.SessionChangeRequest> reqs);
+
+    /**
+     * @param ses Session.
+     * @return Session state change requests.
+     */
+    @Nullable public List<GridNioServer.SessionChangeRequest> clearSessionRequests(GridNioSession ses);
+
+    /**
+     * @param ses Session to register write interest for.
+     */
+    public void registerWrite(GridSelectorNioSessionImpl ses);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
index 63c9845ca3fc3..66f9176f93745 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
@@ -20,9 +20,11 @@
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.channels.SelectionKey;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 import java.util.concurrent.Semaphore;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -37,17 +39,14 @@
  */
 class GridSelectorNioSessionImpl extends GridNioSessionImpl {
     /** Pending write requests. */
-    private final ConcurrentLinkedDeque8<GridNioFuture<?>> queue = new ConcurrentLinkedDeque8<>();
+    private final ConcurrentLinkedDeque8<SessionWriteRequest> queue = new ConcurrentLinkedDeque8<>();
 
     /** Selection key associated with this session. */
     @GridToStringExclude
     private SelectionKey key;
 
-    /** Worker index for server */
-    private final int selectorIdx;
-
-    /** Size counter. */
-    private final AtomicInteger queueSize = new AtomicInteger();
+    /** Current worker thread. */
+    private volatile GridNioWorker worker;
 
     /** Semaphore. */
     @GridToStringExclude
@@ -59,17 +58,29 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
     /** Read buffer. */
     private ByteBuffer readBuf;
 
-    /** Recovery data. */
-    private GridNioRecoveryDescriptor recovery;
+    /** Incoming recovery data. */
+    private GridNioRecoveryDescriptor inRecovery;
+
+    /** Outgoing recovery data. */
+    private GridNioRecoveryDescriptor outRecovery;
 
     /** Logger. */
     private final IgniteLogger log;
 
+    /** */
+    private List<GridNioServer.SessionChangeRequest> pendingStateChanges;
+
+    /** */
+    final AtomicBoolean procWrite = new AtomicBoolean();
+
+    /** */
+    private Object sysMsg;
+
     /**
      * Creates session instance.
      *
      * @param log Logger.
-     * @param selectorIdx Selector index for this session.
+     * @param worker NIO worker thread.
      * @param filterChain Filter chain that will handle requests.
      * @param locAddr Local address.
      * @param rmtAddr Remote address.
@@ -80,7 +91,7 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
      */
     GridSelectorNioSessionImpl(
         IgniteLogger log,
-        int selectorIdx,
+        GridNioWorker worker,
         GridNioFilterChain filterChain,
         InetSocketAddress locAddr,
         InetSocketAddress rmtAddr,
@@ -91,7 +102,7 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
     ) {
         super(filterChain, locAddr, rmtAddr, accepted);
 
-        assert selectorIdx >= 0;
+        assert worker != null;
         assert sndQueueLimit >= 0;
 
         assert locAddr != null : "GridSelectorNioSessionImpl should have local socket address.";
@@ -101,7 +112,7 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
 
         this.log = log;
 
-        this.selectorIdx = selectorIdx;
+        this.worker = worker;
 
         sem = sndQueueLimit > 0 ? new Semaphore(sndQueueLimit) : null;
 
@@ -118,13 +129,20 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
         }
     }
 
+    /**
+     * @return Worker.
+     */
+    GridNioWorker worker() {
+        return worker;
+    }
+
     /**
      * Sets selection key for this session.
      *
      * @param key Selection key.
      */
     void key(SelectionKey key) {
-        assert this.key == null;
+        assert key != null;
 
         this.key = key;
     }
@@ -151,10 +169,88 @@ SelectionKey key() {
     }
 
     /**
-     * @return Selector index.
+     * @param from Current session worker.
+     * @param fut Move future.
+     * @return {@code True} if session move was scheduled.
+     */
+    boolean offerMove(GridNioWorker from, GridNioServer.SessionChangeRequest fut) {
+        synchronized (this) {
+            if (log.isDebugEnabled())
+                log.debug("Offered move [ses=" + this + ", fut=" + fut + ']');
+
+            GridNioWorker worker0 = worker;
+
+            if (worker0 != from)
+                return false;
+
+            worker.offer(fut);
+        }
+
+        return true;
+    }
+
+    /**
+     * @param fut Future.
+     */
+    void offerStateChange(GridNioServer.SessionChangeRequest fut) {
+        synchronized (this) {
+            if (log.isDebugEnabled())
+                log.debug("Offered move [ses=" + this + ", fut=" + fut + ']');
+
+            GridNioWorker worker0 = worker;
+
+            if (worker0 == null) {
+                if (pendingStateChanges == null)
+                    pendingStateChanges = new ArrayList<>();
+
+                pendingStateChanges.add(fut);
+            }
+            else
+                worker0.offer(fut);
+        }
+    }
+
+    /**
+     * @param moveFrom Current session worker.
      */
-    int selectorIndex() {
-        return selectorIdx;
+    void startMoveSession(GridNioWorker moveFrom) {
+        synchronized (this) {
+            assert this.worker == moveFrom;
+
+            if (log.isDebugEnabled())
+                log.debug("Started moving [ses=" + this + ", from=" + moveFrom + ']');
+
+            List<GridNioServer.SessionChangeRequest> sesReqs = moveFrom.clearSessionRequests(this);
+
+            worker = null;
+
+            if (sesReqs != null) {
+                if (pendingStateChanges == null)
+                    pendingStateChanges = new ArrayList<>();
+
+                pendingStateChanges.addAll(sesReqs);
+            }
+        }
+    }
+
+    /**
+     * @param moveTo New session worker.
+     */
+    void finishMoveSession(GridNioWorker moveTo) {
+        synchronized (this) {
+            assert worker == null;
+
+            if (log.isDebugEnabled())
+                log.debug("Finishing moving [ses=" + this + ", to=" + moveTo + ']');
+
+            worker = moveTo;
+
+            if (pendingStateChanges != null) {
+                moveTo.offer(pendingStateChanges);
+
+                pendingStateChanges = null;
+            }
+        }
     }
 
     /**
@@ -163,14 +259,14 @@ int selectorIndex() {
      * @param writeFut Write request.
      * @return Updated size of the queue.
      */
-    int offerSystemFuture(GridNioFuture<?> writeFut) {
+    int offerSystemFuture(SessionWriteRequest writeFut) {
         writeFut.messageThread(true);
 
         boolean res = queue.offerFirst(writeFut);
 
         assert res : "Future was not added to queue";
 
-        return queueSize.incrementAndGet();
+        return queue.sizex();
     }
 
     /**
@@ -183,7 +279,7 @@ int offerSystemFuture(GridNioFuture<?> writeFut) {
      * @param writeFut Write request to add.
      * @return Updated size of the queue.
      */
-    int offerFuture(GridNioFuture<?> writeFut) {
+    int offerFuture(SessionWriteRequest writeFut) {
         boolean msgThread = GridNioBackPressureControl.threadProcessingMessage();
 
         if (sem != null && !msgThread)
@@ -195,47 +291,41 @@ int offerFuture(GridNioFuture<?> writeFut) {
 
         assert res : "Future was not added to queue";
 
-        return queueSize.incrementAndGet();
+        return queue.sizex();
     }
 
     /**
      * @param futs Futures to resend.
      */
-    void resend(Collection<GridNioFuture<?>> futs) {
+    void resend(Collection<SessionWriteRequest> futs) {
         assert queue.isEmpty() : queue.size();
 
         boolean add = queue.addAll(futs);
 
         assert add;
-
-        boolean set = queueSize.compareAndSet(0, futs.size());
-
-        assert set;
     }
 
     /**
      * @return Message that is in the head of the queue, {@code null} if queue is empty.
      */
-    @Nullable GridNioFuture<?> pollFuture() {
-        GridNioFuture<?> last = queue.poll();
+    @Nullable SessionWriteRequest pollFuture() {
+        SessionWriteRequest last = queue.poll();
 
         if (last != null) {
-            queueSize.decrementAndGet();
-
             if (sem != null && !last.messageThread())
                 sem.release();
 
-            if (recovery != null) {
-                if (!recovery.add(last)) {
+            if (outRecovery != null) {
+                if (!outRecovery.add(last)) {
                     LT.warn(log, "Unacknowledged messages queue size overflow, will attempt to reconnect " +
                         "[remoteAddr=" + remoteAddress() +
-                        ", queueLimit=" + recovery.queueLimit() + ']');
+                        ", queueLimit=" + outRecovery.queueLimit() + ']');
 
                     if (log.isDebugEnabled())
                         log.debug("Unacknowledged messages queue size overflow, will attempt to reconnect " +
                             "[remoteAddr=" + remoteAddress() +
-                            ", queueSize=" + recovery.messagesFutures().size() +
-                            ", queueLimit=" + recovery.queueLimit() + ']');
+                            ", queueSize=" + outRecovery.messagesRequests().size() +
+                            ", queueLimit=" + outRecovery.queueLimit() + ']');
 
                     close();
                 }
@@ -249,7 +339,7 @@ void resend(Collection<GridNioFuture<?>> futs) {
      * @param fut Future.
      * @return {@code True} if future was removed from queue.
      */
-    boolean removeFuture(GridNioFuture<?> fut) {
+    boolean removeFuture(SessionWriteRequest fut) {
         assert closed();
 
         return queue.removeLastOccurrence(fut);
@@ -261,35 +351,49 @@ boolean removeFuture(GridNioFuture<?> fut) {
      * @return Number of write requests.
      */
     int writeQueueSize() {
-        return queueSize.get();
+        return queue.sizex();
     }
 
     /**
      * @return Write requests.
      */
-    Collection<GridNioFuture<?>> writeQueue() {
+    Collection<SessionWriteRequest> writeQueue() {
         return queue;
     }
 
     /** {@inheritDoc} */
-    @Override public void recoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+    @Override public void outRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
         assert recoveryDesc != null;
 
-        recovery = recoveryDesc;
+        outRecovery = recoveryDesc;
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public GridNioRecoveryDescriptor recoveryDescriptor() {
-        return recovery;
+    @Nullable @Override public GridNioRecoveryDescriptor outRecoveryDescriptor() {
+        return outRecovery;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void inRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+        assert recoveryDesc != null;
+
+        inRecovery = recoveryDesc;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public GridNioRecoveryDescriptor inRecoveryDescriptor() {
+        return inRecovery;
     }
 
     /** {@inheritDoc} */
     @Override public <T> T addMeta(int key, @Nullable T val) {
-        if (val instanceof GridNioRecoveryDescriptor) {
-            recovery = (GridNioRecoveryDescriptor)val;
+        if (!accepted() && val instanceof GridNioRecoveryDescriptor) {
+            outRecovery = (GridNioRecoveryDescriptor)val;
+
+            if (!outRecovery.pairedConnections())
+                inRecovery = outRecovery;
 
-            if (!accepted())
-                recovery.connected();
+            outRecovery.onConnected();
 
             return null;
         }
@@ -312,6 +416,31 @@ void onClosed() {
             sem.release(1_000_000);
     }
 
+    /** {@inheritDoc} */
+    @Override public void systemMessage(Object sysMsg) {
+        this.sysMsg = sysMsg;
+    }
+
+    /**
+     * @return {@code True} if have pending system message to send.
+     */
+    boolean hasSystemMessage() {
+        return sysMsg != null;
+    }
+
+    /**
+     * Gets and clears pending system message.
+     *
+     * @return Pending system message.
+     */
+    Object systemMessage() {
+        Object ret = sysMsg;
+
+        sysMsg = null;
+
+        return ret;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridSelectorNioSessionImpl.class, this, super.toString());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridShmemCommunicationClient.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridShmemCommunicationClient.java
index ebe86fbcefd8b..d941bae2cb9db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridShmemCommunicationClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridShmemCommunicationClient.java
@@ -48,6 +48,7 @@ public class GridShmemCommunicationClient extends GridAbstractCommunicationClien
     private final MessageFormatter formatter;
 
     /**
+     * @param connIdx Connection index.
      * @param metricsLsnr Metrics listener.
      * @param port Shared memory IPC server port.
      * @param connTimeout Connection timeout.
@@ -55,14 +56,16 @@ public class GridShmemCommunicationClient extends GridAbstractCommunicationClien
      * @param formatter Message formatter.
      * @throws IgniteCheckedException If failed.
      */
-    public GridShmemCommunicationClient(GridNioMetricsListener metricsLsnr,
+    public GridShmemCommunicationClient(
+        int connIdx,
+        GridNioMetricsListener metricsLsnr,
         int port,
         long connTimeout,
         IgniteLogger log,
         MessageFormatter formatter)
         throws IgniteCheckedException
     {
-        super(metricsLsnr);
+        super(connIdx, metricsLsnr);
 
         assert metricsLsnr != null;
         assert port > 0 && port < 0xffff;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridTcpNioCommunicationClient.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridTcpNioCommunicationClient.java
index 5fe521d7e9c8f..3397772af7982 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridTcpNioCommunicationClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridTcpNioCommunicationClient.java
@@ -45,11 +45,16 @@ public class GridTcpNioCommunicationClient extends GridAbstractCommunicationClie
     private final IgniteLogger log;
 
     /**
+     * @param connIdx Connection index.
      * @param ses Session.
      * @param log Logger.
      */
-    public GridTcpNioCommunicationClient(GridNioSession ses, IgniteLogger log) {
-        super(null);
+    public GridTcpNioCommunicationClient(
+        int connIdx,
+        GridNioSession ses,
+        IgniteLogger log
+    ) {
+        super(connIdx, null);
 
         assert ses != null;
         assert log != null;
@@ -104,40 +109,36 @@ public GridNioSession session() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean sendMessage(@Nullable UUID nodeId, Message msg, IgniteInClosure<IgniteException> closure)
+    @Override public boolean sendMessage(@Nullable UUID nodeId, Message msg, IgniteInClosure<IgniteException> c)
         throws IgniteCheckedException {
-        // Node ID is never provided in asynchronous send mode.
-        assert nodeId == null;
+        try {
+            // Node ID is never provided in asynchronous send mode.
+            assert nodeId == null;
 
-        if (closure != null)
-            ses.addMeta(ACK_CLOSURE.ordinal(), closure);
+            if (c != null)
+                ses.addMeta(ACK_CLOSURE.ordinal(), c);
 
-        GridNioFuture<?> fut = ses.send(msg);
+            ses.sendNoFuture(msg);
 
-        if (fut.isDone()) {
-            try {
-                fut.get();
-            }
-            catch (IgniteCheckedException e) {
-                if (closure != null)
-                    ses.removeMeta(ACK_CLOSURE.ordinal());
+            if (c != null)
+                ses.removeMeta(ACK_CLOSURE.ordinal());
+        }
+        catch (IgniteCheckedException e) {
+            if (c != null)
+                ses.removeMeta(ACK_CLOSURE.ordinal());
 
-                if (log.isDebugEnabled())
-                    log.debug("Failed to send message [client=" + this + ", err=" + e + ']');
+            if (log.isDebugEnabled())
+                log.debug("Failed to send message [client=" + this + ", err=" + e + ']');
 
-                if (e.getCause() instanceof IOException) {
-                    ses.close();
+            if (e.getCause() instanceof IOException) {
+                ses.close();
 
-                    return true;
-                }
-                else
-                    throw new IgniteCheckedException("Failed to send message [client=" + this + ']', e);
+                return true;
             }
+            else
+                throw new IgniteCheckedException("Failed to send message [client=" + this + ']', e);
         }
 
-        if (closure != null)
-            ses.removeMeta(ACK_CLOSURE.ordinal());
-
         return false;
     }
 
@@ -159,4 +160,4 @@ public GridNioSession session() {
     @Override public String toString() {
         return S.toString(GridTcpNioCommunicationClient.class, this, super.toString());
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/SessionWriteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/SessionWriteRequest.java
new file mode 100644
index 0000000000000..508c791cde5a6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/SessionWriteRequest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.nio;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.lang.IgniteInClosure;
+
+/**
+ *
+ */
+public interface SessionWriteRequest {
+    /**
+     * Sets flag indicating that message send future was created in thread that was processing a message.
+     *
+     * @param msgThread {@code True} if future was created in thread that is processing message.
+     */
+    public void messageThread(boolean msgThread);
+
+    /**
+     * @return {@code True} if future was created in thread that was processing message.
+     */
+    public boolean messageThread();
+
+    /**
+     * @return {@code True} if skip recovery for this operation.
+     */
+    public boolean skipRecovery();
+
+    /**
+     * Sets ack closure which will be applied when ack received.
+     *
+     * @param c Ack closure.
+     */
+    public void ackClosure(IgniteInClosure<IgniteException> c);
+
+    /**
+     * The method will be called when ack received.
+     */
+    public void onAckReceived();
+
+    /**
+     * @return Ack closure.
+     */
+    public IgniteInClosure<IgniteException> ackClosure();
+
+    /**
+     * @return Session.
+     */
+    public GridNioSession session();
+
+    /**
+     * @param ses Session.
+     */
+    public void resetSession(GridNioSession ses);
+
+    /**
+     *
+     */
+    public void onError(Exception e);
+
+    /**
+     * @return Message.
+     */
+    public Object message();
+
+    /**
+     *
+     */
+    public void onMessageWritten();
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
index d6f9d106264f1..8ed7db0a12a82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
@@ -282,9 +282,13 @@ public ByteBuffer encrypt(GridNioSession ses, ByteBuffer input) throws SSLExcept
     }
 
     /** {@inheritDoc} */
-    @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+    @Override public GridNioFuture<?> onSessionWrite(
+        GridNioSession ses,
+        Object msg,
+        boolean fut
+    ) throws IgniteCheckedException {
         if (directMode)
-            return proceedSessionWrite(ses, msg);
+            return proceedSessionWrite(ses, msg, fut);
 
         ByteBuffer input = checkMessage(ses, msg);
 
@@ -441,4 +445,4 @@ private ByteBuffer checkMessage(GridNioSession ses, Object msg) throws GridNioEx
 
         return (ByteBuffer)msg;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslHandler.java
index eb8dad48a3134..269e8b9153df8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslHandler.java
@@ -437,7 +437,7 @@ void flushDeferredWrites() throws IgniteCheckedException {
         while (!deferredWriteQueue.isEmpty()) {
             WriteRequest req = deferredWriteQueue.poll();
 
-            req.future().onDone((GridNioFuture<Object>)parent.proceedSessionWrite(ses, req.buffer()));
+            req.future().onDone((GridNioFuture<Object>)parent.proceedSessionWrite(ses, req.buffer(), true));
         }
     }
 
@@ -482,7 +482,7 @@ GridNioFuture<?> writeNetBuffer() throws IgniteCheckedException {
 
         ByteBuffer cp = copy(outNetBuf);
 
-        return parent.proceedSessionWrite(ses, cp);
+        return parent.proceedSessionWrite(ses, cp, true);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
index b29d7cddc7366..86aa7a1af11e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
@@ -598,4 +598,4 @@ else if (!f.isAnnotationPresent(GridToStringExclude.class) &&
 
         return cd;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 1fe437cc710c0..0c90414de365a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -46,6 +46,7 @@
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLException;
 import org.apache.ignite.Ignite;
@@ -53,6 +54,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.AddressResolver;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -103,6 +105,7 @@
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.extensions.communication.Message;
@@ -179,6 +182,7 @@
  * <li>Node local IP address (see {@link #setLocalAddress(String)})</li>
  * <li>Node local port number (see {@link #setLocalPort(int)})</li>
  * <li>Local port range (see {@link #setLocalPortRange(int)}</li>
+ * <li>Connections per node (see {@link #setConnectionsPerNode(int)})</li>
  * <li>Connection buffer flush frequency (see {@link #setConnectionBufferFlushFrequency(long)})</li>
  * <li>Connection buffer size (see {@link #setConnectionBufferSize(int)})</li>
  * <li>Idle connection timeout (see {@link #setIdleConnectionTimeout(long)})</li>
@@ -238,6 +242,9 @@
 @IgniteSpiConsistencyChecked(optional = false)
 public class TcpCommunicationSpi extends IgniteSpiAdapter
     implements CommunicationSpi<Message>, TcpCommunicationSpiMBean {
+    /** */
+    private static final IgniteProductVersion MULTIPLE_CONN_SINCE_VER = IgniteProductVersion.fromString("1.8.2");
+
     /** IPC error message. */
     public static final String OUT_OF_RESOURCES_TCP_MSG = "Failed to allocate shared memory segment " +
         "(switching to TCP, may be slower).";
@@ -257,11 +264,14 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     /** Node attribute that is mapped to node's external addresses (value is <tt>comm.tcp.ext-addrs</tt>). */
     public static final String ATTR_EXT_ADDRS = "comm.tcp.ext-addrs";
 
+    /** */
+    public static final String ATTR_PAIRED_CONN = "comm.tcp.pairedConnection";
+
     /** Default port which node sets listener to (value is <tt>47100</tt>). */
     public static final int DFLT_PORT = 47100;
 
     /** Default port which node sets listener for shared memory connections (value is <tt>48100</tt>). */
-    public static final int DFLT_SHMEM_PORT = 48100;
+    public static final int DFLT_SHMEM_PORT = -1;
 
     /** Default idle connection timeout (value is <tt>30000</tt>ms). */
     public static final long DFLT_IDLE_CONN_TIMEOUT = 30000;
@@ -283,12 +293,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
 
     /**
      * Default count of selectors for TCP server equals to
-     * {@code "Math.min(4, Runtime.getRuntime().availableProcessors())"}.
+     * {@code "Math.min(8, Runtime.getRuntime().availableProcessors())"}.
      */
-    public static final int DFLT_SELECTORS_CNT = Math.min(4, Runtime.getRuntime().availableProcessors());
+    public static final int DFLT_SELECTORS_CNT = Math.max(4, Runtime.getRuntime().availableProcessors() / 2);
 
-    /** Node ID meta for session. */
-    private static final int NODE_ID_META = GridNioSessionMetaKey.nextUniqueKey();
+    /** Connection index meta for session. */
+    private static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
 
     /** Message tracker meta for session. */
     private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
@@ -303,11 +313,14 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     public static final boolean DFLT_TCP_NODELAY = true;
 
     /** Default received messages threshold for sending ack. */
-    public static final int DFLT_ACK_SND_THRESHOLD = 16;
+    public static final int DFLT_ACK_SND_THRESHOLD = 32;
 
     /** Default socket write timeout. */
     public static final long DFLT_SOCK_WRITE_TIMEOUT = 2000;
 
+    /** Default connections per node. */
+    public static final int DFLT_CONN_PER_NODE = 1;
+
     /** No-op runnable. */
     private static final IgniteRunnable NOOP = new IgniteRunnable() {
         @Override public void run() {
@@ -327,11 +340,14 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     /** */
     private ConnectGateway connectGate;
 
+    /** */
+    private ConnectionPolicy connPlc;
+
     /** Server listener. */
     private final GridNioServerListener<Message> srvLsnr =
         new GridNioServerListenerAdapter<Message>() {
             @Override public void onSessionWriteTimeout(GridNioSession ses) {
-                LT.warn(log, "Communication SPI Session write timed out (consider increasing " +
+                LT.warn(log,"Communication SPI session write timed out (consider increasing " +
                     "'socketWriteTimeout' " + "configuration property) [remoteAddr=" + ses.remoteAddress() +
                     ", writeTimeout=" + sockWriteTimeout + ']');
 
@@ -347,46 +363,53 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
                     if (log.isDebugEnabled())
                         log.debug("Sending local node ID to newly accepted session: " + ses);
 
-                    ses.send(nodeIdMessage());
+                    try {
+                        ses.sendNoFuture(nodeIdMessage());
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to send message: " + e, e);
+                    }
                 }
             }
 
             @Override public void onDisconnected(GridNioSession ses, @Nullable Exception e) {
-                UUID id = ses.meta(NODE_ID_META);
+                ConnectionKey connId = ses.meta(CONN_IDX_META);
 
-                if (id != null) {
-                    GridCommunicationClient client = clients.get(id);
+                if (connId != null) {
+                    UUID id = connId.nodeId();
 
-                    if (client instanceof GridTcpNioCommunicationClient &&
-                        ((GridTcpNioCommunicationClient) client).session() == ses) {
-                        client.close();
+                    GridCommunicationClient[] nodeClients = clients.get(id);
 
-                        clients.remove(id, client);
+                    if (nodeClients != null) {
+                        for (GridCommunicationClient client : nodeClients) {
+                            if (client instanceof GridTcpNioCommunicationClient &&
+                                ((GridTcpNioCommunicationClient)client).session() == ses) {
+                                client.close();
+
+                                removeNodeClient(id, client);
+                            }
+                        }
                     }
 
                     if (!stopping) {
-                        boolean reconnect = false;
-
-                        GridNioRecoveryDescriptor recoveryData = ses.recoveryDescriptor();
-
-                        if (recoveryData != null) {
-                            if (recoveryData.nodeAlive(getSpiContext().node(id))) {
-                                if (!recoveryData.messagesFutures().isEmpty()) {
-                                    reconnect = true;
+                        GridNioRecoveryDescriptor outDesc = ses.outRecoveryDescriptor();
 
+                        if (outDesc != null) {
+                            if (outDesc.nodeAlive(getSpiContext().node(id))) {
+                                if (!outDesc.messagesRequests().isEmpty()) {
                                     if (log.isDebugEnabled())
                                         log.debug("Session was closed but there are unacknowledged messages, " +
-                                            "will try to reconnect [rmtNode=" + recoveryData.node().id() + ']');
+                                            "will try to reconnect [rmtNode=" + outDesc.node().id() + ']');
+
+                                    DisconnectedSessionInfo disconnectData =
+                                        new DisconnectedSessionInfo(outDesc, connId.connectionIndex());
+
+                                    commWorker.addProcessDisconnectRequest(disconnectData);
                                 }
                             }
                             else
-                                recoveryData.onNodeLeft();
+                                outDesc.onNodeLeft();
                         }
-
-                        DisconnectedSessionInfo disconnectData = new DisconnectedSessionInfo(recoveryData,
-                            reconnect);
-
-                        commWorker.addProcessDisconnectRequest(disconnectData);
                     }
 
                     CommunicationListener<Message> lsnr0 = lsnr;
@@ -403,21 +426,24 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
             private void onFirstMessage(GridNioSession ses, Message msg) {
                 UUID sndId;
 
-                if (msg instanceof NodeIdMessage)
-                    sndId = U.bytesToUuid(((NodeIdMessage)msg).nodeIdBytes, 0);
+                ConnectionKey connKey;
+
+                if (msg instanceof NodeIdMessage) {
+                    sndId = U.bytesToUuid(((NodeIdMessage) msg).nodeIdBytes, 0);
+                    connKey = new ConnectionKey(sndId, 0, -1);
+                }
                 else {
                     assert msg instanceof HandshakeMessage : msg;
 
+                    HandshakeMessage msg0 = (HandshakeMessage)msg;
+
                     sndId = ((HandshakeMessage)msg).nodeId();
+                    connKey = new ConnectionKey(sndId, msg0.connectionIndex(), msg0.connectCount());
                 }
 
                 if (log.isDebugEnabled())
                     log.debug("Remote node ID received: " + sndId);
 
-                final UUID old = ses.addMeta(NODE_ID_META, sndId);
-
-                assert old == null;
-
                 final ClusterNode rmtNode = getSpiContext().node(sndId);
 
                 if (rmtNode == null) {
@@ -429,57 +455,65 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                     return;
                 }
 
+                final ConnectionKey old = ses.addMeta(CONN_IDX_META, connKey);
+
+                assert old == null;
+
                 ClusterNode locNode = getSpiContext().localNode();
 
                 if (ses.remoteAddress() == null)
                     return;
 
-                GridCommunicationClient oldClient = clients.get(sndId);
+                assert msg instanceof HandshakeMessage : msg;
 
-                boolean hasShmemClient = false;
+                HandshakeMessage msg0 = (HandshakeMessage)msg;
 
-                if (oldClient != null) {
-                    if (oldClient instanceof GridTcpNioCommunicationClient) {
-                        if (log.isDebugEnabled())
-                            log.debug("Received incoming connection when already connected " +
-                                    "to this node, rejecting [locNode=" + locNode.id() +
-                                    ", rmtNode=" + sndId + ']');
+                if (usePairedConnections(rmtNode)) {
+                    final GridNioRecoveryDescriptor recoveryDesc = inRecoveryDescriptor(rmtNode, connKey);
 
-                        ses.send(new RecoveryLastReceivedMessage(-1));
+                    ConnectClosureNew c = new ConnectClosureNew(ses, recoveryDesc, rmtNode);
 
-                        return;
-                    }
+                    boolean reserve = recoveryDesc.tryReserve(msg0.connectCount(), c);
+
+                    if (reserve)
+                        connectedNew(recoveryDesc, ses, true);
                     else {
-                        assert oldClient instanceof GridShmemCommunicationClient;
+                        if (c.failed) {
+                            ses.send(new RecoveryLastReceivedMessage(-1));
+
+                            for (GridNioSession ses0 : nioSrvr.sessions()) {
+                                ConnectionKey key0 = ses0.meta(CONN_IDX_META);
 
-                        hasShmemClient = true;
+                                if (ses0.accepted() && key0 != null &&
+                                    key0.nodeId().equals(connKey.nodeId()) &&
+                                    key0.connectionIndex() == connKey.connectionIndex() &&
+                                    key0.connectCount() < connKey.connectCount())
+                                    ses0.close();
+                            }
+                        }
                     }
                 }
+                else {
+                    assert connKey.connectionIndex() >= 0 : connKey;
 
-                GridFutureAdapter<GridCommunicationClient> fut = new GridFutureAdapter<>();
-
-                GridFutureAdapter<GridCommunicationClient> oldFut = clientFuts.putIfAbsent(sndId, fut);
-
-                assert msg instanceof HandshakeMessage : msg;
-
-                HandshakeMessage msg0 = (HandshakeMessage)msg;
+                    GridCommunicationClient[] curClients = clients.get(sndId);
 
-                final GridNioRecoveryDescriptor recoveryDesc = recoveryDescriptor(rmtNode);
+                    GridCommunicationClient oldClient =
+                        curClients != null && connKey.connectionIndex() < curClients.length ?
+                            curClients[connKey.connectionIndex()] :
+                            null;
 
-                if (oldFut == null) {
-                    oldClient = clients.get(sndId);
+                    boolean hasShmemClient = false;
 
                     if (oldClient != null) {
                         if (oldClient instanceof GridTcpNioCommunicationClient) {
                             if (log.isDebugEnabled())
                                 log.debug("Received incoming connection when already connected " +
-                                        "to this node, rejecting [locNode=" + locNode.id() +
-                                        ", rmtNode=" + sndId + ']');
+                                    "to this node, rejecting [locNode=" + locNode.id() +
+                                    ", rmtNode=" + sndId + ']');
 
                             ses.send(new RecoveryLastReceivedMessage(-1));
 
-                            fut.onDone(oldClient);
-
                             return;
                         }
                         else {
@@ -489,51 +523,86 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                         }
                     }
 
-                    boolean reserved = recoveryDesc.tryReserve(msg0.connectCount(),
-                            new ConnectClosure(ses, recoveryDesc, rmtNode, msg0, !hasShmemClient, fut));
+                    GridFutureAdapter<GridCommunicationClient> fut = new GridFutureAdapter<>();
 
-                    if (log.isDebugEnabled())
-                        log.debug("Received incoming connection from remote node " +
+                    GridFutureAdapter<GridCommunicationClient> oldFut = clientFuts.putIfAbsent(connKey, fut);
+
+                    final GridNioRecoveryDescriptor recoveryDesc = inRecoveryDescriptor(rmtNode, connKey);
+
+                    if (oldFut == null) {
+                        curClients = clients.get(sndId);
+
+                        oldClient = curClients != null && connKey.connectionIndex() < curClients.length ?
+                            curClients[connKey.connectionIndex()] : null;
+
+                        if (oldClient != null) {
+                            if (oldClient instanceof GridTcpNioCommunicationClient) {
+                                assert oldClient.connectionIndex() == connKey.connectionIndex() : oldClient;
+
+                                if (log.isDebugEnabled())
+                                    log.debug("Received incoming connection when already connected " +
+                                        "to this node, rejecting [locNode=" + locNode.id() +
+                                        ", rmtNode=" + sndId + ']');
+
+                                ses.send(new RecoveryLastReceivedMessage(-1));
+
+                                fut.onDone(oldClient);
+
+                                return;
+                            }
+                            else {
+                                assert oldClient instanceof GridShmemCommunicationClient;
+
+                                hasShmemClient = true;
+                            }
+                        }
+
+                        boolean reserved = recoveryDesc.tryReserve(msg0.connectCount(),
+                            new ConnectClosure(ses, recoveryDesc, rmtNode, connKey, msg0, !hasShmemClient, fut));
+
+                        if (log.isDebugEnabled())
+                            log.debug("Received incoming connection from remote node " +
                                 "[rmtNode=" + rmtNode.id() + ", reserved=" + reserved + ']');
 
-                    if (reserved) {
-                        try {
-                            GridTcpNioCommunicationClient client =
+                        if (reserved) {
+                            try {
+                                GridTcpNioCommunicationClient client =
                                     connected(recoveryDesc, ses, rmtNode, msg0.received(), true, !hasShmemClient);
 
-                            fut.onDone(client);
-                        }
-                        finally {
-                            clientFuts.remove(rmtNode.id(), fut);
+                                fut.onDone(client);
+                            }
+                            finally {
+                                clientFuts.remove(connKey, fut);
+                            }
                         }
                     }
-                }
-                else {
-                    if (oldFut instanceof ConnectFuture && locNode.order() < rmtNode.order()) {
-                        if (log.isDebugEnabled()) {
-                            log.debug("Received incoming connection from remote node while " +
+                    else {
+                        if (oldFut instanceof ConnectFuture && locNode.order() < rmtNode.order()) {
+                            if (log.isDebugEnabled()) {
+                                log.debug("Received incoming connection from remote node while " +
                                     "connecting to this node, rejecting [locNode=" + locNode.id() +
                                     ", locNodeOrder=" + locNode.order() + ", rmtNode=" + rmtNode.id() +
                                     ", rmtNodeOrder=" + rmtNode.order() + ']');
-                        }
+                            }
 
-                        ses.send(new RecoveryLastReceivedMessage(-1));
-                    }
-                    else {
-                        // The code below causes a race condition between shmem and TCP (see IGNITE-1294)
-                        boolean reserved = recoveryDesc.tryReserve(msg0.connectCount(),
-                                new ConnectClosure(ses, recoveryDesc, rmtNode, msg0, !hasShmemClient, fut));
+                            ses.send(new RecoveryLastReceivedMessage(-1));
+                        }
+                        else {
+                            // The code below causes a race condition between shmem and TCP (see IGNITE-1294)
+                            boolean reserved = recoveryDesc.tryReserve(msg0.connectCount(),
+                                new ConnectClosure(ses, recoveryDesc, rmtNode, connKey, msg0, !hasShmemClient, fut));
 
-                        if (reserved)
-                            connected(recoveryDesc, ses, rmtNode, msg0.received(), true, !hasShmemClient);
+                            if (reserved)
+                                connected(recoveryDesc, ses, rmtNode, msg0.received(), true, !hasShmemClient);
+                        }
                     }
                 }
             }
 
             @Override public void onMessage(GridNioSession ses, Message msg) {
-                UUID sndId = ses.meta(NODE_ID_META);
+                ConnectionKey connKey = ses.meta(CONN_IDX_META);
 
-                if (sndId == null) {
+                if (connKey == null) {
                     assert ses.accepted() : ses;
 
                     if (!connectGate.tryEnter()) {
@@ -555,29 +624,37 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                 else {
                     rcvdMsgsCnt.increment();
 
-                    GridNioRecoveryDescriptor recovery = ses.recoveryDescriptor();
+                    if (msg instanceof RecoveryLastReceivedMessage) {
+                        GridNioRecoveryDescriptor recovery = ses.outRecoveryDescriptor();
 
-                    if (recovery != null) {
-                        if (msg instanceof RecoveryLastReceivedMessage) {
+                        if (recovery != null) {
                             RecoveryLastReceivedMessage msg0 = (RecoveryLastReceivedMessage)msg;
 
-                            if (log.isDebugEnabled())
-                                log.debug("Received recovery acknowledgement [rmtNode=" + sndId +
+                            if (log.isDebugEnabled()) {
+                                log.debug("Received recovery acknowledgement [rmtNode=" + connKey.nodeId() +
+                                    ", connIdx=" + connKey.connectionIndex() +
                                     ", rcvCnt=" + msg0.received() + ']');
+                            }
 
                             recovery.ackReceived(msg0.received());
 
                             return;
                         }
-                        else {
+                    }
+                    else {
+                        GridNioRecoveryDescriptor recovery = ses.inRecoveryDescriptor();
+
+                        if (recovery != null) {
                             long rcvCnt = recovery.onReceived();
 
                             if (rcvCnt % ackSndThreshold == 0) {
-                                if (log.isDebugEnabled())
-                                    log.debug("Send recovery acknowledgement [rmtNode=" + sndId +
+                                if (log.isDebugEnabled()) {
+                                    log.debug("Send recovery acknowledgement [rmtNode=" + connKey.nodeId() +
+                                        ", connIdx=" + connKey.connectionIndex() +
                                         ", rcvCnt=" + rcvCnt + ']');
+                                }
 
-                                nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(rcvCnt));
+                                ses.systemMessage(new RecoveryLastReceivedMessage(rcvCnt));
 
                                 recovery.lastAcknowledged(rcvCnt);
                             }
@@ -603,7 +680,7 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                     else
                         c = NOOP;
 
-                    notifyListener(sndId, msg, c);
+                    notifyListener(connKey.nodeId(), msg, c);
                 }
             }
 
@@ -611,7 +688,7 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
              * @param recovery Recovery descriptor.
              * @param ses Session.
              * @param node Node.
-             * @param rcvCnt Number of received messages..
+             * @param rcvCnt Number of received messages.
              * @param sndRes If {@code true} sends response for recovery handshake.
              * @param createClient If {@code true} creates NIO communication client.
              * @return Client.
@@ -623,31 +700,127 @@ private GridTcpNioCommunicationClient connected(
                 long rcvCnt,
                 boolean sndRes,
                 boolean createClient) {
+                ConnectionKey connKey = ses.meta(CONN_IDX_META);
+
+                assert connKey != null && connKey.connectionIndex() >= 0 : connKey;
+                assert !usePairedConnections(node);
+
                 recovery.onHandshake(rcvCnt);
 
-                ses.recoveryDescriptor(recovery);
+                ses.inRecoveryDescriptor(recovery);
+                ses.outRecoveryDescriptor(recovery);
 
                 nioSrvr.resend(ses);
 
-                if (sndRes)
-                    nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recovery.received()));
+                try {
+                    if (sndRes)
+                        nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recovery.received()));
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to send message: " + e, e);
+                }
 
-                recovery.connected();
+                recovery.onConnected();
 
                 GridTcpNioCommunicationClient client = null;
 
                 if (createClient) {
-                    client = new GridTcpNioCommunicationClient(ses, log);
-
-                    GridCommunicationClient oldClient = clients.putIfAbsent(node.id(), client);
+                    client = new GridTcpNioCommunicationClient(connKey.connectionIndex(), ses, log);
 
-                    assert oldClient == null : "Client already created [node=" + node + ", client=" + client +
-                        ", oldClient=" + oldClient + ", recoveryDesc=" + recovery + ']';
+                    addNodeClient(node, connKey.connectionIndex(), client);
                 }
 
                 return client;
             }
 
+            /**
+             * @param recovery Recovery descriptor.
+             * @param ses Session.
+             * @param sndRes If {@code true} sends response for recovery handshake.
+             */
+            private void connectedNew(
+                GridNioRecoveryDescriptor recovery,
+                GridNioSession ses,
+                boolean sndRes) {
+                try {
+                    ses.inRecoveryDescriptor(recovery);
+
+                    if (sndRes)
+                        nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recovery.received()));
+
+                    recovery.onConnected();
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to send message: " + e, e);
+                }
+            }
+
+            /**
+             *
+             */
+            class ConnectClosureNew implements IgniteInClosure<Boolean> {
+                /** */
+                private static final long serialVersionUID = 0L;
+
+                /** */
+                private final GridNioSession ses;
+
+                /** */
+                private final GridNioRecoveryDescriptor recoveryDesc;
+
+                /** */
+                private final ClusterNode rmtNode;
+
+                /** */
+                private boolean failed;
+
+                /**
+                 * @param ses Incoming session.
+                 * @param recoveryDesc Recovery descriptor.
+                 * @param rmtNode Remote node.
+                 */
+                ConnectClosureNew(GridNioSession ses,
+                    GridNioRecoveryDescriptor recoveryDesc,
+                    ClusterNode rmtNode) {
+                    this.ses = ses;
+                    this.recoveryDesc = recoveryDesc;
+                    this.rmtNode = rmtNode;
+                }
+
+                /** {@inheritDoc} */
+                @Override public void apply(Boolean success) {
+                    try {
+                        failed = !success;
+
+                        if (success) {
+                            IgniteInClosure<IgniteInternalFuture<?>> lsnr = new IgniteInClosure<IgniteInternalFuture<?>>() {
+                                @Override public void apply(IgniteInternalFuture<?> msgFut) {
+                                    try {
+                                        msgFut.get();
+
+                                        connectedNew(recoveryDesc, ses, false);
+                                    }
+                                    catch (IgniteCheckedException e) {
+                                        if (log.isDebugEnabled())
+                                            log.debug("Failed to send recovery handshake " +
+                                                    "[rmtNode=" + rmtNode.id() + ", err=" + e + ']');
+
+                                        recoveryDesc.release();
+                                    }
+                                }
+                            };
+
+                            nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recoveryDesc.received()), lsnr);
+                        }
+                        else
+                            nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(-1));
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to send message: " + e, e);
+                    }
+                }
+            }
+
             /**
              *
              */
@@ -674,10 +847,14 @@ class ConnectClosure implements IgniteInClosure<Boolean> {
                 /** */
                 private final boolean createClient;
 
+                /** */
+                private final ConnectionKey connKey;
+
                 /**
                  * @param ses Incoming session.
                  * @param recoveryDesc Recovery descriptor.
                  * @param rmtNode Remote node.
+                 * @param connKey Connection key.
                  * @param msg Handshake message.
                  * @param createClient If {@code true} creates NIO communication client..
                  * @param fut Connect future.
@@ -685,12 +862,14 @@ class ConnectClosure implements IgniteInClosure<Boolean> {
                 ConnectClosure(GridNioSession ses,
                     GridNioRecoveryDescriptor recoveryDesc,
                     ClusterNode rmtNode,
+                    ConnectionKey connKey,
                     HandshakeMessage msg,
                     boolean createClient,
                     GridFutureAdapter<GridCommunicationClient> fut) {
                     this.ses = ses;
                     this.recoveryDesc = recoveryDesc;
                     this.rmtNode = rmtNode;
+                    this.connKey = connKey;
                     this.msg = msg;
                     this.createClient = createClient;
                     this.fut = fut;
@@ -699,39 +878,44 @@ class ConnectClosure implements IgniteInClosure<Boolean> {
                 /** {@inheritDoc} */
                 @Override public void apply(Boolean success) {
                     if (success) {
-                        IgniteInClosure<IgniteInternalFuture<?>> lsnr = new IgniteInClosure<IgniteInternalFuture<?>>() {
-                            @Override public void apply(IgniteInternalFuture<?> msgFut) {
-                                try {
-                                    msgFut.get();
+                        try {
+                            IgniteInClosure<IgniteInternalFuture<?>> lsnr = new IgniteInClosure<IgniteInternalFuture<?>>() {
+                                @Override public void apply(IgniteInternalFuture<?> msgFut) {
+                                    try {
+                                        msgFut.get();
 
-                                    GridTcpNioCommunicationClient client =
-                                        connected(recoveryDesc, ses, rmtNode, msg.received(), false, createClient);
+                                        GridTcpNioCommunicationClient client =
+                                                connected(recoveryDesc, ses, rmtNode, msg.received(), false, createClient);
 
-                                    fut.onDone(client);
-                                }
-                                catch (IgniteCheckedException e) {
-                                    if (log.isDebugEnabled())
-                                        log.debug("Failed to send recovery handshake " +
-                                            "[rmtNode=" + rmtNode.id() + ", err=" + e + ']');
+                                        fut.onDone(client);
+                                    }
+                                    catch (IgniteCheckedException e) {
+                                        if (log.isDebugEnabled())
+                                            log.debug("Failed to send recovery handshake " +
+                                                    "[rmtNode=" + rmtNode.id() + ", err=" + e + ']');
 
-                                    recoveryDesc.release();
+                                        recoveryDesc.release();
 
-                                    fut.onDone();
-                                }
-                                finally {
-                                    clientFuts.remove(rmtNode.id(), fut);
+                                        fut.onDone();
+                                    }
+                                    finally {
+                                        clientFuts.remove(connKey, fut);
+                                    }
                                 }
-                            }
-                        };
+                            };
 
-                        nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recoveryDesc.received()), lsnr);
+                            nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recoveryDesc.received()), lsnr);
+                        }
+                        catch (IgniteCheckedException e) {
+                            U.error(log, "Failed to send message: " + e, e);
+                        }
                     }
                     else {
                         try {
                             fut.onDone();
                         }
                         finally {
-                            clientFuts.remove(rmtNode.id(), fut);
+                            clientFuts.remove(connKey, fut);
                         }
                     }
                 }
@@ -794,6 +978,12 @@ class ConnectClosure implements IgniteInClosure<Boolean> {
     /** Shared memory server. */
     private IpcSharedMemoryServerEndpoint shmemSrv;
 
+    /** */
+    private boolean usePairedConnections = true;
+
+    /** */
+    private int connectionsPerNode = DFLT_CONN_PER_NODE;
+
     /** {@code TCP_NODELAY} option value for created sockets. */
     private boolean tcpNoDelay = DFLT_TCP_NODELAY;
 
@@ -816,7 +1006,7 @@ class ConnectClosure implements IgniteInClosure<Boolean> {
     private final Collection<ShmemWorker> shmemWorkers = new ConcurrentLinkedDeque8<>();
 
     /** Clients. */
-    private final ConcurrentMap<UUID, GridCommunicationClient> clients = GridConcurrentFactory.newMap();
+    private final ConcurrentMap<UUID, GridCommunicationClient[]> clients = GridConcurrentFactory.newMap();
 
     /** SPI listener. */
     private volatile CommunicationListener<Message> lsnr;
@@ -830,6 +1020,13 @@ class ConnectClosure implements IgniteInClosure<Boolean> {
     /** Count of selectors to use in TCP server. */
     private int selectorsCnt = DFLT_SELECTORS_CNT;
 
+    /**
+     * Defines how many non-blocking {@code selector.selectNow()} should be made before
+     * falling into {@code selector.select(long)} in NIO server. Long value. Default is {@code 0}.
+     * Can be set to {@code Long.MAX_VALUE} so selector threads will never block.
+     */
+    private long selectorSpins = IgniteSystemProperties.getLong("IGNITE_SELECTOR_SPINS", 0L);
+
     /** Address resolver. */
     private AddressResolver addrRslvr;
 
@@ -863,11 +1060,17 @@ class ConnectClosure implements IgniteInClosure<Boolean> {
     };
 
     /** Client connect futures. */
-    private final ConcurrentMap<UUID, GridFutureAdapter<GridCommunicationClient>> clientFuts =
+    private final ConcurrentMap<ConnectionKey, GridFutureAdapter<GridCommunicationClient>> clientFuts =
         GridConcurrentFactory.newMap();
 
     /** */
-    private final ConcurrentMap<ClientKey, GridNioRecoveryDescriptor> recoveryDescs = GridConcurrentFactory.newMap();
+    private final ConcurrentMap<ConnectionKey, GridNioRecoveryDescriptor> recoveryDescs = GridConcurrentFactory.newMap();
+
+    /** */
+    private final ConcurrentMap<ConnectionKey, GridNioRecoveryDescriptor> outRecDescs = GridConcurrentFactory.newMap();
+
+    /** */
+    private final ConcurrentMap<ConnectionKey, GridNioRecoveryDescriptor> inRecDescs = GridConcurrentFactory.newMap();
 
     /** Discovery listener. */
     private final GridLocalEventListener discoLsnr = new GridLocalEventListener() {
@@ -976,6 +1179,49 @@ public void setLocalPortRange(int locPortRange) {
         return locPortRange;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean isUsePairedConnections() {
+        return usePairedConnections;
+    }
+
+    /**
+     * Set this to {@code true} if {@code TcpCommunicationSpi} should
+     * maintain connection for outgoing and incoming messages separately.
+     * In this case total number of connections between local and each remote node
+     * is {@link #getConnectionsPerNode()} * 2.
+     * <p>
+     * Set this to {@code false} if each connection of {@link #getConnectionsPerNode()}
+     * should be used for outgoing and incoming messages. In this case total number
+     * of connections between local and each remote node is {@link #getConnectionsPerNode()}.
+     * In this case load NIO selectors load
+     * balancing of {@link GridNioServer} will be disabled.
+     * <p>
+     * Default is {@code true}.
+     *
+     * @param usePairedConnections {@code true} to use paired connections and {@code false} otherwise.
+     * @see #getConnectionsPerNode()
+     */
+    public void setUsePairedConnections(boolean usePairedConnections) {
+        this.usePairedConnections = usePairedConnections;
+    }
+
+    /**
+     * Sets number of connections to each remote node. if {@link #isUsePairedConnections()}
+     * is {@code true} then number of connections is doubled and half is used for incoming and
+     * half for outgoing messages.
+     *
+     * @param maxConnectionsPerNode Number of connections per node.
+     * @see #isUsePairedConnections()
+     */
+    public void setConnectionsPerNode(int maxConnectionsPerNode) {
+        this.connectionsPerNode = maxConnectionsPerNode;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getConnectionsPerNode() {
+        return connectionsPerNode;
+    }
+
     /**
      * Sets local port to accept shared memory connections.
      * <p>
@@ -1222,6 +1468,22 @@ public void setSelectorsCount(int selectorsCnt) {
         return selectorsCnt;
     }
 
+    /** {@inheritDoc} */
+    @Override public long getSelectorSpins() {
+        return selectorSpins;
+    }
+
+    /**
+     * Defines how many non-blocking {@code selector.selectNow()} should be made before
+     * falling into {@code selector.select(long)} in NIO server. Long value. Default is {@code 0}.
+     * Can be set to {@code Long.MAX_VALUE} so selector threads will never block.
+     *
+     * @param selectorSpins Selector thread busy-loop iterations.
+     */
+    public void setSelectorSpins(long selectorSpins) {
+        this.selectorSpins = selectorSpins;
+    }
+
     /**
      * Sets value for {@code TCP_NODELAY} socket option. Each
      * socket will be opened using provided value.
@@ -1396,7 +1658,7 @@ public CommunicationListener getListener() {
         if (log != null) {
             StringBuilder sb = new StringBuilder("Communication SPI recovery descriptors: ").append(U.nl());
 
-            for (Map.Entry<ClientKey, GridNioRecoveryDescriptor> entry : recoveryDescs.entrySet()) {
+            for (Map.Entry<ConnectionKey, GridNioRecoveryDescriptor> entry : recoveryDescs.entrySet()) {
                 GridNioRecoveryDescriptor desc = entry.getValue();
 
                 sb.append("    [key=").append(entry.getKey())
@@ -1409,14 +1671,48 @@ public CommunicationListener getListener() {
                     .append(']').append(U.nl());
             }
 
-            sb.append("Communication SPI clients: ").append(U.nl());
+            for (Map.Entry<ConnectionKey, GridNioRecoveryDescriptor> entry : outRecDescs.entrySet()) {
+                GridNioRecoveryDescriptor desc = entry.getValue();
+
+                sb.append("    [key=").append(entry.getKey())
+                    .append(", msgsSent=").append(desc.sent())
+                    .append(", msgsAckedByRmt=").append(desc.acked())
+                    .append(", reserveCnt=").append(desc.reserveCount())
+                    .append(", connected=").append(desc.connected())
+                    .append(", reserved=").append(desc.reserved())
+                    .append(", descIdHash=").append(System.identityHashCode(desc))
+                    .append(']').append(U.nl());
+            }
+
+            for (Map.Entry<ConnectionKey, GridNioRecoveryDescriptor> entry : inRecDescs.entrySet()) {
+                GridNioRecoveryDescriptor desc = entry.getValue();
 
-            for (Map.Entry<UUID, GridCommunicationClient> entry : clients.entrySet()) {
-                sb.append("    [node=").append(entry.getKey())
-                    .append(", client=").append(entry.getValue())
+                sb.append("    [key=").append(entry.getKey())
+                    .append(", msgsRcvd=").append(desc.received())
+                    .append(", lastAcked=").append(desc.lastAcknowledged())
+                    .append(", reserveCnt=").append(desc.reserveCount())
+                    .append(", connected=").append(desc.connected())
+                    .append(", reserved=").append(desc.reserved())
+                    .append(", handshakeIdx=").append(desc.handshakeIndex())
+                    .append(", descIdHash=").append(System.identityHashCode(desc))
                     .append(']').append(U.nl());
             }
 
+            sb.append("Communication SPI clients: ").append(U.nl());
+
+            for (Map.Entry<UUID, GridCommunicationClient[]> entry : clients.entrySet()) {
+                UUID nodeId = entry.getKey();
+                GridCommunicationClient[] clients0 = entry.getValue();
+
+                for (GridCommunicationClient client : clients0) {
+                    if (client != null) {
+                        sb.append("    [node=").append(nodeId)
+                            .append(", client=").append(client)
+                            .append(']').append(U.nl());
+                    }
+                }
+            }
+
             U.warn(log, sb.toString());
         }
 
@@ -1426,6 +1722,12 @@ public CommunicationListener getListener() {
             nioSrvr.dumpStats();
     }
 
+    /** */
+    private final ThreadLocal<Integer> threadConnIdx = new ThreadLocal<>();
+
+    /** */
+    private final AtomicInteger connIdx = new AtomicInteger();
+
     /** {@inheritDoc} */
     @Override public Map<String, Object> getNodeAttributes() throws IgniteSpiException {
         initFailureDetectionTimeout();
@@ -1439,6 +1741,8 @@ public CommunicationListener getListener() {
         assertParameter(msgQueueLimit >= 0, "msgQueueLimit >= 0");
         assertParameter(shmemPort > 0 || shmemPort == -1, "shmemPort > 0 || shmemPort == -1");
         assertParameter(selectorsCnt > 0, "selectorsCnt > 0");
+        assertParameter(connectionsPerNode > 0, "connectionsPerNode > 0");
+        assertParameter(connectionsPerNode <= 1024, "connectionsPerNode <= 1024");
 
         if (!failureDetectionTimeoutEnabled()) {
             assertParameter(reconCnt > 0, "reconnectCnt > 0");
@@ -1458,6 +1762,21 @@ public CommunicationListener getListener() {
                 "Specified 'unackedMsgsBufSize' is too low, it should be at least 'ackSndThreshold * 5'.");
         }
 
+        if (connectionsPerNode > 1) {
+            connPlc = new ConnectionPolicy() {
+                @Override public int connectionIndex() {
+                    return (int)(U.safeAbs(Thread.currentThread().getId()) % connectionsPerNode);
+                }
+            };
+        }
+        else {
+            connPlc = new ConnectionPolicy() {
+                @Override public int connectionIndex() {
+                    return 0;
+                }
+            };
+        }
+
         try {
             locHost = U.resolveLocalHost(locAddr);
         }
@@ -1495,6 +1814,7 @@ public CommunicationListener getListener() {
             res.put(createSpiAttributeName(ATTR_PORT), boundTcpPort);
             res.put(createSpiAttributeName(ATTR_SHMEM_PORT), boundTcpShmemPort >= 0 ? boundTcpShmemPort : null);
             res.put(createSpiAttributeName(ATTR_EXT_ADDRS), extAddrs);
+            res.put(createSpiAttributeName(ATTR_PAIRED_CONN), usePairedConnections);
 
             return res;
         }
@@ -1524,6 +1844,7 @@ public CommunicationListener getListener() {
             log.debug(configInfo("sockRcvBuf", sockRcvBuf));
             log.debug(configInfo("shmemPort", shmemPort));
             log.debug(configInfo("msgQueueLimit", msgQueueLimit));
+            log.debug(configInfo("connectionsPerNode", connectionsPerNode));
 
             if (failureDetectionTimeoutEnabled()) {
                 log.debug(configInfo("connTimeout", connTimeout));
@@ -1548,6 +1869,11 @@ public CommunicationListener getListener() {
                 ", slowClientQueueLimit=" + slowClientQueueLimit + ']');
         }
 
+        if (msgQueueLimit == 0)
+            U.quietAndWarn(log, "Message queue limit is set to 0 which may lead to " +
+                "potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes " +
+                "due to message queues growth on sender and reciever sides.");
+
         registerMBean(gridName, this, TcpCommunicationSpiMBean.class);
 
         connectGate = new ConnectGateway();
@@ -1642,9 +1968,9 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
 
                         assert formatter != null;
 
-                        UUID rmtNodeId = ses.meta(NODE_ID_META);
+                        ConnectionKey key = ses.meta(CONN_IDX_META);
 
-                        return rmtNodeId != null ? formatter.reader(rmtNodeId, msgFactory) : null;
+                        return key != null ? formatter.reader(key.nodeId(), msgFactory) : null;
                     }
                 };
 
@@ -1657,9 +1983,9 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
 
                         assert formatter != null;
 
-                        UUID rmtNodeId = ses.meta(NODE_ID_META);
+                        ConnectionKey key = ses.meta(CONN_IDX_META);
 
-                        return rmtNodeId != null ? formatter.writer(rmtNodeId) : null;
+                        return key != null ? formatter.writer(key.nodeId()) : null;
                     }
                 };
 
@@ -1716,6 +2042,7 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
                         .logger(log)
                         .selectorCount(selectorsCnt)
                         .gridName(gridName)
+                        .serverName("tcp-comm")
                         .tcpNoDelay(tcpNoDelay)
                         .directBuffer(directBuf)
                         .byteOrder(ByteOrder.nativeOrder())
@@ -1725,18 +2052,21 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
                         .directMode(true)
                         .metricsListener(metricsLsnr)
                         .writeTimeout(sockWriteTimeout)
+                        .selectorSpins(selectorSpins)
                         .filters(filters)
                         .writerFactory(writerFactory)
                         .skipRecoveryPredicate(skipRecoveryPred)
                         .messageQueueSizeListener(queueSizeMonitor)
+                        .balancing(usePairedConnections) // Current balancing logic assumes separate in/out connections.
                         .build();
 
                 boundTcpPort = port;
 
                 // Ack Port the TCP server was bound to.
                 if (log.isInfoEnabled())
-                    log.info("Successfully bound to TCP port [port=" + boundTcpPort +
-                        ", locHost=" + locHost + ']');
+                    log.info("Successfully bound communication NIO server to TCP port " +
+                        "[port=" + boundTcpPort + ", locHost=" + locHost + ", selectorsCnt=" + selectorsCnt +
+                        ", selectorSpins=" + srvr.selectorSpins() + ']');
 
                 srvr.idleTimeout(idleConnTimeout);
 
@@ -1837,8 +2167,12 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
         shmemWorkers.clear();
 
         // Force closing on stop (safety).
-        for (GridCommunicationClient client : clients.values())
-            client.forceClose();
+        for (GridCommunicationClient[] clients0 : clients.values()) {
+            for (GridCommunicationClient client : clients0) {
+                if (client != null)
+                    client.forceClose();
+            }
+        }
 
         // Clear resources.
         nioSrvr = null;
@@ -1863,8 +2197,12 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
             connectGate.stopped();
 
         // Force closing.
-        for (GridCommunicationClient client : clients.values())
-            client.forceClose();
+        for (GridCommunicationClient[] clients0 : clients.values()) {
+            for (GridCommunicationClient client : clients0) {
+                if (client != null)
+                    client.forceClose();
+            }
+        }
 
         getSpiContext().deregisterPorts();
 
@@ -1875,8 +2213,12 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
     @Override public void onClientDisconnected(IgniteFuture<?> reconnectFut) {
         connectGate.disconnected(reconnectFut);
 
-        for (GridCommunicationClient client : clients.values())
-            client.forceClose();
+        for (GridCommunicationClient[] clients0 : clients.values()) {
+            for (GridCommunicationClient client : clients0) {
+                if (client != null)
+                    client.forceClose();
+            }
+        }
 
         IgniteClientDisconnectedCheckedException err = new IgniteClientDisconnectedCheckedException(reconnectFut,
             "Failed to connect client node disconnected.");
@@ -1885,6 +2227,8 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
             clientFut.onDone(err);
 
         recoveryDescs.clear();
+        inRecDescs.clear();
+        outRecDescs.clear();
     }
 
     /** {@inheritDoc} */
@@ -1898,16 +2242,18 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
     void onNodeLeft(UUID nodeId) {
         assert nodeId != null;
 
-        GridCommunicationClient client = clients.get(nodeId);
-
-        if (client != null) {
-            if (log.isDebugEnabled())
-                log.debug("Forcing NIO client close since node has left [nodeId=" + nodeId +
-                    ", client=" + client + ']');
+        GridCommunicationClient[] clients0 = clients.remove(nodeId);
 
-            client.forceClose();
+        if (clients0 != null) {
+            for (GridCommunicationClient client : clients0) {
+                if (client != null) {
+                    if (log.isDebugEnabled())
+                        log.debug("Forcing NIO client close since node has left [nodeId=" + nodeId +
+                            ", client=" + client + ']');
 
-            clients.remove(nodeId, client);
+                    client.forceClose();
+                }
+            }
         }
     }
 
@@ -1982,11 +2328,13 @@ private void sendMessage0(ClusterNode node, Message msg, IgniteInClosure<IgniteE
         else {
             GridCommunicationClient client = null;
 
+            int connIdx = useMultipleConnections(node) ? connPlc.connectionIndex() : 0;
+
             try {
                 boolean retry;
 
                 do {
-                    client = reserveClient(node);
+                    client = reserveClient(node, connIdx);
 
                     UUID nodeId = null;
 
@@ -2000,7 +2348,7 @@ private void sendMessage0(ClusterNode node, Message msg, IgniteInClosure<IgniteE
                     if (!retry)
                         sentMsgsCnt.increment();
                     else {
-                        clients.remove(node.id(), client);
+                        removeNodeClient(node.id(), client);
 
                         ClusterNode node0 = getSpiContext().node(node.id());
 
@@ -2017,26 +2365,94 @@ private void sendMessage0(ClusterNode node, Message msg, IgniteInClosure<IgniteE
                 throw new IgniteSpiException("Failed to send message to remote node: " + node, e);
             }
             finally {
-                if (client != null && clients.remove(node.id(), client))
+                if (client != null && removeNodeClient(node.id(), client))
                     client.forceClose();
             }
         }
     }
 
+    /**
+     * @param nodeId Node ID.
+     * @param rmvClient Client to remove.
+     * @return {@code True} if client was removed.
+     */
+    private boolean removeNodeClient(UUID nodeId, GridCommunicationClient rmvClient) {
+        for (;;) {
+            GridCommunicationClient[] curClients = clients.get(nodeId);
+
+            if (curClients == null || rmvClient.connectionIndex() >= curClients.length || curClients[rmvClient.connectionIndex()] != rmvClient)
+                return false;
+
+            GridCommunicationClient[] newClients = Arrays.copyOf(curClients, curClients.length);
+
+            newClients[rmvClient.connectionIndex()] = null;
+
+            if (clients.replace(nodeId, curClients, newClients))
+                return true;
+        }
+    }
+
+    /**
+     * @param node Node.
+     * @param connIdx Connection index.
+     * @param addClient Client to add.
+     */
+    private void addNodeClient(ClusterNode node, int connIdx, GridCommunicationClient addClient) {
+        assert connectionsPerNode > 0 : connectionsPerNode;
+        assert connIdx == addClient.connectionIndex() : addClient;
+
+        if (connIdx >= connectionsPerNode) {
+            assert !usePairedConnections(node);
+
+            return;
+        }
+
+        for (;;) {
+            GridCommunicationClient[] curClients = clients.get(node.id());
+
+            assert curClients == null || curClients[connIdx] == null : "Client already created [node=" + node.id() +
+                ", connIdx=" + connIdx +
+                ", client=" + addClient +
+                ", oldClient=" + curClients[connIdx] + ']';
+
+            GridCommunicationClient[] newClients;
+
+            if (curClients == null) {
+                newClients = new GridCommunicationClient[useMultipleConnections(node) ? connectionsPerNode : 1];
+                newClients[connIdx] = addClient;
+
+                if (clients.putIfAbsent(node.id(), newClients) == null)
+                    break;
+            }
+            else {
+                newClients = Arrays.copyOf(curClients, curClients.length);
+                newClients[connIdx] = addClient;
+
+                if (clients.replace(node.id(), curClients, newClients))
+                    break;
+            }
+        }
+    }
+
     /**
      * Returns existing or just created client to node.
      *
      * @param node Node to which client should be open.
+     * @param connIdx Connection index.
      * @return The existing or just created client.
      * @throws IgniteCheckedException Thrown if any exception occurs.
      */
-    private GridCommunicationClient reserveClient(ClusterNode node) throws IgniteCheckedException {
+    private GridCommunicationClient reserveClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
         assert node != null;
+        assert (connIdx >= 0 && connIdx < connectionsPerNode) || !usePairedConnections(node) : connIdx;
 
         UUID nodeId = node.id();
 
         while (true) {
-            GridCommunicationClient client = clients.get(nodeId);
+            GridCommunicationClient[] curClients = clients.get(nodeId);
+
+            GridCommunicationClient client = curClients != null && connIdx < curClients.length ?
+                curClients[connIdx] : null;
 
             if (client == null) {
                 if (stopping)
@@ -2045,25 +2461,27 @@ private GridCommunicationClient reserveClient(ClusterNode node) throws IgniteChe
                 // Do not allow concurrent connects.
                 GridFutureAdapter<GridCommunicationClient> fut = new ConnectFuture();
 
-                GridFutureAdapter<GridCommunicationClient> oldFut = clientFuts.putIfAbsent(nodeId, fut);
+                ConnectionKey connKey = new ConnectionKey(nodeId, connIdx, -1);
+
+                GridFutureAdapter<GridCommunicationClient> oldFut = clientFuts.putIfAbsent(connKey, fut);
 
                 if (oldFut == null) {
                     try {
-                        GridCommunicationClient client0 = clients.get(nodeId);
+                        GridCommunicationClient[] curClients0 = clients.get(nodeId);
+
+                        GridCommunicationClient client0 = curClients0 != null && connIdx < curClients0.length ?
+                            curClients0[connIdx] : null;
 
                         if (client0 == null) {
-                            client0 = createNioClient(node);
+                            client0 = createNioClient(node, connIdx);
 
                             if (client0 != null) {
-                                GridCommunicationClient old = clients.put(nodeId, client0);
-
-                                assert old == null : "Client already created " +
-                                    "[node=" + node + ", client=" + client0 + ", oldClient=" + old + ']';
+                                addNodeClient(node, connIdx, client0);
 
                                 if (client0 instanceof GridTcpNioCommunicationClient) {
                                     GridTcpNioCommunicationClient tcpClient = ((GridTcpNioCommunicationClient)client0);
 
-                                    if (tcpClient.session().closeTime() > 0 && clients.remove(nodeId, client0)) {
+                                    if (tcpClient.session().closeTime() > 0 && removeNodeClient(nodeId, client0)) {
                                         if (log.isDebugEnabled())
                                             log.debug("Session was closed after client creation, will retry " +
                                                 "[node=" + node + ", client=" + client0 + ']');
@@ -2085,7 +2503,7 @@ private GridCommunicationClient reserveClient(ClusterNode node) throws IgniteChe
                             throw (Error)e;
                     }
                     finally {
-                        clientFuts.remove(nodeId, fut);
+                        clientFuts.remove(connKey, fut);
                     }
                 }
                 else
@@ -2097,27 +2515,31 @@ private GridCommunicationClient reserveClient(ClusterNode node) throws IgniteChe
                     continue;
 
                 if (getSpiContext().node(nodeId) == null) {
-                    if (clients.remove(nodeId, client))
+                    if (removeNodeClient(nodeId, client))
                         client.forceClose();
 
                     throw new IgniteSpiException("Destination node is not in topology: " + node.id());
                 }
             }
 
+            assert connIdx == client.connectionIndex() : client;
+
             if (client.reserve())
                 return client;
             else
                 // Client has just been closed by idle worker. Help it and try again.
-                clients.remove(nodeId, client);
+                removeNodeClient(nodeId, client);
         }
     }
 
     /**
      * @param node Node to create client for.
+     * @param connIdx Connection index.
      * @return Client.
      * @throws IgniteCheckedException If failed.
      */
-    @Nullable protected GridCommunicationClient createNioClient(ClusterNode node) throws IgniteCheckedException {
+    @Nullable private GridCommunicationClient createNioClient(ClusterNode node, int connIdx)
+        throws IgniteCheckedException {
         assert node != null;
 
         Integer shmemPort = node.attribute(createSpiAttributeName(ATTR_SHMEM_PORT));
@@ -2136,6 +2558,7 @@ private GridCommunicationClient reserveClient(ClusterNode node) throws IgniteChe
             try {
                 GridCommunicationClient client = createShmemClient(
                     node,
+                    connIdx,
                     shmemPort);
 
                 if (log.isDebugEnabled())
@@ -2158,7 +2581,7 @@ else if (log.isDebugEnabled())
         connectGate.enter();
 
         try {
-            GridCommunicationClient client = createTcpClient(node);
+            GridCommunicationClient client = createTcpClient(node, connIdx);
 
             if (log.isDebugEnabled())
                 log.debug("TCP client created: " + client);
@@ -2173,10 +2596,12 @@ else if (log.isDebugEnabled())
     /**
      * @param node Node.
      * @param port Port.
+     * @param connIdx Connection index.
      * @return Client.
      * @throws IgniteCheckedException If failed.
      */
-    @Nullable protected GridCommunicationClient createShmemClient(ClusterNode node,
+    @Nullable private GridCommunicationClient createShmemClient(ClusterNode node,
+        int connIdx,
         Integer port) throws IgniteCheckedException {
         int attempt = 1;
 
@@ -2190,7 +2615,9 @@ else if (log.isDebugEnabled())
             GridCommunicationClient client;
 
             try {
-                client = new GridShmemCommunicationClient(metricsLsnr,
+                client = new GridShmemCommunicationClient(
+                    connIdx,
+                    metricsLsnr,
                     port,
                     timeoutHelper.nextTimeoutChunk(connTimeout),
                     log,
@@ -2211,7 +2638,12 @@ else if (log.isDebugEnabled())
             }
 
             try {
-                safeHandshake(client, null, node.id(), timeoutHelper.nextTimeoutChunk(connTimeout0), null);
+                safeHandshake(client,
+                    null,
+                    node.id(),
+                    timeoutHelper.nextTimeoutChunk(connTimeout0),
+                    null,
+                    null);
             }
             catch (HandshakeTimeoutException | IgniteSpiOperationTimeoutException e) {
                 client.forceClose();
@@ -2270,10 +2702,10 @@ else if (log.isDebugEnabled())
      */
     private void checkClientQueueSize(GridNioSession ses, int msgQueueSize) {
         if (slowClientQueueLimit > 0 && msgQueueSize > slowClientQueueLimit) {
-            UUID id = ses.meta(NODE_ID_META);
+            ConnectionKey id = ses.meta(CONN_IDX_META);
 
             if (id != null) {
-                ClusterNode node = getSpiContext().node(id);
+                ClusterNode node = getSpiContext().node(id.nodeId);
 
                 if (node != null && node.isClient()) {
                     String msg = "Client node outbound message queue size exceeded slowClientQueueLimit, " +
@@ -2283,11 +2715,9 @@ private void checkClientQueueSize(GridNioSession ses, int msgQueueSize) {
                         ", clientNode=" + node +
                         ", slowClientQueueLimit=" + slowClientQueueLimit + ']';
 
-                    U.quietAndWarn(
-                        log,
-                        msg);
+                    U.quietAndWarn(log, msg);
 
-                    getSpiContext().failNode(id, msg);
+                    getSpiContext().failNode(id.nodeId(), msg);
                 }
             }
         }
@@ -2297,10 +2727,11 @@ private void checkClientQueueSize(GridNioSession ses, int msgQueueSize) {
      * Establish TCP connection to remote node and returns client.
      *
      * @param node Remote node.
+     * @param connIdx Connection index.
      * @return Client.
      * @throws IgniteCheckedException If failed.
      */
-    protected GridCommunicationClient createTcpClient(ClusterNode node) throws IgniteCheckedException {
+    protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
         Collection<String> rmtAddrs0 = node.attribute(createSpiAttributeName(ATTR_ADDRS));
         Collection<String> rmtHostNames0 = node.attribute(createSpiAttributeName(ATTR_HOST_NAMES));
         Integer boundPort = node.attribute(createSpiAttributeName(ATTR_PORT));
@@ -2368,7 +2799,9 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                             "(node left topology): " + node);
                     }
 
-                    GridNioRecoveryDescriptor recoveryDesc = recoveryDescriptor(node);
+                    ConnectionKey connKey = new ConnectionKey(node.id(), connIdx, -1);
+
+                    GridNioRecoveryDescriptor recoveryDesc = outRecoveryDescriptor(node, connKey);
 
                     if (!recoveryDesc.reserve()) {
                         U.closeQuiet(ch);
@@ -2395,11 +2828,14 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                             sslMeta.sslEngine(sslEngine);
                         }
 
+                        Integer handshakeConnIdx = useMultipleConnections(node) ? connIdx : null;
+
                         rcvCnt = safeHandshake(ch,
                             recoveryDesc,
                             node.id(),
                             timeoutHelper.nextTimeoutChunk(connTimeout0),
-                            sslMeta);
+                            sslMeta,
+                            handshakeConnIdx);
 
                         if (rcvCnt == -1)
                             return null;
@@ -2410,7 +2846,7 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                     }
 
                     try {
-                        meta.put(NODE_ID_META, node.id());
+                        meta.put(CONN_IDX_META, connKey);
 
                         if (recoveryDesc != null) {
                             recoveryDesc.onHandshake(rcvCnt);
@@ -2420,7 +2856,7 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
 
                         GridNioSession ses = nioSrvr.createSession(ch, meta).get();
 
-                        client = new GridTcpNioCommunicationClient(ses, log);
+                        client = new GridTcpNioCommunicationClient(connIdx, ses, log);
 
                         conn = true;
                     }
@@ -2564,6 +3000,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
      * @param rmtNodeId Remote node.
      * @param timeout Timeout for handshake.
      * @param sslMeta Session meta.
+     * @param handshakeConnIdx Non null connection index if need send it in handshake.
      * @throws IgniteCheckedException If handshake failed or wasn't completed withing timeout.
      * @return Handshake response.
      */
@@ -2573,7 +3010,8 @@ private <T> long safeHandshake(
         @Nullable GridNioRecoveryDescriptor recovery,
         UUID rmtNodeId,
         long timeout,
-        GridSslMeta sslMeta
+        GridSslMeta sslMeta,
+        @Nullable Integer handshakeConnIdx
     ) throws IgniteCheckedException {
         HandshakeTimeoutObject<T> obj = new HandshakeTimeoutObject<>(client, U.currentTimeMillis() + timeout);
 
@@ -2655,14 +3093,28 @@ else if (log.isDebugEnabled())
                             "fully initialized [isStopping=" + getSpiContext().isStopping() + ']');
 
                     if (recovery != null) {
-                        HandshakeMessage msg = new HandshakeMessage(locNode.id(),
-                            recovery.incrementConnectCount(),
-                            recovery.received());
+                        HandshakeMessage msg;
+
+                        int msgSize = 33;
+
+                        if (handshakeConnIdx != null) {
+                            msg = new HandshakeMessage2(locNode.id(),
+                                recovery.incrementConnectCount(),
+                                recovery.received(),
+                                handshakeConnIdx);
+
+                            msgSize += 4;
+                        }
+                        else {
+                            msg = new HandshakeMessage(locNode.id(),
+                                recovery.incrementConnectCount(),
+                                recovery.received());
+                        }
 
                         if (log.isDebugEnabled())
                             log.debug("Write handshake message [rmtNode=" + rmtNodeId + ", msg=" + msg + ']');
 
-                        buf = ByteBuffer.allocate(33);
+                        buf = ByteBuffer.allocate(msgSize);
 
                         buf.order(ByteOrder.nativeOrder());
 
@@ -2689,6 +3141,7 @@ else if (log.isDebugEnabled())
                         else
                             ch.write(ByteBuffer.wrap(nodeIdMessage().nodeIdBytesWithType));
                     }
+
                     if (recovery != null) {
                         if (log.isDebugEnabled())
                             log.debug("Waiting for handshake [rmtNode=" + rmtNodeId + ']');
@@ -2818,26 +3271,81 @@ public void simulateNodeFailure() {
 
         U.join(commWorker, log);
 
-        for (GridCommunicationClient client : clients.values())
-            client.forceClose();
+        for (GridCommunicationClient[] clients0 : clients.values()) {
+            for (GridCommunicationClient client : clients0) {
+                if (client != null)
+                    client.forceClose();
+            }
+        }
+    }
+
+    /**
+     * @param node Node.
+     * @param key Connection key.
+     * @return Recovery descriptor for outgoing connection.
+     */
+    private GridNioRecoveryDescriptor outRecoveryDescriptor(ClusterNode node, ConnectionKey key) {
+        if (usePairedConnections(node))
+            return recoveryDescriptor(outRecDescs, true, node, key);
+        else
+            return recoveryDescriptor(recoveryDescs, false, node, key);
     }
 
     /**
      * @param node Node.
-     * @return Recovery receive data for given node.
+     * @param key Connection key.
+     * @return Recovery descriptor for incoming connection.
+     */
+    private GridNioRecoveryDescriptor inRecoveryDescriptor(ClusterNode node, ConnectionKey key) {
+        if (usePairedConnections(node))
+            return recoveryDescriptor(inRecDescs, true, node, key);
+        else
+            return recoveryDescriptor(recoveryDescs, false, node, key);
+    }
+
+    /**
+     * @param node Node.
+     * @return {@code True} if given node supports multiple connections per-node for communication.
+     */
+    private boolean useMultipleConnections(ClusterNode node) {
+        return node.version().compareToIgnoreTimestamp(MULTIPLE_CONN_SINCE_VER) >= 0;
+    }
+
+    /**
+     * @param node Node.
+     * @return {@code True} if can use in/out connection pair for communication.
      */
-    private GridNioRecoveryDescriptor recoveryDescriptor(ClusterNode node) {
-        ClientKey id = new ClientKey(node.id(), node.order());
+    private boolean usePairedConnections(ClusterNode node) {
+        if (usePairedConnections) {
+            Boolean attr = node.attribute(createSpiAttributeName(ATTR_PAIRED_CONN));
+
+            return attr != null && attr;
+        }
+
+        return false;
+    }
 
-        GridNioRecoveryDescriptor recovery = recoveryDescs.get(id);
+    /**
+     * @param recoveryDescs Descriptors map.
+     * @param pairedConnections {@code True} if in/out connections pair is used for communication with node.
+     * @param node Node.
+     * @param key Connection key.
+     * @return Recovery receive data for given node.
+     */
+    private GridNioRecoveryDescriptor recoveryDescriptor(
+        ConcurrentMap<ConnectionKey, GridNioRecoveryDescriptor> recoveryDescs,
+        boolean pairedConnections,
+        ClusterNode node,
+        ConnectionKey key) {
+        GridNioRecoveryDescriptor recovery = recoveryDescs.get(key);
 
         if (recovery == null) {
             int maxSize = Math.max(msgQueueLimit, ackSndThreshold);
 
-            int queueLimit = unackedMsgsBufSize != 0 ? unackedMsgsBufSize : (maxSize * 5);
+            int queueLimit = unackedMsgsBufSize != 0 ? unackedMsgsBufSize : (maxSize * 128);
 
-            GridNioRecoveryDescriptor old =
-                recoveryDescs.putIfAbsent(id, recovery = new GridNioRecoveryDescriptor(queueLimit, node, log));
+            GridNioRecoveryDescriptor old = recoveryDescs.putIfAbsent(key,
+                recovery = new GridNioRecoveryDescriptor(pairedConnections, queueLimit, node, log));
 
             if (old != null)
                 recovery = old;
@@ -2879,54 +3387,6 @@ private NodeIdMessage nodeIdMessage() {
         return S.toString(TcpCommunicationSpi.class, this);
     }
 
-    /**
-     *
-     */
-    private static class ClientKey {
-        /** */
-        private UUID nodeId;
-
-        /** */
-        private long order;
-
-        /**
-         * @param nodeId Node ID.
-         * @param order Node order.
-         */
-        private ClientKey(UUID nodeId, long order) {
-            this.nodeId = nodeId;
-            this.order = order;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean equals(Object obj) {
-            if (this == obj)
-                return true;
-
-            if (obj == null || getClass() != obj.getClass())
-                return false;
-
-            ClientKey other = (ClientKey)obj;
-
-            return order == other.order && nodeId.equals(other.nodeId);
-
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hashCode() {
-            int res = nodeId.hashCode();
-
-            res = 31 * res + (int)(order ^ (order >>> 32));
-
-            return res;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return S.toString(ClientKey.class, this);
-        }
-    }
-
     /** Internal exception class for proper timeout handling. */
     private static class HandshakeTimeoutException extends IgniteCheckedException {
         /** */
@@ -3026,9 +3486,9 @@ private ShmemWorker(IpcEndpoint endpoint) {
 
                         assert formatter != null;
 
-                        UUID rmtNodeId = ses.meta(NODE_ID_META);
+                        ConnectionKey connKey = ses.meta(CONN_IDX_META);
 
-                        return rmtNodeId != null ? formatter.writer(rmtNodeId) : null;
+                        return connKey != null ? formatter.writer(connKey.nodeId()) : null;
                     }
                 };
 
@@ -3042,9 +3502,9 @@ private ShmemWorker(IpcEndpoint endpoint) {
 
                         assert formatter != null;
 
-                        UUID rmtNodeId = ses.meta(NODE_ID_META);
+                        ConnectionKey connKey = ses.meta(CONN_IDX_META);
 
-                        return rmtNodeId != null ? formatter.reader(rmtNodeId, msgFactory) : null;
+                        return connKey != null ? formatter.reader(connKey.nodeId(), msgFactory) : null;
                     }
                 };
 
@@ -3125,62 +3585,108 @@ private CommunicationWorker(String gridName) {
         private void processIdle() {
             cleanupRecovery();
 
-            for (Map.Entry<UUID, GridCommunicationClient> e : clients.entrySet()) {
+            for (Map.Entry<UUID, GridCommunicationClient[]> e : clients.entrySet()) {
                 UUID nodeId = e.getKey();
 
-                GridCommunicationClient client = e.getValue();
+                for (GridCommunicationClient client : e.getValue()) {
+                    if (client == null)
+                        continue;
 
-                ClusterNode node = getSpiContext().node(nodeId);
+                    ClusterNode node = getSpiContext().node(nodeId);
 
-                if (node == null) {
-                    if (log.isDebugEnabled())
-                        log.debug("Forcing close of non-existent node connection: " + nodeId);
+                    if (node == null) {
+                        if (log.isDebugEnabled())
+                            log.debug("Forcing close of non-existent node connection: " + nodeId);
 
-                    client.forceClose();
+                        client.forceClose();
 
-                    clients.remove(nodeId, client);
+                        removeNodeClient(nodeId, client);
 
-                    continue;
-                }
+                        continue;
+                    }
 
-                GridNioRecoveryDescriptor recovery = null;
+                    GridNioRecoveryDescriptor recovery = null;
 
-                if (client instanceof GridTcpNioCommunicationClient) {
-                    recovery = recoveryDescs.get(new ClientKey(node.id(), node.order()));
+                    if (!usePairedConnections(node) && client instanceof GridTcpNioCommunicationClient) {
+                        recovery = recoveryDescs.get(new ConnectionKey(node.id(), client.connectionIndex(), -1));
 
-                    if (recovery != null && recovery.lastAcknowledged() != recovery.received()) {
-                        RecoveryLastReceivedMessage msg = new RecoveryLastReceivedMessage(recovery.received());
+                        if (recovery != null && recovery.lastAcknowledged() != recovery.received()) {
+                            RecoveryLastReceivedMessage msg = new RecoveryLastReceivedMessage(recovery.received());
 
-                        if (log.isDebugEnabled())
-                            log.debug("Send recovery acknowledgement on timeout [rmtNode=" + nodeId +
-                                ", rcvCnt=" + msg.received() + ']');
+                            if (log.isDebugEnabled())
+                                log.debug("Send recovery acknowledgement on timeout [rmtNode=" + nodeId +
+                                    ", rcvCnt=" + msg.received() + ']');
 
-                        nioSrvr.sendSystem(((GridTcpNioCommunicationClient)client).session(), msg);
+                            try {
+                                nioSrvr.sendSystem(((GridTcpNioCommunicationClient)client).session(), msg);
 
-                        recovery.lastAcknowledged(msg.received());
+                                recovery.lastAcknowledged(msg.received());
+                            }
+                            catch (IgniteCheckedException err) {
+                                U.error(log, "Failed to send message: " + err, err);
+                            }
 
-                        continue;
+                            continue;
+                        }
                     }
-                }
 
-                long idleTime = client.getIdleTime();
+                    long idleTime = client.getIdleTime();
+
+                    if (idleTime >= idleConnTimeout) {
+                        if (recovery == null && usePairedConnections(node))
+                            recovery = outRecDescs.get(new ConnectionKey(node.id(), client.connectionIndex(), -1));
+
+                        if (recovery != null &&
+                            recovery.nodeAlive(getSpiContext().node(nodeId)) &&
+                            !recovery.messagesRequests().isEmpty()) {
+                            if (log.isDebugEnabled())
+                                log.debug("Node connection is idle, but there are unacknowledged messages, " +
+                                    "will wait: " + nodeId);
+
+                            continue;
+                        }
 
-                if (idleTime >= idleConnTimeout) {
-                    if (recovery != null &&
-                        recovery.nodeAlive(getSpiContext().node(nodeId)) &&
-                        !recovery.messagesFutures().isEmpty()) {
                         if (log.isDebugEnabled())
-                            log.debug("Node connection is idle, but there are unacknowledged messages, " +
-                                "will wait: " + nodeId);
+                            log.debug("Closing idle node connection: " + nodeId);
 
-                        continue;
+                        if (client.close() || client.closed())
+                            removeNodeClient(nodeId, client);
                     }
+                }
+            }
 
-                    if (log.isDebugEnabled())
-                        log.debug("Closing idle node connection: " + nodeId);
+            for (GridNioSession ses : nioSrvr.sessions()) {
+                GridNioRecoveryDescriptor recovery = ses.inRecoveryDescriptor();
+
+                if (recovery != null && usePairedConnections(recovery.node())) {
+                    assert ses.accepted() : ses;
+
+                    sendAckOnTimeout(recovery, ses);
+                }
+            }
+        }
+
+        /**
+         * @param recovery Recovery descriptor.
+         * @param ses Session.
+         */
+        private void sendAckOnTimeout(GridNioRecoveryDescriptor recovery, GridNioSession ses) {
+            if (recovery != null && recovery.lastAcknowledged() != recovery.received()) {
+                RecoveryLastReceivedMessage msg = new RecoveryLastReceivedMessage(recovery.received());
+
+                if (log.isDebugEnabled()) {
+                    log.debug("Send recovery acknowledgement on timeout [rmtNode=" + recovery.node().id() +
+                        ", rcvCnt=" + msg.received() +
+                        ", lastAcked=" + recovery.lastAcknowledged() + ']');
+                }
+
+                try {
+                    nioSrvr.sendSystem(ses, msg);
 
-                    if (client.close() || client.closed())
-                        clients.remove(nodeId, client);
+                    recovery.lastAcknowledged(msg.received());
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to send message: " + e, e);
                 }
             }
         }
@@ -3189,15 +3695,24 @@ private void processIdle() {
          *
          */
         private void cleanupRecovery() {
-            Set<ClientKey> left = null;
+            cleanupRecovery(recoveryDescs);
+            cleanupRecovery(inRecDescs);
+            cleanupRecovery(outRecDescs);
+        }
+
+        /**
+         * @param recoveryDescs Recovery descriptors to cleanup.
+         */
+        private void cleanupRecovery(ConcurrentMap<ConnectionKey, GridNioRecoveryDescriptor> recoveryDescs) {
+            Set<ConnectionKey> left = null;
 
-            for (Map.Entry<ClientKey, GridNioRecoveryDescriptor> e : recoveryDescs.entrySet()) {
+            for (Map.Entry<ConnectionKey, GridNioRecoveryDescriptor> e : recoveryDescs.entrySet()) {
                 if (left != null && left.contains(e.getKey()))
                     continue;
 
-                GridNioRecoveryDescriptor recoverySnd = e.getValue();
+                GridNioRecoveryDescriptor recoveryDesc = e.getValue();
 
-                if (!recoverySnd.nodeAlive(getSpiContext().node(recoverySnd.node().id()))) {
+                if (!recoveryDesc.nodeAlive(getSpiContext().node(e.getKey().nodeId()))) {
                     if (left == null)
                         left = new HashSet<>();
 
@@ -3208,11 +3723,11 @@ private void cleanupRecovery() {
             if (left != null) {
                 assert !left.isEmpty();
 
-                for (ClientKey id : left) {
-                    GridNioRecoveryDescriptor recoverySnd = recoveryDescs.get(id);
+                for (ConnectionKey id : left) {
+                    GridNioRecoveryDescriptor recoveryDesc = recoveryDescs.get(id);
 
-                    if (recoverySnd != null && recoverySnd.onNodeLeft())
-                        recoveryDescs.remove(id);
+                    if (recoveryDesc != null && recoveryDesc.onNodeLeft())
+                        recoveryDescs.remove(id, recoveryDesc);
                 }
             }
         }
@@ -3221,45 +3736,43 @@ private void cleanupRecovery() {
          * @param sesInfo Disconnected session information.
          */
         private void processDisconnect(DisconnectedSessionInfo sesInfo) {
-            if (sesInfo.reconnect) {
-                GridNioRecoveryDescriptor recoveryDesc = sesInfo.recoveryDesc;
-
-                ClusterNode node = recoveryDesc.node();
+            GridNioRecoveryDescriptor recoveryDesc = sesInfo.recoveryDesc;
 
-                if (!recoveryDesc.nodeAlive(getSpiContext().node(node.id())))
-                    return;
+            ClusterNode node = recoveryDesc.node();
 
-                try {
-                    if (log.isDebugEnabled())
-                        log.debug("Recovery reconnect [rmtNode=" + recoveryDesc.node().id() + ']');
+            if (!recoveryDesc.nodeAlive(getSpiContext().node(node.id())))
+                return;
 
-                    GridCommunicationClient client = reserveClient(node);
+            try {
+                if (log.isDebugEnabled())
+                    log.debug("Recovery reconnect [rmtNode=" + recoveryDesc.node().id() + ']');
 
-                    client.release();
-                }
-                catch (IgniteCheckedException | IgniteException e) {
-                    try {
-                        if (recoveryDesc.nodeAlive(getSpiContext().node(node.id())) && getSpiContext().pingNode(node.id())) {
-                            if (log.isDebugEnabled())
-                                log.debug("Recovery reconnect failed, will retry " +
-                                    "[rmtNode=" + recoveryDesc.node().id() + ", err=" + e + ']');
+                GridCommunicationClient client = reserveClient(node, sesInfo.connIdx);
 
-                            addProcessDisconnectRequest(sesInfo);
-                        }
-                        else {
-                            if (log.isDebugEnabled())
-                                log.debug("Recovery reconnect failed, " +
-                                    "node left [rmtNode=" + recoveryDesc.node().id() + ", err=" + e + ']');
+                client.release();
+            }
+            catch (IgniteCheckedException | IgniteException e) {
+                try {
+                    if (recoveryDesc.nodeAlive(getSpiContext().node(node.id())) && getSpiContext().pingNode(node.id())) {
+                        if (log.isDebugEnabled())
+                            log.debug("Recovery reconnect failed, will retry " +
+                                "[rmtNode=" + recoveryDesc.node().id() + ", err=" + e + ']');
 
-                            onException("Recovery reconnect failed, node left [rmtNode=" + recoveryDesc.node().id() + "]",
-                                e);
-                        }
+                        addProcessDisconnectRequest(sesInfo);
                     }
-                    catch (IgniteClientDisconnectedException e0) {
+                    else {
                         if (log.isDebugEnabled())
-                            log.debug("Failed to ping node, client disconnected.");
+                            log.debug("Recovery reconnect failed, " +
+                                "node left [rmtNode=" + recoveryDesc.node().id() + ", err=" + e + ']');
+
+                        onException("Recovery reconnect failed, node left [rmtNode=" + recoveryDesc.node().id() + "]",
+                            e);
                     }
                 }
+                catch (IgniteClientDisconnectedException e0) {
+                    if (log.isDebugEnabled())
+                        log.debug("Failed to ping node, client disconnected.");
+                }
             }
         }
 
@@ -3462,6 +3975,13 @@ public HandshakeMessage(UUID nodeId, long connectCnt, long rcvCnt) {
             this.rcvCnt = rcvCnt;
         }
 
+        /**
+         * @return Connection index.
+         */
+        public int connectionIndex() {
+            return 0;
+        }
+
         /**
          * @return Connect count.
          */
@@ -3493,7 +4013,7 @@ public UUID nodeId() {
             if (buf.remaining() < 33)
                 return false;
 
-            buf.put(HANDSHAKE_MSG_TYPE);
+            buf.put(directType());
 
             byte[] bytes = U.uuidToBytes(nodeId);
 
@@ -3542,6 +4062,78 @@ public UUID nodeId() {
         }
     }
 
+    /**
+     * Updated handshake message.
+     */
+    @SuppressWarnings("PublicInnerClass")
+    public static class HandshakeMessage2 extends HandshakeMessage {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private int connIdx;
+
+        /**
+         *
+         */
+        public HandshakeMessage2() {
+            // No-op.
+        }
+
+        /**
+         * @param nodeId Node ID.
+         * @param connectCnt Connect count.
+         * @param rcvCnt Number of received messages.
+         * @param connIdx Connection index.
+         */
+        HandshakeMessage2(UUID nodeId, long connectCnt, long rcvCnt, int connIdx) {
+            super(nodeId, connectCnt, rcvCnt);
+
+            this.connIdx = connIdx;
+        }
+
+        /** {@inheritDoc} */
+        @Override public byte directType() {
+            return -44;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int connectionIndex() {
+            return connIdx;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+            if (!super.writeTo(buf, writer))
+                return false;
+
+            if (buf.remaining() < 4)
+                return false;
+
+            buf.putInt(connIdx);
+
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+            if (!super.readFrom(buf, reader))
+                return false;
+
+            if (buf.remaining() < 4)
+                return false;
+
+            connIdx = buf.getInt();
+
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(HandshakeMessage2.class, this);
+        }
+    }
+
     /**
      * Recovery acknowledgment message.
      */
@@ -3787,16 +4379,15 @@ private static class DisconnectedSessionInfo {
         private final GridNioRecoveryDescriptor recoveryDesc;
 
         /** */
-        private final boolean reconnect;
+        private int connIdx;
 
         /**
          * @param recoveryDesc Recovery descriptor.
-         * @param reconnect Reconnect flag.
+         * @param connIdx Connection index.
          */
-        DisconnectedSessionInfo(@Nullable GridNioRecoveryDescriptor recoveryDesc,
-            boolean reconnect) {
+        DisconnectedSessionInfo(@Nullable GridNioRecoveryDescriptor recoveryDesc, int connIdx) {
             this.recoveryDesc = recoveryDesc;
-            this.reconnect = reconnect;
+            this.connIdx = connIdx;
         }
 
         /** {@inheritDoc} */
@@ -3804,4 +4395,85 @@ private static class DisconnectedSessionInfo {
             return S.toString(DisconnectedSessionInfo.class, this);
         }
     }
+
+    /**
+     *
+     */
+    private static class ConnectionKey {
+        /** */
+        private final UUID nodeId;
+
+        /** */
+        private final int idx;
+
+        /** */
+        private final long connCnt;
+
+        /**
+         * @param nodeId Node ID.
+         * @param idx Connection index.
+         * @param connCnt Connection counter (set only for incoming connections).
+         */
+        ConnectionKey(UUID nodeId, int idx, long connCnt) {
+            this.nodeId = nodeId;
+            this.idx = idx;
+            this.connCnt = connCnt;
+        }
+
+        /**
+         * @return Connection counter.
+         */
+        long connectCount() {
+            return connCnt;
+        }
+
+        /**
+         * @return Node ID.
+         */
+        UUID nodeId() {
+            return nodeId;
+        }
+
+        /**
+         * @return Connection index.
+         */
+        int connectionIndex() {
+            return idx;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            ConnectionKey key = (ConnectionKey) o;
+
+            return idx == key.idx && nodeId.equals(key.nodeId);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            int res = nodeId.hashCode();
+            res = 31 * res + idx;
+            return res;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(ConnectionKey.class, this);
+        }
+    }
+
+    /**
+     *
+     */
+    interface ConnectionPolicy {
+        /**
+         * @return Thread connection index.
+         */
+        int connectionIndex();
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java
index 482e2efbc4616..c7a1a535c69a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.spi.communication.tcp;
 
+import org.apache.ignite.internal.util.nio.GridNioServer;
 import org.apache.ignite.mxbean.MXBeanDescription;
 import org.apache.ignite.spi.IgniteSpiManagementMBean;
 
@@ -43,6 +44,35 @@ public interface TcpCommunicationSpiMBean extends IgniteSpiManagementMBean {
     @MXBeanDescription("Port number.")
     public int getLocalPort();
 
+    /**
+     * Returns {@code true} if {@code TcpCommunicationSpi} should
+     * maintain connection for outgoing and incoming messages separately.
+     * In this case total number of connections between local and some remote node
+     * is {@link #getConnectionsPerNode()} * 2.
+     * <p>
+     * Returns {@code false} if each connection of {@link #getConnectionsPerNode()}
+     * should be used for outgoing and incoming messages. In this case load NIO selectors load
+     * balancing of {@link GridNioServer} will be disabled.
+     * <p>
+     * Default is {@code true}.
+     *
+     * @return {@code true} to use paired connections and {@code false} otherwise.
+     * @see #getConnectionsPerNode()
+     */
+    @MXBeanDescription("Paired connections used.")
+    public boolean isUsePairedConnections();
+
+    /**
+     * Gets number of connections to each remote node. if {@link #isUsePairedConnections()}
+     * is {@code true} then number of connections is doubled and half is used for incoming and
+     * half for outgoing messages.
+     *
+     * @return Number of connections per node.
+     * @see #isUsePairedConnections()
+     */
+    @MXBeanDescription("Connections per node.")
+    public int getConnectionsPerNode();
+
     /**
      * Gets local port for shared memory communication.
      *
@@ -152,6 +182,16 @@ public interface TcpCommunicationSpiMBean extends IgniteSpiManagementMBean {
     @MXBeanDescription("Reconnect count on connection failure.")
     public int getReconnectCount();
 
+    /**
+     * Defines how many non-blocking {@code selector.selectNow()} should be made before
+     * falling into {@code selector.select(long)} in NIO server. Long value. Default is {@code 0}.
+     * Can be set to {@code Long.MAX_VALUE} so selector threads will never block.
+     *
+     * @return Selector thread busy-loop iterations.
+     */
+    @MXBeanDescription("Selector thread busy-loop iterations.")
+    public long getSelectorSpins();
+
     /**
      * Gets value for {@code TCP_NODELAY} socket option.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 9a36f1a9d7011..8a9f1c9446739 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -3408,7 +3408,7 @@ else if (log.isDebugEnabled())
                     if (log.isDebugEnabled())
                         log.debug("Node validation failed [res=" + err + ", node=" + node + ']');
 
-                    utilityPool.submit(
+                    utilityPool.execute(
                         new Runnable() {
                             @Override public void run() {
                                 boolean ping = node.id().equals(err.nodeId()) ? pingNode(node) : pingNode(err.nodeId());
@@ -3453,7 +3453,7 @@ else if (log.isDebugEnabled())
                 final String rmtMarsh = node.attribute(ATTR_MARSHALLER);
 
                 if (!F.eq(locMarsh, rmtMarsh)) {
-                    utilityPool.submit(
+                    utilityPool.execute(
                         new Runnable() {
                             @Override public void run() {
                                 String errMsg = "Local node's marshaller differs from remote node's marshaller " +
@@ -3510,7 +3510,7 @@ else if (log.isDebugEnabled())
                 boolean locLateAssignBool = locLateAssign != null ? locLateAssign : false;
 
                 if (locMarshUseDfltSuidBool != rmtMarshUseDfltSuidBool) {
-                    utilityPool.submit(
+                    utilityPool.execute(
                         new Runnable() {
                             @Override public void run() {
                                 String errMsg = "Local node's " + IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID +
@@ -3552,7 +3552,7 @@ else if (log.isDebugEnabled())
                 final boolean rmtMarshCompactFooterBool = rmtMarshCompactFooter != null ? rmtMarshCompactFooter : false;
 
                 if (locMarshCompactFooterBool != rmtMarshCompactFooterBool) {
-                    utilityPool.submit(
+                    utilityPool.execute(
                         new Runnable() {
                             @Override public void run() {
                                 String errMsg = "Local node's binary marshaller \"compactFooter\" property differs from " +
@@ -3590,7 +3590,7 @@ else if (log.isDebugEnabled())
                 final boolean rmtMarshStrSerialVer2Bool = rmtMarshStrSerialVer2 != null ? rmtMarshStrSerialVer2 : false;
 
                 if (locMarshStrSerialVer2Bool != rmtMarshStrSerialVer2Bool) {
-                    utilityPool.submit(
+                    utilityPool.execute(
                         new Runnable() {
                             @Override public void run() {
                                 String errMsg = "Local node's " + IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 +
@@ -3663,7 +3663,7 @@ else if (log.isDebugEnabled())
                     final Boolean rmtSrvcCompatibilityEnabled = node.attribute(ATTR_SERVICES_COMPATIBILITY_MODE);
 
                     if (!F.eq(locSrvcCompatibilityEnabled, rmtSrvcCompatibilityEnabled)) {
-                        utilityPool.submit(
+                        utilityPool.execute(
                             new Runnable() {
                                 @Override public void run() {
                                     String errMsg = "Local node's " + IGNITE_SERVICES_COMPATIBILITY_MODE +
@@ -3698,7 +3698,7 @@ else if (log.isDebugEnabled())
                     }
                 }
                 else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
-                    utilityPool.submit(
+                    utilityPool.execute(
                         new Runnable() {
                             @Override public void run() {
                                 String errMsg = "Remote node doesn't support lazy services configuration and " +
diff --git a/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java b/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java
index d1c8d19f203ed..127778bfb2089 100644
--- a/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java
+++ b/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java
@@ -184,6 +184,7 @@ public void start() {
         try {
             srv = new GridNioServer.Builder<byte[]>()
                 .address(addr == null ? InetAddress.getLocalHost() : addr)
+                .serverName("sock-streamer")
                 .port(port)
                 .listener(lsnr)
                 .logger(log)
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
index 55557dddccf4e..d1735943b8efc 100644
--- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
@@ -20,6 +20,7 @@
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.NotNull;
 
 /**
@@ -62,4 +63,9 @@ public IgniteThreadFactory(String gridName, String threadName) {
     @Override public Thread newThread(@NotNull Runnable r) {
         return new IgniteThread(gridName, threadName, r, idxGen.incrementAndGet());
     }
-}
\ No newline at end of file
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IgniteThreadFactory.class, this, super.toString());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteSlowClientDetectionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteSlowClientDetectionSelfTest.java
index 760313b8737ea..57218876d0a46 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteSlowClientDetectionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteSlowClientDetectionSelfTest.java
@@ -75,6 +75,7 @@ private int nodeCount() {
         commSpi.setSlowClientQueueLimit(50);
         commSpi.setSharedMemoryPort(-1);
         commSpi.setIdleConnectionTimeout(300_000);
+        commSpi.setConnectionsPerNode(1);
 
         cfg.setCommunicationSpi(commSpi);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java
new file mode 100644
index 0000000000000..e95b1ec0b514c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+/**
+ *
+ */
+public class IgniteCommunicationBalanceMultipleConnectionsTest extends IgniteCommunicationBalanceTest {
+    /** {@inheritDoc} */
+    @Override protected int connectionsPerNode() {
+        return 5;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
new file mode 100644
index 0000000000000..e142aefdb2343
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
+import org.apache.ignite.internal.util.nio.GridNioServer;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgniteCommunicationBalanceTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private boolean client;
+
+    /** */
+    private int selectors;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpCommunicationSpi commSpi = ((TcpCommunicationSpi)cfg.getCommunicationSpi());
+
+        commSpi.setSharedMemoryPort(-1);
+        commSpi.setConnectionsPerNode(connectionsPerNode());
+
+        if (selectors > 0)
+            commSpi.setSelectorsCount(selectors);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /**
+     * @return Connections per node.
+     */
+    protected int connectionsPerNode() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBalance1() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "5000");
+
+        try {
+            selectors = 4;
+
+            final int SRVS = 4;
+
+            startGridsMultiThreaded(SRVS);
+
+            client = true;
+
+            final Ignite client = startGrid(SRVS);
+
+            for (int i = 0; i < 4; i++) {
+                ClusterNode node = client.cluster().node(ignite(i).cluster().localNode().id());
+
+                client.compute(client.cluster().forNode(node)).call(new DummyCallable(null));
+            }
+
+            waitNioBalanceStop(Collections.singletonList(client), 10_000);
+
+            final GridNioServer srv = GridTestUtils.getFieldValue(client.configuration().getCommunicationSpi(), "nioSrvr");
+
+            ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+            long readMoveCnt1 = srv.readerMoveCount();
+            long writeMoveCnt1 = srv.writerMoveCount();
+
+            int prevNodeIdx = -1;
+
+            for (int iter = 0; iter < 10; iter++) {
+                int nodeIdx = rnd.nextInt(SRVS);
+
+                while (prevNodeIdx == nodeIdx)
+                    nodeIdx = rnd.nextInt(SRVS);
+
+                prevNodeIdx = nodeIdx;
+
+                log.info("Iteration [iter=" + iter + ", node=" + nodeIdx + ']');
+
+                final long readMoveCnt = readMoveCnt1;
+                final long writeMoveCnt = writeMoveCnt1;
+
+                final int nodeIdx0 = nodeIdx;
+
+                GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                    @Override public boolean apply() {
+                        byte[] data = new byte[100_000];
+
+                        for (int j = 0; j < 10; j++) {
+                            for (int i = 0; i < SRVS; i++) {
+                                ClusterNode node = client.cluster().node(ignite(i).cluster().localNode().id());
+
+                                IgniteCompute compute = client.compute(client.cluster().forNode(node));
+
+                                compute.call(new DummyCallable(i == nodeIdx0 ? data : null));
+                            }
+                        }
+
+                        return srv.readerMoveCount() > readMoveCnt && srv.writerMoveCount() > writeMoveCnt;
+                    }
+                }, 30_000);
+
+                waitNioBalanceStop(Collections.singletonList(client), 30_000);
+
+                long readMoveCnt2 = srv.readerMoveCount();
+                long writeMoveCnt2 = srv.writerMoveCount();
+
+                log.info("Move counts [rc1=" + readMoveCnt1 +
+                    ", wc1=" + writeMoveCnt1 +
+                    ", rc2=" + readMoveCnt2 +
+                    ", wc2=" + writeMoveCnt2 + ']');
+
+                assertTrue(readMoveCnt2 > readMoveCnt1);
+                assertTrue(writeMoveCnt2 > writeMoveCnt1);
+
+                readMoveCnt1 = readMoveCnt2;
+                writeMoveCnt1 = writeMoveCnt2;
+            }
+
+            waitNioBalanceStop(G.allGrids(), 10_000);
+        }
+        finally {
+            System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "");
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBalance2() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "1000");
+
+        try {
+            startGridsMultiThreaded(5);
+
+            client = true;
+
+            startGridsMultiThreaded(5, 5);
+
+            for (int i = 0; i < 5; i++) {
+                log.info("Iteration: " + i);
+
+                final AtomicInteger idx = new AtomicInteger();
+
+                GridTestUtils.runMultiThreaded(new Callable<Void>() {
+                    @Override public Void call() throws Exception {
+                        Ignite node = ignite(idx.incrementAndGet() % 10);
+
+                        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                        int msgs = rnd.nextInt(500, 600);
+
+                        for (int i = 0; i < msgs; i++) {
+                            int sndTo = rnd.nextInt(10);
+
+                            ClusterNode sntToNode = node.cluster().node(ignite(sndTo).cluster().localNode().id());
+
+                            IgniteCompute compute = node.compute(node.cluster().forNode(sntToNode));
+
+                            compute.call(new DummyCallable(new byte[rnd.nextInt(rnd.nextInt(256, 1024))]));
+                        }
+
+                        return null;
+                    }
+                }, 30, "test-thread");
+
+                waitNioBalanceStop(G.allGrids(), 10_000);
+            }
+        }
+        finally {
+            System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "");
+        }
+    }
+
+    /**
+     * @param nodes Node.
+     * @param timeout Timeout.
+     * @throws Exception If failed.
+     */
+    private void waitNioBalanceStop(List<Ignite> nodes, long timeout) throws Exception {
+        final List<GridNioServer> srvs = new ArrayList<>();
+
+        for (Ignite node : nodes) {
+            TcpCommunicationSpi spi = (TcpCommunicationSpi) node.configuration().getCommunicationSpi();
+
+            GridNioServer srv = GridTestUtils.getFieldValue(spi, "nioSrvr");
+
+            srvs.add(srv);
+        }
+
+        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicateX() {
+            @Override public boolean applyx() throws IgniteCheckedException {
+                List<Long> rCnts = new ArrayList<>();
+                List<Long> wCnts = new ArrayList<>();
+
+                for (GridNioServer srv : srvs) {
+                    long readerMovCnt1 = srv.readerMoveCount();
+                    long writerMovCnt1 = srv.writerMoveCount();
+
+                    rCnts.add(readerMovCnt1);
+                    wCnts.add(writerMovCnt1);
+                }
+
+                U.sleep(2000);
+
+                for (int i = 0; i < srvs.size(); i++) {
+                    GridNioServer srv = srvs.get(i);
+
+                    long readerMovCnt1 = rCnts.get(i);
+                    long writerMovCnt1 = wCnts.get(i);
+
+                    long readerMovCnt2 = srv.readerMoveCount();
+                    long writerMovCnt2 = srv.writerMoveCount();
+
+                    if (readerMovCnt1 != readerMovCnt2) {
+                        log.info("Readers balance is in progress [node=" + i + ", cnt1=" + readerMovCnt1 +
+                            ", cnt2=" + readerMovCnt2 + ']');
+
+                        return false;
+                    }
+                    if (writerMovCnt1 != writerMovCnt2) {
+                        log.info("Writers balance is in progress [node=" + i + ", cnt1=" + writerMovCnt1 +
+                            ", cnt2=" + writerMovCnt2 + ']');
+
+                        return false;
+                    }
+                }
+
+                return true;
+            }
+        }, timeout));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRandomBalance() throws Exception {
+        System.setProperty(GridNioServer.IGNITE_IO_BALANCE_RANDOM_BALANCE, "true");
+        System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "500");
+
+        try {
+            final int NODES = 10;
+
+            startGridsMultiThreaded(NODES);
+
+            final long stopTime = System.currentTimeMillis() + 60_000;
+
+            GridTestUtils.runMultiThreaded(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                    while (System.currentTimeMillis() < stopTime)
+                        ignite(rnd.nextInt(NODES)).compute().broadcast(new DummyCallable(null));
+
+                    return null;
+                }
+            }, 20, "test-thread");
+        }
+        finally {
+            System.setProperty(GridNioServer.IGNITE_IO_BALANCE_RANDOM_BALANCE, "");
+            System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "");
+        }
+    }
+
+    /**
+     *
+     */
+    private static class DummyCallable implements IgniteCallable<Object> {
+        /** */
+        private byte[] data;
+
+        /**
+         * @param data Data.
+         */
+        DummyCallable(byte[] data) {
+            this.data = data;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object call() throws Exception {
+            return data;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessagesTest.java
new file mode 100644
index 0000000000000..b6448786a1cf8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessagesTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgniteIoTestMessagesTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrids(3);
+
+        client = true;
+
+        startGrid(3);
+
+        startGrid(4);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        super.afterTestsStopped();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testIoTestMessages() throws Exception {
+        for (Ignite node : G.allGrids()) {
+            IgniteKernal ignite = (IgniteKernal)node;
+
+            List<ClusterNode> rmts = new ArrayList<>(ignite.cluster().forRemotes().nodes());
+
+            assertEquals(4, rmts.size());
+
+            for (ClusterNode rmt : rmts) {
+                ignite.sendIoTest(rmt, new byte[1024], false);
+
+                ignite.sendIoTest(rmt, new byte[1024], true);
+
+                ignite.sendIoTest(rmts, new byte[1024], false);
+
+                ignite.sendIoTest(rmts, new byte[1024], true);
+            }
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteVariousConnectionNumberTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteVariousConnectionNumberTest.java
new file mode 100644
index 0000000000000..510751e334f5c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteVariousConnectionNumberTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class IgniteVariousConnectionNumberTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int NODES = 6;
+
+    /** */
+    private static Random rnd = new Random();
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        int connections = rnd.nextInt(10) + 1;
+
+        log.info("Node connections [name=" + gridName + ", connections=" + connections + ']');
+
+        ((TcpCommunicationSpi)cfg.getCommunicationSpi()).setConnectionsPerNode(connections);
+        ((TcpCommunicationSpi)cfg.getCommunicationSpi()).setUsePairedConnections(rnd.nextBoolean());
+        ((TcpCommunicationSpi)cfg.getCommunicationSpi()).setSharedMemoryPort(-1);
+
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        long seed = U.currentTimeMillis();
+
+        rnd.setSeed(seed);
+
+        log.info("Random seed: " + seed);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        super.afterTestsStopped();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testVariousConnectionNumber() throws Exception {
+        startGridsMultiThreaded(3);
+
+        client = true;
+
+        startGridsMultiThreaded(3, 3);
+
+        CacheConfiguration ccfg = new CacheConfiguration();
+
+        ccfg.setCacheMode(REPLICATED);
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+        ignite(0).createCache(ccfg);
+
+        for (int i = 0; i < 10; i++) {
+            log.info("Iteration: " + i);
+
+            runOperations(5000);
+
+            awaitPartitionMapExchange();
+
+            int idx = ThreadLocalRandom.current().nextInt(NODES);
+
+            Ignite node = ignite(idx);
+
+            client = node.configuration().isClientMode();
+
+            stopGrid(idx);
+
+            startGrid(idx);
+        }
+    }
+
+    /**
+     * @param time Execution time.
+     * @throws Exception If failed.
+     */
+    private void runOperations(final long time) throws Exception {
+        final AtomicInteger idx = new AtomicInteger();
+
+        GridTestUtils.runMultiThreaded(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                Ignite node = ignite(idx.getAndIncrement() % NODES);
+
+                IgniteCache cache = node.cache(null);
+
+                long stopTime = U.currentTimeMillis() + time;
+
+                ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                while (U.currentTimeMillis() < stopTime) {
+                    cache.put(rnd.nextInt(10_000), 0);
+
+                    node.compute().broadcast(new DummyJob());
+                }
+
+                return null;
+            }
+        }, NODES * 10, "test-thread");
+    }
+
+    /**
+     *
+     */
+    private static class DummyJob implements IgniteRunnable {
+        /** {@inheritDoc} */
+        @Override public void run() {
+            // No-op.
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java
index 67ec3712e92b4..eaa9923cb4723 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java
@@ -85,6 +85,11 @@ public class CrossCacheTxRandomOperationsTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 6 * 60 * 1000;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
@@ -170,9 +175,17 @@ protected CacheConfiguration cacheConfiguration(String name,
     }
 
     /**
+     * @param cacheMode Cache mode.
+     * @param writeSync Write synchronization mode.
+     * @param fairAff Fair affinity flag.
+     * @param ignite Node to use.
+     * @param name Cache name.
      */
-    protected void createCache(CacheMode cacheMode, CacheWriteSynchronizationMode writeSync, boolean fairAff,
-        Ignite ignite, String name) {
+    protected void createCache(CacheMode cacheMode,
+        CacheWriteSynchronizationMode writeSync,
+        boolean fairAff,
+        Ignite ignite,
+        String name) {
         ignite.createCache(cacheConfiguration(name, cacheMode, writeSync, fairAff));
     }
 
@@ -269,9 +282,18 @@ private void txOperations(TransactionConcurrency concurrency,
 
             boolean checkData = fullSync && !optimistic;
 
+            long stopTime = System.currentTimeMillis() + 10_000;
+
             for (int i = 0; i < 10_000; i++) {
-                if (i % 100 == 0)
+                if (i % 100 == 0) {
+                    if (System.currentTimeMillis() > stopTime) {
+                        log.info("Stop on timeout, iteration: " + i);
+
+                        break;
+                    }
+
                     log.info("Iteration: " + i);
+                }
 
                 boolean rollback = i % 10 == 0;
 
@@ -557,4 +579,4 @@ public TestEntryProcessor(@Nullable Long val) {
             return old;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
index 9405a190e58f9..3a2bc81643840 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
@@ -200,7 +200,9 @@ public void testGetAndPut() throws Exception {
     private void testRebalance(final Operation operation) throws Exception {
         interceptor = new RebalanceUpdateInterceptor();
 
-        for (int iter = 0; iter < TEST_ITERATIONS; iter++) {
+        long stopTime = System.currentTimeMillis() + 2 * 60_000;
+
+        for (int iter = 0; iter < TEST_ITERATIONS && System.currentTimeMillis() < stopTime; iter++) {
             log.info("Iteration: " + iter);
 
             failed = false;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffHeapMultiThreadedUpdateSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffHeapMultiThreadedUpdateSelfTest.java
index 9458a63e83fa7..6e2e91f58f493 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffHeapMultiThreadedUpdateSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffHeapMultiThreadedUpdateSelfTest.java
@@ -115,10 +115,10 @@ public void testPutTxPessimistic() throws Exception {
         if (gridCount() > 1)
             testPutTx(keyForNode(1), PESSIMISTIC);
     }
-    
+
     /**
      * TODO: IGNITE-592.
-     *  
+     *
      * @throws Exception If failed.
      */
     public void testPutTxOptimistic() throws Exception {
@@ -227,4 +227,4 @@ private void testPutxIfAbsentTx(final Integer key, final TransactionConcurrency
 
         assertFalse(failed);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecovery10ConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecovery10ConnectionsTest.java
new file mode 100644
index 0000000000000..30fc9ef0e5a9d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecovery10ConnectionsTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+/**
+ *
+ */
+public class IgniteCacheAtomicMessageRecovery10ConnectionsTest extends IgniteCacheAtomicMessageRecoveryTest {
+    /** {@inheritDoc} */
+    @Override protected int connectionsPerNode() {
+        return 10;
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java
new file mode 100644
index 0000000000000..71772efeadc99
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+
+/**
+ *
+ */
+public class IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest extends IgniteCacheAtomicMessageRecoveryTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpCommunicationSpi commSpi = (TcpCommunicationSpi)cfg.getCommunicationSpi();
+
+        assertTrue(commSpi.isUsePairedConnections());
+
+        commSpi.setUsePairedConnections(false);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return ATOMIC;
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecovery10ConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecovery10ConnectionsTest.java
new file mode 100644
index 0000000000000..919aea69c902b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecovery10ConnectionsTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+
+/**
+ *
+ */
+public class IgniteCacheConnectionRecovery10ConnectionsTest extends IgniteCacheConnectionRecoveryTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpCommunicationSpi)cfg.getCommunicationSpi()).setConnectionsPerNode(10);
+
+        return cfg;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java
index 2f700f31f45fc..a91de67e52f74 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java
@@ -107,7 +107,7 @@ public void testStartNodes() throws Exception {
         try {
             int iter = 0;
 
-            while (System.currentTimeMillis() < stopTime) {
+            while (System.currentTimeMillis() < stopTime && iter < 5) {
                 log.info("Iteration: " + iter++);
 
                 try {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryAbstractTest.java
index 0460a8ff6b9f0..1bfd7275cd4f7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryAbstractTest.java
@@ -58,6 +58,7 @@ public abstract class IgniteCacheMessageRecoveryAbstractTest extends GridCommonA
 
         commSpi.setSocketWriteTimeout(1000);
         commSpi.setSharedMemoryPort(-1);
+        commSpi.setConnectionsPerNode(connectionsPerNode());
 
         cfg.setCommunicationSpi(commSpi);
 
@@ -75,6 +76,13 @@ public abstract class IgniteCacheMessageRecoveryAbstractTest extends GridCommonA
         return cfg;
     }
 
+    /**
+     * @return Value for {@link TcpCommunicationSpi#setConnectionsPerNode(int)}.
+     */
+    protected int connectionsPerNode() {
+        return TcpCommunicationSpi.DFLT_CONN_PER_NODE;
+    }
+
     /**
      * @return Cache atomicity mode.
      */
@@ -174,18 +182,22 @@ public void testMessageRecovery() throws Exception {
     static boolean closeSessions(Ignite ignite) throws Exception {
         TcpCommunicationSpi commSpi = (TcpCommunicationSpi)ignite.configuration().getCommunicationSpi();
 
-        Map<UUID, GridCommunicationClient> clients = U.field(commSpi, "clients");
+        Map<UUID, GridCommunicationClient[]> clients = U.field(commSpi, "clients");
 
         boolean closed = false;
 
-        for (GridCommunicationClient client : clients.values()) {
-            GridTcpNioCommunicationClient client0 = (GridTcpNioCommunicationClient)client;
+        for (GridCommunicationClient[] clients0 : clients.values()) {
+            for (GridCommunicationClient client : clients0) {
+                if (client != null) {
+                    GridTcpNioCommunicationClient client0 = (GridTcpNioCommunicationClient)client;
 
-            GridNioSession ses = client0.session();
+                    GridNioSession ses = client0.session();
 
-            ses.close();
+                    ses.close();
 
-            closed = true;
+                    closed = true;
+                }
+            }
         }
 
         return closed;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index 6256225f0a54e..0dd4079733081 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -50,8 +50,8 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
         // Try provoke connection close on socket writeTimeout.
         commSpi.setSharedMemoryPort(-1);
         commSpi.setMessageQueueLimit(10);
-        commSpi.setSocketReceiveBuffer(32);
-        commSpi.setSocketSendBuffer(32);
+        commSpi.setSocketReceiveBuffer(40);
+        commSpi.setSocketSendBuffer(40);
         commSpi.setSocketWriteTimeout(100);
         commSpi.setUnacknowledgedMessagesBufferSize(1000);
         commSpi.setConnectTimeout(10_000);
@@ -66,15 +66,20 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
         super.afterTest();
     }
 
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 10 * 60_000;
+    }
+
     /**
      * @throws Exception If failed.
      */
     public void testMessageQueueLimit() throws Exception {
-        startGridsMultiThreaded(3);
-
-        for (int i = 0; i < 15; i++) {
+        for (int i = 0; i < 3; i++) {
             log.info("Iteration: " + i);
 
+            startGridsMultiThreaded(3);
+
             IgniteInternalFuture<?> fut1 = startJobThreads(50);
 
             U.sleep(100);
@@ -83,6 +88,8 @@ public void testMessageQueueLimit() throws Exception {
 
             fut1.get();
             fut2.get();
+
+            stopAllGrids();
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java
index 3fca826c7d0c5..322690c8cd16e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java
@@ -86,7 +86,6 @@ public class IgniteCacheMultiTxLockSelfTest extends GridCommonAbstractTest {
         plc.setMaxSize(100000);
 
         ccfg.setEvictionPolicy(plc);
-        ccfg.setEvictSynchronized(true);
 
         c.setCacheConfiguration(ccfg);
 
@@ -95,6 +94,11 @@ public class IgniteCacheMultiTxLockSelfTest extends GridCommonAbstractTest {
         return c;
     }
 
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 60_000;
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest.java
new file mode 100644
index 0000000000000..e8175e5ced12d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest extends
+    GridCacheAtomicPrimaryWriteOrderMultiNodeFullApiSelfTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setStripedPoolSize(-1);
+
+        return cfg;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest.java
new file mode 100644
index 0000000000000..05fe85f8f539e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest extends
+    GridCachePartitionedMultiNodeFullApiSelfTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setStripedPoolSize(-1);
+
+        return cfg;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionNoHangsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionNoHangsTest.java
index c9d18ebdf95df..e9d74ffa6537b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionNoHangsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionNoHangsTest.java
@@ -211,7 +211,7 @@ private void doTest(final TransactionConcurrency concurrency) throws IgniteCheck
                             tx.commit();
                         }
                         catch (Exception e) {
-                            e.printStackTrace();
+                            log.info("Ignore error: " + e);
                         }
                     }
                 }, NODES_CNT * 3, "tx-thread");
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
index aa240aae439a9..f6a06c29a9fbe 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
@@ -111,6 +111,9 @@ public class TxOptimisticDeadlockDetectionTest extends GridCommonAbstractTest {
 
         cfg.setClientMode(client);
 
+        // Test spi blocks message send, this can cause hang with striped pool.
+        cfg.setStripedPoolSize(-1);
+
         return cfg;
     }
 
@@ -274,8 +277,8 @@ private void doTestDeadlock(
 
                     Object k;
 
-                    log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
-                        ", tx=" + tx + ", key=" + transformer.apply(key) + ']');
+                    log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode().id() +
+                        ", tx=" + tx.xid() + ", key=" + transformer.apply(key) + ']');
 
                     cache.put(transformer.apply(key), 0);
 
@@ -309,23 +312,27 @@ private void doTestDeadlock(
                         entries.put(k, 2);
                     }
 
-                    log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
-                        ", tx=" + tx + ", entries=" + entries + ']');
+                    log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode().id() +
+                        ", tx=" + tx.xid() + ", entries=" + entries + ']');
 
                     cache.putAll(entries);
 
                     tx.commit();
                 }
                 catch (Throwable e) {
-                    U.error(log, "Expected exception: ", e);
+                    log.info("Expected exception: " + e);
+
+                    e.printStackTrace(System.out);
 
                     // At least one stack trace should contain TransactionDeadlockException.
                     if (hasCause(e, TransactionTimeoutException.class) &&
-                        hasCause(e, TransactionDeadlockException.class)
-                        ) {
-                        if (deadlockErr.compareAndSet(null, cause(e, TransactionDeadlockException.class)))
-                            U.error(log, "At least one stack trace should contain " +
-                                TransactionDeadlockException.class.getSimpleName(), e);
+                        hasCause(e, TransactionDeadlockException.class)) {
+                        if (deadlockErr.compareAndSet(null, cause(e, TransactionDeadlockException.class))) {
+                            log.info("At least one stack trace should contain " +
+                                TransactionDeadlockException.class.getSimpleName());
+
+                            e.printStackTrace(System.out);
+                        }
                     }
                 }
             }
@@ -344,7 +351,7 @@ private void doTestDeadlock(
 
         TransactionDeadlockException deadlockE = deadlockErr.get();
 
-        assertNotNull(deadlockE);
+        assertNotNull("Failed to detect deadlock", deadlockE);
 
         boolean fail = false;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
index 6fc7e02e42946..7b5abf5d2f90d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
@@ -372,4 +372,4 @@ protected static class MapServiceImpl<K, V> implements MapService<K, V>, Service
             X.println("Executing cache service: " + ctx.name());
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
index adcd144966318..4bc9f01b5fb84 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
@@ -19,6 +19,7 @@
 
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
@@ -227,87 +228,98 @@ public void testListenNotify() throws Exception {
      *
      * @throws Exception In case of any exception.
      */
-    @SuppressWarnings("ErrorNotRethrown")
     public void testChaining() throws Exception {
+        checkChaining(null);
+
+        ExecutorService exec = Executors.newFixedThreadPool(1);
+
+        try {
+            checkChaining(exec);
+
+            GridFinishedFuture<Integer> fut = new GridFinishedFuture<>(1);
+
+            IgniteInternalFuture<Object> chain = fut.chain(new CX1<IgniteInternalFuture<Integer>, Object>() {
+                @Override public Object applyx(IgniteInternalFuture<Integer> fut) throws IgniteCheckedException {
+                    return fut.get() + 1;
+                }
+            }, exec);
+
+            assertEquals(2, chain.get());
+        }
+        finally {
+            exec.shutdown();
+        }
+    }
+
+    /**
+     * @param exec Executor for chain callback.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("ErrorNotRethrown")
+    private void checkChaining(ExecutorService exec) throws Exception {
         final CX1<IgniteInternalFuture<Object>, Object> passThrough = new CX1<IgniteInternalFuture<Object>, Object>() {
             @Override public Object applyx(IgniteInternalFuture<Object> f) throws IgniteCheckedException {
                 return f.get();
             }
         };
 
-        final GridTestKernalContext ctx = new GridTestKernalContext(log);
-
-        ctx.setExecutorService(Executors.newFixedThreadPool(1));
-        ctx.setSystemExecutorService(Executors.newFixedThreadPool(1));
-
-        ctx.add(new PoolProcessor(ctx));
-        ctx.add(new GridClosureProcessor(ctx));
+        GridFutureAdapter<Object> fut = new GridFutureAdapter<>();
+        IgniteInternalFuture<Object> chain = exec != null ? fut.chain(passThrough, exec) : fut.chain(passThrough);
 
-        ctx.start();
+        assertFalse(fut.isDone());
+        assertFalse(chain.isDone());
 
         try {
-            // Test result returned.
-
-            GridFutureAdapter<Object> fut = new GridFutureAdapter<>();
-            IgniteInternalFuture<Object> chain = fut.chain(passThrough);
+            chain.get(20);
 
-            assertFalse(fut.isDone());
-            assertFalse(chain.isDone());
-
-            try {
-                chain.get(20);
-
-                fail("Expects timeout exception.");
-            }
-            catch (IgniteFutureTimeoutCheckedException e) {
-                info("Expected timeout exception: " + e.getMessage());
-            }
+            fail("Expects timeout exception.");
+        }
+        catch (IgniteFutureTimeoutCheckedException e) {
+            info("Expected timeout exception: " + e.getMessage());
+        }
 
-            fut.onDone("result");
+        fut.onDone("result");
 
-            assertEquals("result", chain.get(1));
+        assertEquals("result", chain.get(1));
 
-            // Test exception re-thrown.
+        // Test exception re-thrown.
 
-            fut = new GridFutureAdapter<>();
-            chain = fut.chain(passThrough);
+        fut = new GridFutureAdapter<>();
+        chain = exec != null ? fut.chain(passThrough, exec) : fut.chain(passThrough);
 
-            fut.onDone(new ClusterGroupEmptyCheckedException("test exception"));
+        fut.onDone(new ClusterGroupEmptyCheckedException("test exception"));
 
-            try {
-                chain.get();
+        try {
+            chain.get();
 
-                fail("Expects failed with exception.");
-            }
-            catch (ClusterGroupEmptyCheckedException e) {
-                info("Expected exception: " + e.getMessage());
-            }
+            fail("Expects failed with exception.");
+        }
+        catch (ClusterGroupEmptyCheckedException e) {
+            info("Expected exception: " + e.getMessage());
+        }
 
-            // Test error re-thrown.
+        // Test error re-thrown.
 
-            fut = new GridFutureAdapter<>();
-            chain = fut.chain(passThrough);
+        fut = new GridFutureAdapter<>();
+        chain = exec != null ? fut.chain(passThrough, exec) : fut.chain(passThrough);
 
-            try {
-                fut.onDone(new StackOverflowError("test error"));
+        try {
+            fut.onDone(new StackOverflowError("test error"));
 
+            if (exec == null)
                 fail("Expects failed with error.");
-            }
-            catch (StackOverflowError e) {
-                info("Expected error: " + e.getMessage());
-            }
+        }
+        catch (StackOverflowError e) {
+            info("Expected error: " + e.getMessage());
+        }
 
-            try {
-                chain.get();
+        try {
+            chain.get();
 
-                fail("Expects failed with error.");
-            }
-            catch (StackOverflowError e) {
-                info("Expected error: " + e.getMessage());
-            }
+            fail("Expects failed with error.");
         }
-        finally {
-            ctx.stop(false);
+        catch (StackOverflowError e) {
+            info("Expected error: " + e.getMessage());
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/nio/impl/GridNioFilterChainSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/nio/impl/GridNioFilterChainSelfTest.java
index 201fd2741201f..d403784b0c2ad 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/nio/impl/GridNioFilterChainSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/nio/impl/GridNioFilterChainSelfTest.java
@@ -114,7 +114,7 @@ public void testChainEvents() throws Exception {
                 proceedExceptionCaught(ses, ex);
             }
 
-            @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) {
+            @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) {
                 sndEvt.compareAndSet(null, ses.<String>meta(MESSAGE_WRITE_META_NAME));
 
                 sndMsgObj.compareAndSet(null, msg);
@@ -155,7 +155,7 @@ public void testChainEvents() throws Exception {
         chain.onSessionIdleTimeout(ses);
         chain.onSessionWriteTimeout(ses);
         assertNull(chain.onSessionClose(ses));
-        assertNull(chain.onSessionWrite(ses, snd));
+        assertNull(chain.onSessionWrite(ses, snd, true));
 
         assertEquals("DCBA", connectedEvt.get());
         assertEquals("DCBA", disconnectedEvt.get());
@@ -210,10 +210,10 @@ private AppendingFilter(String param) {
         }
 
         /** {@inheritDoc} */
-        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) throws IgniteCheckedException {
             chainMeta(ses, MESSAGE_WRITE_META_NAME);
 
-            return proceedSessionWrite(ses, msg);
+            return proceedSessionWrite(ses, msg, fut);
         }
 
         /** {@inheritDoc} */
@@ -348,6 +348,11 @@ public MockNioSession(InetSocketAddress locAddr, InetSocketAddress rmtAddr) {
             return new GridNioFinishedFuture<>(true);
         }
 
+        /** {@inheritDoc} */
+        @Override public void sendNoFuture(Object msg) {
+            // No-op.
+        }
+
         /** {@inheritDoc} */
         @Override public GridNioFuture<Object> resumeReads() {
             return null;
@@ -369,13 +374,28 @@ public MockNioSession(InetSocketAddress locAddr, InetSocketAddress rmtAddr) {
         }
 
         /** {@inheritDoc} */
-        @Override public void recoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+        @Override public void outRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public GridNioRecoveryDescriptor outRecoveryDescriptor() {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void inRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
             // No-op.
         }
 
         /** {@inheritDoc} */
-        @Nullable @Override public GridNioRecoveryDescriptor recoveryDescriptor() {
+        @Nullable @Override public GridNioRecoveryDescriptor inRecoveryDescriptor() {
             return null;
         }
+
+        /** {@inheritDoc} */
+        @Override public void systemMessage(Object msg) {
+            // No-op.
+        }
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/nio/GridNioBenchmarkClient.java b/modules/core/src/test/java/org/apache/ignite/loadtests/nio/GridNioBenchmarkClient.java
index 61a13b1f27e6f..25dd7801c558a 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/nio/GridNioBenchmarkClient.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/nio/GridNioBenchmarkClient.java
@@ -83,7 +83,7 @@ public GridNioBenchmarkClient(int connCnt, String host, int port) {
      */
     public void run() throws IOException, InterruptedException {
         for (int i = 0; i < connCnt; i++)
-            exec.submit(new ClientThread());
+            exec.execute(new ClientThread());
 
         Thread.sleep(5*60*1000);
 
@@ -167,4 +167,4 @@ private long doIteration(InputStream in, OutputStream out) throws IOException {
             return read;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PRecursionTaskSelfTest.java b/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PRecursionTaskSelfTest.java
index f21f31bc16a31..a18ef32aa4b59 100644
--- a/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PRecursionTaskSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PRecursionTaskSelfTest.java
@@ -196,4 +196,4 @@ private static class FactorialJob extends ComputeJobAdapter {
             return ignite.compute().execute(FactorialTask.class, arg);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
index 652e47f0841db..5ca8f26500bcf 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
@@ -69,7 +69,7 @@ public class GridTcpSpiForwardingSelfTest extends GridCommonAbstractTest {
     private static final int commExtPort2 = 20100;
 
     /** */
-    private AddressResolver resolver;
+    private AddressResolver rslvr;
 
     /** */
     private boolean ipFinderUseLocPorts;
@@ -111,14 +111,15 @@ else if (getTestGridName(1).equals(gridName)) {
         cfg.setConnectorConfiguration(null);
 
         TcpCommunicationSpi commSpi = new TcpCommunicationSpi() {
-            @Override protected GridCommunicationClient createTcpClient(ClusterNode node) throws IgniteCheckedException {
+            @Override protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
+                throws IgniteCheckedException {
                 Map<String, Object> attrs = new HashMap<>(node.attributes());
 
                 attrs.remove(createSpiAttributeName(ATTR_PORT));
 
                 ((TcpDiscoveryNode)node).setAttributes(attrs);
 
-                return super.createTcpClient(node);
+                return super.createTcpClient(node, connIdx);
             }
         };
 
@@ -126,12 +127,13 @@ else if (getTestGridName(1).equals(gridName)) {
         commSpi.setLocalPort(commLocPort);
         commSpi.setLocalPortRange(1);
         commSpi.setSharedMemoryPort(-1);
+        commSpi.setConnectionsPerNode(1);
 
         cfg.setCommunicationSpi(commSpi);
 
-        assert resolver != null;
+        assert rslvr != null;
 
-        cfg.setAddressResolver(resolver);
+        cfg.setAddressResolver(rslvr);
 
         return cfg;
     }
@@ -147,7 +149,7 @@ public void testCustomResolver() throws Exception {
         map.put(new InetSocketAddress("127.0.0.1", locPort2), F.asList(new InetSocketAddress("127.0.0.1", extPort2)));
         map.put(new InetSocketAddress("127.0.0.1", commLocPort2), F.asList(new InetSocketAddress("127.0.0.1", commExtPort2)));
 
-        resolver = new AddressResolver() {
+        rslvr = new AddressResolver() {
             @Override public Collection<InetSocketAddress> getExternalAddresses(InetSocketAddress addr) {
                 return map.get(addr);
             }
@@ -167,7 +169,7 @@ public void testBasicResolverMapPorts() throws Exception {
         map.put("127.0.0.1:" + locPort2, "127.0.0.1:" + extPort2);
         map.put("127.0.0.1:" + commLocPort2, "127.0.0.1:" + commExtPort2);
 
-        resolver = new BasicAddressResolver(map);
+        rslvr = new BasicAddressResolver(map);
 
         doTestForward();
     }
@@ -180,7 +182,7 @@ public void testBasicResolverMapAddress() throws Exception {
 
         map.put("127.0.0.1", "127.0.0.1");
 
-        resolver = new BasicAddressResolver(map);
+        rslvr = new BasicAddressResolver(map);
 
         ipFinderUseLocPorts = true;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
index 076724dd67d43..3c4fea0453282 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
@@ -90,16 +90,36 @@ protected GridTcpCommunicationSpiAbstractTest(boolean useShmem) {
         super.afterTest();
 
         for (CommunicationSpi spi : spis.values()) {
-            ConcurrentMap<UUID, GridCommunicationClient> clients = U.field(spi, "clients");
+            ConcurrentMap<UUID, GridCommunicationClient[]> clients = U.field(spi, "clients");
+
+            for (int i = 0; i < 20; i++) {
+                GridCommunicationClient client0 = null;
+
+                for (GridCommunicationClient[] clients0 : clients.values()) {
+                    for (GridCommunicationClient client : clients0) {
+                        if (client != null) {
+                            client0 = client;
+
+                            break;
+                        }
+                    }
+
+                    if (client0 != null)
+                        break;
+                }
+
+                if (client0 == null)
+                    return;
 
-            for (int i = 0; i < 20 && !clients.isEmpty(); i++) {
                 info("Check failed for SPI [grid=" +
-                    GridTestUtils.getFieldValue(spi, IgniteSpiAdapter.class, "gridName") + ", spi=" + spi + ']');
+                    GridTestUtils.getFieldValue(spi, IgniteSpiAdapter.class, "gridName") +
+                    ", client=" + client0 +
+                    ", spi=" + spi + ']');
 
                 U.sleep(1000);
             }
 
-            assert clients.isEmpty() : "Clients: " + clients;
+            fail("Failed to wait when clients are closed.");
         }
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
index 8635d94e2bc8b..a649130fb14af 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
@@ -31,6 +31,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -83,6 +84,12 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
     /** Use ssl. */
     protected boolean useSsl;
 
+    /** */
+    private int connectionsPerNode = 1;
+
+    /** */
+    private boolean pairedConnections = true;
+
     /**
      *
      */
@@ -160,6 +167,34 @@ public void testMultithreaded() throws Exception {
         concurrentConnect(threads, 10, ITERS, false, false);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMultithreaded_10Connections() throws Exception {
+        connectionsPerNode = 10;
+
+        testMultithreaded();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMultithreaded_NoPairedConnections() throws Exception {
+        pairedConnections = false;
+
+        testMultithreaded();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMultithreaded_10ConnectionsNoPaired() throws Exception {
+        pairedConnections = false;
+        connectionsPerNode = 10;
+
+        testMultithreaded();
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -244,7 +279,7 @@ private void concurrentConnect(final int threads,
                 final AtomicInteger idx = new AtomicInteger();
 
                 try {
-                    GridTestUtils.runMultiThreaded(new Callable<Void>() {
+                    final Callable<Void> c = new Callable<Void>() {
                         @Override public Void call() throws Exception {
                             int idx0 = idx.getAndIncrement();
 
@@ -270,7 +305,40 @@ private void concurrentConnect(final int threads,
 
                             return null;
                         }
-                    }, threads, "test");
+                    };
+
+                    List<Thread> threadsList = new ArrayList<>();
+
+                    final AtomicBoolean fail = new AtomicBoolean();
+
+                    final AtomicLong tId = new AtomicLong();
+
+                    for (int t = 0; t < threads; t++) {
+                        Thread t0 = new Thread(new Runnable() {
+                            @Override public void run() {
+                                try {
+                                    c.call();
+                                }
+                                catch (Throwable e) {
+                                    log.error("Unexpected error: " + e, e);
+
+                                    fail.set(true);
+                                }
+                            }
+                        }) {
+                            @Override public long getId() {
+                                // Override getId to use all connections.
+                                return tId.getAndIncrement();
+                            }
+                        };
+
+                        threadsList.add(t0);
+
+                        t0.start();
+                    }
+
+                    for (Thread t0 : threadsList)
+                        t0.join();
 
                     assertTrue(latch.await(10, TimeUnit.SECONDS));
 
@@ -281,17 +349,19 @@ private void concurrentConnect(final int threads,
 
                         final GridNioServer srv = U.field(spi, "nioSrvr");
 
+                        final int conns = pairedConnections ? 2 : 1;
+
                         GridTestUtils.waitForCondition(new GridAbsPredicate() {
                             @Override public boolean apply() {
                                 Collection sessions = U.field(srv, "sessions");
 
-                                return sessions.size() == 1;
+                                return sessions.size() == conns * connectionsPerNode;
                             }
                         }, 5000);
 
                         Collection sessions = U.field(srv, "sessions");
 
-                        assertEquals(1, sessions.size());
+                        assertEquals(conns * connectionsPerNode, sessions.size());
                     }
 
                     assertEquals(expMsgs, lsnr.cntr.get());
@@ -320,6 +390,8 @@ private CommunicationSpi createSpi() {
         spi.setIdleConnectionTimeout(60_000);
         spi.setConnectTimeout(10_000);
         spi.setSharedMemoryPort(-1);
+        spi.setConnectionsPerNode(connectionsPerNode);
+        spi.setUsePairedConnections(pairedConnections);
 
         return spi;
     }
@@ -434,4 +506,4 @@ private void stopSpis() throws Exception {
             rsrcs.stopThreads();
     }
 
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConfigSelfTest.java
index b0353a6bd889b..5345a9b5cf7ad 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConfigSelfTest.java
@@ -18,8 +18,6 @@
 package org.apache.ignite.spi.communication.tcp;
 
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.spi.GridSpiAbstractConfigTest;
 import org.apache.ignite.testframework.junits.spi.GridSpiTest;
 
@@ -47,6 +45,9 @@ public void testNegativeConfig() throws Exception {
         checkNegativeSpiProperty(new TcpCommunicationSpi(), "ackSendThreshold", 0);
         checkNegativeSpiProperty(new TcpCommunicationSpi(), "ackSendThreshold", -1);
         checkNegativeSpiProperty(new TcpCommunicationSpi(), "unacknowledgedMessagesBufferSize", -1);
+        checkNegativeSpiProperty(new TcpCommunicationSpi(), "connectionsPerNode", 0);
+        checkNegativeSpiProperty(new TcpCommunicationSpi(), "connectionsPerNode", -1);
+        checkNegativeSpiProperty(new TcpCommunicationSpi(), "connectionsPerNode", Integer.MAX_VALUE);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiMultithreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiMultithreadedSelfTest.java
index a74a07a884536..0c57105f167c9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiMultithreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiMultithreadedSelfTest.java
@@ -109,7 +109,7 @@ public class GridTcpCommunicationSpiMultithreadedSelfTest extends GridSpiAbstrac
     /**
      * @param useShmem Use shared mem.
      */
-    protected GridTcpCommunicationSpiMultithreadedSelfTest(boolean useShmem) {
+    GridTcpCommunicationSpiMultithreadedSelfTest(boolean useShmem) {
         super(false);
 
         this.useShmem = useShmem;
@@ -370,17 +370,17 @@ public void testFlowSend() throws Exception {
             Collection<? extends GridNioSession> sessions = GridTestUtils.getFieldValue(srv, "sessions");
 
             for (GridNioSession ses : sessions) {
-                final GridNioRecoveryDescriptor snd = ses.recoveryDescriptor();
+                final GridNioRecoveryDescriptor snd = ses.outRecoveryDescriptor();
 
                 if (snd != null) {
                     GridTestUtils.waitForCondition(new GridAbsPredicate() {
                         @Override public boolean apply() {
-                            return snd.messagesFutures().isEmpty();
+                            return snd.messagesRequests().isEmpty();
                         }
                     }, 10_000);
 
-                    assertEquals("Unexpected messages: " + snd.messagesFutures(), 0,
-                        snd.messagesFutures().size());
+                    assertEquals("Unexpected messages: " + snd.messagesRequests(), 0,
+                        snd.messagesRequests().size());
                 }
             }
         }
@@ -547,11 +547,18 @@ private int getSpiCount() {
         }
 
         for (CommunicationSpi spi : spis.values()) {
-            final ConcurrentMap<UUID, GridCommunicationClient> clients = U.field(spi, "clients");
+            final ConcurrentMap<UUID, GridCommunicationClient[]> clients = U.field(spi, "clients");
 
             assert GridTestUtils.waitForCondition(new PA() {
                 @Override public boolean apply() {
-                    return clients.isEmpty();
+                    for (GridCommunicationClient[] clients0 : clients.values()) {
+                        for (GridCommunicationClient client : clients0) {
+                            if (client != null)
+                                return false;
+                        }
+                    }
+
+                    return true;
                 }
             }, getTestTimeout()) : "Clients: " + clients;
         }
@@ -583,4 +590,4 @@ private int getSpiCount() {
         spis.clear();
         nodes.clear();
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
index 34872c671cfdb..12c2edb99e41b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
@@ -173,7 +173,7 @@ private void checkAck(int ackCnt, int idleTimeout, int msgPerIter) throws Except
                     boolean found = false;
 
                     for (GridNioSession ses : sessions) {
-                        final GridNioRecoveryDescriptor recoveryDesc = ses.recoveryDescriptor();
+                        final GridNioRecoveryDescriptor recoveryDesc = ses.outRecoveryDescriptor();
 
                         if (recoveryDesc != null) {
                             found = true;
@@ -188,12 +188,12 @@ private void checkAck(int ackCnt, int idleTimeout, int msgPerIter) throws Except
 
                             GridTestUtils.waitForCondition(new GridAbsPredicate() {
                                 @Override public boolean apply() {
-                                    return recoveryDesc.messagesFutures().isEmpty();
+                                    return recoveryDesc.messagesRequests().isEmpty();
                                 }
                             }, 10_000);
 
-                            assertEquals("Unexpected messages: " + recoveryDesc.messagesFutures(), 0,
-                                recoveryDesc.messagesFutures().size());
+                            assertEquals("Unexpected messages: " + recoveryDesc.messagesRequests(), 0,
+                                recoveryDesc.messagesRequests().size());
 
                             break;
                         }
@@ -361,6 +361,7 @@ protected TcpCommunicationSpi getSpi(int ackCnt, int idleTimeout, int queueLimit
         spi.setAckSendThreshold(ackCnt);
         spi.setMessageQueueLimit(queueLimit);
         spi.setSharedMemoryPort(-1);
+        spi.setConnectionsPerNode(1);
 
         return spi;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest.java
index 95c9e40a7bf54..b1aa11902ca10 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest.java
@@ -33,6 +33,7 @@ public class GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest extends Gri
         spi.setAckSendThreshold(5);
         spi.setSocketSendBuffer(512);
         spi.setSocketReceiveBuffer(512);
+        spi.setConnectionsPerNode(1);
 
         return spi;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest.java
new file mode 100644
index 0000000000000..8e439378f4afc
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+/**
+ *
+ */
+public class GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest extends GridTcpCommunicationSpiRecoverySelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean usePairedConnections() {
+        return false;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
index 3234d74173701..065a3d7ec9e9d 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
@@ -56,6 +56,7 @@
 /**
  *
  */
+@SuppressWarnings("unchecked")
 @GridSpiTest(spi = TcpCommunicationSpi.class, group = "Communication SPI")
 public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi> extends GridSpiAbstractTest<T> {
     /** */
@@ -305,14 +306,14 @@ public void testBlockRead1() throws Exception {
                 log.info("Iteration: " + i);
 
                 try {
-                    final GridNioSession ses0 = communicationSession(spi0);
-                    final GridNioSession ses1 = communicationSession(spi1);
+                    final GridNioSession ses0 = communicationSession(spi0, false);
+                    final GridNioSession ses1 = communicationSession(spi1, true);
 
                     ses1.pauseReads().get();
 
                     IgniteInternalFuture<?> sndFut = GridTestUtils.runAsync(new Callable<Void>() {
                         @Override public Void call() throws Exception {
-                            for (int i = 0; i < 5000; i++) {
+                            for (int i = 0; i < 6000; i++) {
                                 spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
 
                                 sentCnt.incrementAndGet();
@@ -331,7 +332,12 @@ public void testBlockRead1() throws Exception {
 
                     assertTrue("Failed to wait for session close", ses0.closeTime() != 0);
 
-                    ses1.resumeReads().get();
+                    try {
+                        ses1.resumeReads().get();
+                    }
+                    catch (IgniteCheckedException ignore) {
+                        // Can fail is ses1 was closed.
+                    }
 
                     for (int j = 0; j < 100; j++) {
                         spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
@@ -415,14 +421,14 @@ public void testBlockRead2() throws Exception {
                 log.info("Iteration: " + i);
 
                 try {
-                    final GridNioSession ses0 = communicationSession(spi0);
-                    final GridNioSession ses1 = communicationSession(spi1);
+                    final GridNioSession ses0 = communicationSession(spi0, false);
+                    final GridNioSession ses1 = communicationSession(spi1, true);
 
                     ses1.pauseReads().get();
 
                     IgniteInternalFuture<?> sndFut = GridTestUtils.runAsync(new Callable<Void>() {
                         @Override public Void call() throws Exception {
-                            for (int i = 0; i < 5000; i++) {
+                            for (int i = 0; i < 6000; i++) {
                                 spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
 
                                 expCnt1.incrementAndGet();
@@ -441,12 +447,16 @@ public void testBlockRead2() throws Exception {
 
                     assertTrue("Failed to wait for session close", ses0.closeTime() != 0);
 
-                    ses1.resumeReads().get();
+                    try {
+                        ses1.resumeReads().get();
+                    }
+                    catch (IgniteCheckedException ignore) {
+                        // Can fail is ses1 was closed.
+                    }
 
                     // Wait when session is closed, then try to open new connection from node1.
                     GridTestUtils.waitForCondition(new GridAbsPredicate() {
-                        @Override
-                        public boolean apply() {
+                        @Override public boolean apply() {
                             return ses1.closeTime() != 0;
                         }
                     }, awaitForSocketWriteTimeout());
@@ -532,14 +542,14 @@ public void testBlockRead3() throws Exception {
                 log.info("Iteration: " + i);
 
                 try {
-                    final GridNioSession ses0 = communicationSession(spi0);
-                    final GridNioSession ses1 = communicationSession(spi1);
+                    final GridNioSession ses0 = communicationSession(spi0, false);
+                    final GridNioSession ses1 = communicationSession(spi1, true);
 
                     ses1.pauseReads().get();
 
                     IgniteInternalFuture<?> sndFut = GridTestUtils.runAsync(new Callable<Void>() {
                         @Override public Void call() throws Exception {
-                            for (int i = 0; i < 5000; i++) {
+                            for (int i = 0; i < 6000; i++) {
                                 spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
 
                                 sentCnt.incrementAndGet();
@@ -558,7 +568,12 @@ public void testBlockRead3() throws Exception {
 
                     assertTrue("Failed to wait for session close", ses0.closeTime() != 0);
 
-                    ses1.resumeReads().get();
+                    try {
+                        ses1.resumeReads().get();
+                    }
+                    catch (IgniteCheckedException ignore) {
+                        // Can fail is ses1 was closed.
+                    }
 
                     sndFut.get();
 
@@ -605,11 +620,12 @@ public void testBlockRead3() throws Exception {
 
     /**
      * @param spi SPI.
+     * @param in {@code True} if need find inbound session.
      * @return Session.
      * @throws Exception If failed.
      */
     @SuppressWarnings("unchecked")
-    private GridNioSession communicationSession(TcpCommunicationSpi spi) throws Exception {
+    private GridNioSession communicationSession(TcpCommunicationSpi spi, boolean in) throws Exception {
         final GridNioServer srv = U.field(spi, "nioSrvr");
 
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
@@ -622,9 +638,21 @@ private GridNioSession communicationSession(TcpCommunicationSpi spi) throws Exce
 
         Collection<? extends GridNioSession> sessions = GridTestUtils.getFieldValue(srv, "sessions");
 
-        assertEquals(1, sessions.size());
+        for (GridNioSession ses : sessions) {
+            if (in == ses.accepted())
+                return ses;
+        }
+
+        fail("Failed to find session");
 
-        return sessions.iterator().next();
+        return null;
+    }
+
+    /**
+     * @return {@code True}.
+     */
+    protected boolean usePairedConnections() {
+        return true;
     }
 
     /**
@@ -642,6 +670,8 @@ protected TcpCommunicationSpi getSpi(int idx) {
         spi.setSocketWriteTimeout(1000);
         spi.setSocketSendBuffer(512);
         spi.setSocketReceiveBuffer(512);
+        spi.setConnectionsPerNode(1);
+        spi.setUsePairedConnections(usePairedConnections());
 
         return spi;
     }
@@ -748,9 +778,8 @@ private void stopSpis() throws Exception {
             spi.spiStop();
         }
 
-        for (IgniteTestResources rsrcs : spiRsrcs) {
+        for (IgniteTestResources rsrcs : spiRsrcs)
             rsrcs.stopThreads();
-        }
 
         spis.clear();
         nodes.clear();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
index 25e3611063275..c4930a0ff6e63 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
@@ -187,7 +187,7 @@ private void checkAck(int ackCnt, int idleTimeout, int msgPerIter) throws Except
                     boolean found = false;
 
                     for (GridNioSession ses : sessions) {
-                        final GridNioRecoveryDescriptor recoveryDesc = ses.recoveryDescriptor();
+                        final GridNioRecoveryDescriptor recoveryDesc = ses.outRecoveryDescriptor();
 
                         if (recoveryDesc != null) {
                             found = true;
@@ -202,12 +202,12 @@ private void checkAck(int ackCnt, int idleTimeout, int msgPerIter) throws Except
 
                             GridTestUtils.waitForCondition(new GridAbsPredicate() {
                                 @Override public boolean apply() {
-                                    return recoveryDesc.messagesFutures().isEmpty();
+                                    return recoveryDesc.messagesRequests().isEmpty();
                                 }
                             }, 10_000);
 
-                            assertEquals("Unexpected messages: " + recoveryDesc.messagesFutures(), 0,
-                                recoveryDesc.messagesFutures().size());
+                            assertEquals("Unexpected messages: " + recoveryDesc.messagesRequests(), 0,
+                                recoveryDesc.messagesRequests().size());
 
                             break;
                         }
@@ -397,6 +397,7 @@ protected TcpCommunicationSpi getSpi(int ackCnt, int idleTimeout, int queueLimit
         spi.setAckSendThreshold(ackCnt);
         spi.setMessageQueueLimit(queueLimit);
         spi.setSharedMemoryPort(-1);
+        spi.setConnectionsPerNode(1);
 
         return spi;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
index f9e2ff4e709df..143159dd4c57f 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
@@ -63,7 +63,9 @@ public GridTestKernalContext(IgniteLogger log, IgniteConfiguration cfg) throws I
                 null,
                 null,
                 null,
-                U.allPluginProviders());
+                null,
+                U.allPluginProviders()
+        );
 
         GridTestUtils.setFieldValue(grid(), "cfg", config());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
index 14e58337b7c07..17757ab32ad7d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderFairAffinityMultiNodeFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderMultiNodeFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderMultiNodeP2PDisabledFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderOffHeapFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderOffHeapTieredFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWrityOrderOffHeapMultiNodeFullApiSelfTest;
@@ -77,6 +78,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeP2PDisabledFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNearOnlyNoPrimaryFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedOffHeapFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedOffHeapMultiNodeFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedOffHeapTieredFullApiSelfTest;
@@ -228,6 +230,10 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheReplicatedFullApiMultithreadedSelfTest.class);
         suite.addTestSuite(GridCachePartitionedFullApiMultithreadedSelfTest.class);
 
+        // Disabled striped pool.
+        suite.addTestSuite(GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest.class);
+        suite.addTestSuite(GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest.class);
+
         // Other.
         suite.addTestSuite(GridCacheClearSelfTest.class);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 435fcfb0df09d..554bb3d886080 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -39,6 +39,10 @@
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreTest;
 import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreMultithreadedSelfTest;
 import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreSelfTest;
+import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceMultipleConnectionsTest;
+import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceTest;
+import org.apache.ignite.internal.managers.communication.IgniteIoTestMessagesTest;
+import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest;
 import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest;
 import org.apache.ignite.internal.processors.cache.CacheEntryProcessorCopySelfTest;
@@ -128,7 +132,10 @@
 import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicNearUpdateTopologyChangeTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheTxNearUpdateTopologyChangeTest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheEntrySetIterationPreloadingSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecovery10ConnectionsTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecoveryTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheConnectionRecovery10ConnectionsTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheConnectionRecoveryTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheMessageRecoveryIdleConnectionTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheMessageWriteTimeoutTest;
@@ -259,7 +266,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridCacheStoreValueBytesSelfTest.class);
         GridTestUtils.addTestIfNeeded(suite, DataStreamProcessorSelfTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, DataStreamerUpdateAfterLoadTest.class, ignoredTests);
-            suite.addTestSuite(DataStreamerMultiThreadedSelfTest.class);
+        suite.addTestSuite(DataStreamerMultiThreadedSelfTest.class);
         suite.addTestSuite(DataStreamerMultinodeCreateCacheTest.class);
         suite.addTestSuite(DataStreamerImplSelfTest.class);
         suite.addTestSuite(DataStreamerTimeoutTest.class);
@@ -292,10 +299,13 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridCacheEntrySetIterationPreloadingSelfTest.class);
         suite.addTestSuite(GridCacheMixedPartitionExchangeSelfTest.class);
         suite.addTestSuite(IgniteCacheAtomicMessageRecoveryTest.class);
+        suite.addTestSuite(IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.class);
+        suite.addTestSuite(IgniteCacheAtomicMessageRecovery10ConnectionsTest.class);
         suite.addTestSuite(IgniteCacheTxMessageRecoveryTest.class);
         suite.addTestSuite(IgniteCacheMessageWriteTimeoutTest.class);
         suite.addTestSuite(IgniteCacheMessageRecoveryIdleConnectionTest.class);
         suite.addTestSuite(IgniteCacheConnectionRecoveryTest.class);
+        suite.addTestSuite(IgniteCacheConnectionRecovery10ConnectionsTest.class);
         GridTestUtils.addTestIfNeeded(suite, GridCacheOffHeapTieredEvictionAtomicSelfTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, GridCacheOffHeapTieredEvictionSelfTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, GridCacheOffHeapTieredAtomicSelfTest.class, ignoredTests);
@@ -325,6 +335,11 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
 
         suite.addTestSuite(CacheTxFastFinishTest.class);
 
+        suite.addTestSuite(IgniteVariousConnectionNumberTest.class);
+        suite.addTestSuite(IgniteCommunicationBalanceTest.class);
+        suite.addTestSuite(IgniteCommunicationBalanceMultipleConnectionsTest.class);
+        suite.addTestSuite(IgniteIoTestMessagesTest.class);
+
         return suite;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
index c557fbbcad791..11fcfdab78728 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiMultithreadedShmemTest;
 import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiRecoveryAckSelfTest;
 import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest;
+import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest;
 import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiRecoverySelfTest;
 import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiRecoverySslSelfTest;
 import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiShmemSelfTest;
@@ -50,6 +51,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(GridTcpCommunicationSpiRecoveryAckSelfTest.class));
         suite.addTest(new TestSuite(IgniteTcpCommunicationRecoveryAckClosureSelfTest.class));
         suite.addTest(new TestSuite(GridTcpCommunicationSpiRecoverySelfTest.class));
+        suite.addTest(new TestSuite(GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest.class));
         suite.addTest(new TestSuite(GridTcpCommunicationSpiRecoverySslSelfTest.class));
 
         suite.addTest(new TestSuite(GridTcpCommunicationSpiConcurrentConnectSelfTest.class));
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
index a7255341e1ebd..954237230b527 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
@@ -240,7 +240,7 @@ private IgniteInternalCache<HadoopJobId, HadoopJobMetadata> finishedJobMetaCache
 
                     try {
                         // Must process query callback in a separate thread to avoid deadlocks.
-                        evtProcSvc.submit(new EventHandler() {
+                        evtProcSvc.execute(new EventHandler() {
                             @Override protected void body() throws IgniteCheckedException {
                                 processJobMetadataUpdates(evts);
                             }
@@ -264,7 +264,7 @@ private IgniteInternalCache<HadoopJobId, HadoopJobMetadata> finishedJobMetaCache
 
                 try {
                     // Must process discovery callback in a separate thread to avoid deadlock.
-                    evtProcSvc.submit(new EventHandler() {
+                    evtProcSvc.execute(new EventHandler() {
                         @Override protected void body() {
                             processNodeLeft((DiscoveryEvent)evt);
                         }
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
index bc047e7e2fb3d..ff585091df7d4 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
@@ -639,6 +639,7 @@ private GridNioServer<HadoopMessage> resetNioServer() throws IgniteCheckedExcept
                         .logger(log.getLogger(GridNioServer.class))
                         .selectorCount(selectorsCnt)
                         .gridName(gridName)
+                        .serverName("hadoop")
                         .tcpNoDelay(tcpNoDelay)
                         .directBuffer(directBuf)
                         .byteOrder(ByteOrder.nativeOrder())
@@ -1305,11 +1306,11 @@ protected HandshakeAndBackpressureFilter() {
         }
 
         /** {@inheritDoc} */
-        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) throws IgniteCheckedException {
             if (ses.meta(PROCESS_META) == null && !(msg instanceof ProcessHandshakeMessage))
                 log.warning("Writing message before handshake has finished [ses=" + ses + ", msg=" + msg + ']');
 
-            return proceedSessionWrite(ses, msg);
+            return proceedSessionWrite(ses, msg, fut);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java
index a8de9996e60b2..3f33fb7cc103a 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java
@@ -190,7 +190,7 @@ protected HeadFilter() {
         }
 
         /** {@inheritDoc} */
-        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) {
+        @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) {
             assert ses == HadoopIpcToNioAdapter.this.ses : "ses=" + ses +
                 ", this.ses=" + HadoopIpcToNioAdapter.this.ses;
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java
index eeca564e9f107..24bba884ade0d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java
@@ -57,10 +57,10 @@ public HadoopMarshallerFilter(Marshaller marsh) {
     }
 
     /** {@inheritDoc} */
-    @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+    @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) throws IgniteCheckedException {
         assert msg instanceof HadoopMessage : "Invalid message type: " + msg;
 
-        return proceedSessionWrite(ses, U.marshal(marsh, msg));
+        return proceedSessionWrite(ses, U.marshal(marsh, msg), fut);
     }
 
     @Override public void onMessageReceived(GridNioSession ses, Object msg) throws IgniteCheckedException {
@@ -84,4 +84,4 @@ public HadoopMarshallerFilter(Marshaller marsh) {
     @Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
         proceedSessionWriteTimeout(ses);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java b/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
index f46ee936dbeb0..221538c1f92f5 100644
--- a/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
+++ b/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
@@ -169,7 +169,7 @@ public void start() {
 
         // Now create an object to consume the messages.
         for (final KafkaStream<K, V> stream : streams) {
-            executor.submit(new Runnable() {
+            executor.execute(new Runnable() {
                 @Override public void run() {
                     while (!stopped) {
                         try {
diff --git a/modules/tools/src/main/java/org/apache/ignite/tools/classgen/ClassesGenerator.java b/modules/tools/src/main/java/org/apache/ignite/tools/classgen/ClassesGenerator.java
index 0b10a7550efc8..369e318689adc 100644
--- a/modules/tools/src/main/java/org/apache/ignite/tools/classgen/ClassesGenerator.java
+++ b/modules/tools/src/main/java/org/apache/ignite/tools/classgen/ClassesGenerator.java
@@ -122,7 +122,11 @@ private void generate() throws Exception {
             for (String err : errs)
                 sb.append("    ").append(err).append('\n');
 
-            throw new Exception(sb.toString().trim());
+            String msg = sb.toString().trim();
+
+            System.out.println(msg);
+
+            throw new Exception(msg);
         }
 
         PrintStream out = new PrintStream(new File(basePath,
@@ -246,4 +250,4 @@ private void processClassFile(String path, int prefixLen)
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
index 54ef00d428220..07549d58206ce 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
@@ -30,7 +30,7 @@
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.apache.ignite.transactions.TransactionOptimisticException;
 import org.apache.ignite.transactions.TransactionRollbackException;
-import org.apache.ignite.yardstick.cache.IgniteSqlQueryBenchmark;
+import org.apache.ignite.yardstick.cache.IgnitePutBenchmark;
 import org.yardstickframework.BenchmarkDriver;
 import org.yardstickframework.BenchmarkDriverStartUp;
 
@@ -89,13 +89,13 @@ public static <T> T doInTransaction(IgniteTransactions igniteTx, TransactionConc
     public static void main(String[] args) throws Exception {
         final String cfg = "modules/yardstick/config/ignite-localhost-config.xml";
 
-        final Class<? extends BenchmarkDriver> benchmark = IgniteSqlQueryBenchmark.class;
+        final Class<? extends BenchmarkDriver> benchmark = IgnitePutBenchmark.class;
 
         final int threads = 1;
 
         final boolean clientDriverNode = true;
 
-        final int extraNodes = 2;
+        final int extraNodes = 4;
 
         final int warmUp = 5;
         final int duration = 5;
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java
index a25f9755da980..c8022e25f4ef8 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java
@@ -72,7 +72,7 @@ public class CacheEntryEventProbe implements BenchmarkProbe {
 
                 buildingService = Executors.newSingleThreadExecutor();
 
-                buildingService.submit(new Runnable() {
+                buildingService.execute(new Runnable() {
                     @Override public void run() {
                         try {
                             while (!Thread.currentThread().isInterrupted()) {
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java
new file mode 100644
index 0000000000000..bee45e0a339c9
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ */
+public class IgniteIoTestBenchmark extends IgniteAbstractBenchmark {
+    /** */
+    private List<ClusterNode> targetNodes;
+
+    /** */
+    private IgniteKernal ignite;
+
+    /** {@inheritDoc} */
+    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        ignite = (IgniteKernal)ignite();
+
+        targetNodes = new ArrayList<>();
+
+        ClusterNode loc = ignite().cluster().localNode();
+
+        Collection<ClusterNode> nodes = ignite().cluster().forServers().nodes();
+
+        for (ClusterNode node : nodes) {
+            if (!loc.equals(node))
+                targetNodes.add(node);
+        }
+
+        if (targetNodes.isEmpty())
+            throw new IgniteException("Failed to find remote server nodes [nodes=" + nodes + ']');
+
+        BenchmarkUtils.println(cfg, "Initialized target nodes: " + targetNodes + ']');
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        ClusterNode node = targetNodes.get(nextRandom(targetNodes.size()));
+
+        ignite.sendIoTest(node, null, false).get();
+
+        return true;
+    }
+}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestAbstractBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestAbstractBenchmark.java
new file mode 100644
index 0000000000000..8791c832b1440
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestAbstractBenchmark.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.io;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ *
+ */
+public abstract class IgniteIoTestAbstractBenchmark extends IgniteAbstractBenchmark {
+    /** */
+    protected final List<ClusterNode> targetNodes = new ArrayList<>();
+
+    /** */
+    protected IgniteKernal ignite;
+
+    /** {@inheritDoc} */
+    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        ignite = (IgniteKernal)ignite();
+
+        ClusterNode loc = ignite().cluster().localNode();
+
+        Collection<ClusterNode> nodes = ignite().cluster().forServers().nodes();
+
+        for (ClusterNode node : nodes) {
+            if (!loc.equals(node))
+                targetNodes.add(node);
+        }
+
+        if (targetNodes.isEmpty())
+            throw new IgniteException("Failed to find remote server nodes [nodes=" + nodes + ']');
+
+        BenchmarkUtils.println(cfg, "Initialized target nodes: " + F.nodeIds(targetNodes) + ']');
+    }
+}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendAllBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendAllBenchmark.java
new file mode 100644
index 0000000000000..9011910aafe7a
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendAllBenchmark.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.io;
+
+import java.util.Map;
+
+/**
+ *
+ */
+public class IgniteIoTestSendAllBenchmark extends IgniteIoTestAbstractBenchmark {
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        ignite.sendIoTest(targetNodes, null, false).get();
+
+        return true;
+    }
+}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendRandomBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendRandomBenchmark.java
new file mode 100644
index 0000000000000..88368e0cd3107
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendRandomBenchmark.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.io;
+
+import java.util.Map;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ *
+ */
+public class IgniteIoTestSendRandomBenchmark extends IgniteIoTestAbstractBenchmark {
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        ClusterNode node = targetNodes.get(nextRandom(targetNodes.size()));
+
+        ignite.sendIoTest(node, null, false).get();
+
+        return true;
+    }
+}

From 28dab6ea7573affe95512a403821cee5d82fcaf5 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 15 Dec 2016 12:15:22 +0300
Subject: [PATCH 055/446] ignite-4411 Changed test to avoid deadlocks when
 striped pool is used.

(cherry picked from commit 9a62d53)
---
 ...hePartitionedMultiNodeFullApiSelfTest.java | 37 +++++++++++++++----
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMultiNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMultiNodeFullApiSelfTest.java
index 71b14ebe18edd..34b67bc75e651 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMultiNodeFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMultiNodeFullApiSelfTest.java
@@ -24,6 +24,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Callable;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteAtomicLong;
 import org.apache.ignite.IgniteCache;
@@ -37,10 +38,12 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.testframework.GridTestUtils;
 
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
@@ -194,7 +197,7 @@ public void testUnswapShort() throws Exception {
 
         for (int i = 0; i < gridCount(); i++)
             grid(i).events().localListen(
-                    new SwapUnswapLocalListener(), EVT_CACHE_OBJECT_SWAPPED, EVT_CACHE_OBJECT_UNSWAPPED);
+                new SwapUnswapLocalListener(), EVT_CACHE_OBJECT_SWAPPED, EVT_CACHE_OBJECT_UNSWAPPED);
 
         jcache().put("key", 1);
 
@@ -202,13 +205,19 @@ public void testUnswapShort() throws Exception {
             if (grid(i).affinity(null).isBackup(grid(i).localNode(), "key")) {
                 jcache(i).localEvict(Collections.singleton("key"));
 
-                assert jcache(i).localPeek("key", ONHEAP) == null;
+                assertNull(jcache(i).localPeek("key", ONHEAP));
 
-                assert jcache(i).get("key") == 1;
+                assertEquals((Integer)1, jcache(i).get("key"));
+
+                GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                    @Override public boolean apply() {
+                        return swapEvts.get() == 1 && unswapEvts.get() == 1;
+                    }
+                }, 5000);
 
-                assert swapEvts.get() == 1 : "Swap events: " + swapEvts.get();
+                assertEquals(1, swapEvts.get());
 
-                assert unswapEvts.get() == 1 : "Unswap events: " + unswapEvts.get();
+                assertEquals(1, unswapEvts.get());
 
                 break;
             }
@@ -464,11 +473,25 @@ private static class SwapUnswapLocalListener implements IgnitePredicate<Event> {
 
             switch (evt.type()) {
                 case EVT_CACHE_OBJECT_SWAPPED:
-                    ignite.atomicLong("swapEvts", 0, false).incrementAndGet();
+                    // Run from another thread to avoid deadlock with striped pool.
+                    GridTestUtils.runAsync(new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            ignite.atomicLong("swapEvts", 0, false).incrementAndGet();
+
+                            return null;
+                        }
+                    });
 
                     break;
                 case EVT_CACHE_OBJECT_UNSWAPPED:
-                    ignite.atomicLong("unswapEvts", 0, false).incrementAndGet();
+                    // Run from another thread to avoid deadlock with striped pool.
+                    GridTestUtils.runAsync(new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            ignite.atomicLong("unswapEvts", 0, false).incrementAndGet();
+
+                            return null;
+                        }
+                    });
 
                     break;
             }

From ad785cbd192ca3f34d62bc2155f61a74f4962102 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 16 Dec 2016 19:23:29 +0300
Subject: [PATCH 056/446] ignite-2412 Do not call 'asyncOp' for synchronous
 operations

(cherry picked from commit c530d47)
---
 .../processors/cache/GridCacheAdapter.java    | 721 +++++++-----------
 .../processors/cache/IgniteCacheProxy.java    |   8 -
 .../dht/atomic/GridDhtAtomicCache.java        | 472 +++++++-----
 .../dht/colocated/GridDhtColocatedCache.java  |  13 -
 .../distributed/near/GridNearAtomicCache.java |  65 +-
 .../local/atomic/GridLocalAtomicCache.java    | 177 +----
 ...bledMultiNodeLongTxTimeoutFullApiTest.java |   2 +-
 ...ckMessageSystemPoolStarvationSelfTest.java |  14 +-
 8 files changed, 597 insertions(+), 875 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index a8d9f1d8b6cbe..5f0b8a0a15518 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -248,16 +248,16 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
 
     /** Grid configuration. */
     @GridToStringExclude
-    protected IgniteConfiguration gridCfg;
+    private IgniteConfiguration gridCfg;
 
     /** Cache metrics. */
     protected CacheMetricsImpl metrics;
 
     /** Cache localMxBean. */
-    protected CacheMetricsMXBean localMxBean;
+    private CacheMetricsMXBean locMxBean;
 
     /** Cache mxBean. */
-    protected CacheMetricsMXBean clusterMxBean;
+    private CacheMetricsMXBean clusterMxBean;
 
     /** Logger. */
     protected IgniteLogger log;
@@ -288,9 +288,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
     /** Asynchronous operations limit semaphore. */
     private Semaphore asyncOpsSem;
 
-    /** */
-    protected volatile boolean asyncToggled;
-
     /** {@inheritDoc} */
     @Override public String name() {
         return cacheCfg.getName();
@@ -334,7 +331,7 @@ protected GridCacheAdapter(final GridCacheContext<K, V> ctx, @Nullable GridCache
 
         metrics = new CacheMetricsImpl(ctx);
 
-        localMxBean = new CacheLocalMetricsMXBeanImpl(this);
+        locMxBean = new CacheLocalMetricsMXBeanImpl(this);
         clusterMxBean = new CacheClusterMetricsMXBeanImpl(this);
 
         FileSystemConfiguration[] igfsCfgs = gridCfg.getFileSystemConfiguration();
@@ -366,18 +363,6 @@ protected GridCacheAdapter(final GridCacheContext<K, V> ctx, @Nullable GridCache
         aff = new GridCacheAffinityImpl<>(ctx);
     }
 
-    /**
-     * Toggles async flag if someone calls {@code withAsync()}
-     * on proxy and since that we have to properly handle all cache
-     * operations (sync and async) to put them in proper sequence.
-     *
-     * TODO: https://issues.apache.org/jira/browse/IGNITE-4393
-     */
-    void toggleAsync() {
-        if (!asyncToggled)
-            asyncToggled = true;
-    }
-
     /**
      * Prints memory stats.
      */
@@ -471,49 +456,49 @@ public boolean isDht() {
     public abstract GridCachePreloader preloader();
 
     /** {@inheritDoc} */
-    @Override public Affinity<K> affinity() {
+    @Override public final Affinity<K> affinity() {
         return aff;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings({"unchecked", "RedundantCast"})
-    @Override public <K1, V1> IgniteInternalCache<K1, V1> cache() {
+    @Override public final <K1, V1> IgniteInternalCache<K1, V1> cache() {
         return (IgniteInternalCache<K1, V1>)this;
     }
 
     /** {@inheritDoc} */
-    @Override public GridCacheProxyImpl<K, V> forSubjectId(UUID subjId) {
+    @Override public final GridCacheProxyImpl<K, V> forSubjectId(UUID subjId) {
         CacheOperationContext opCtx = new CacheOperationContext(false, subjId, false, null, false, null);
 
         return new GridCacheProxyImpl<>(ctx, this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Override public boolean skipStore() {
+    @Override public final boolean skipStore() {
         return false;
     }
 
     /** {@inheritDoc} */
-    @Override public GridCacheProxyImpl<K, V> setSkipStore(boolean skipStore) {
+    @Override public final GridCacheProxyImpl<K, V> setSkipStore(boolean skipStore) {
         CacheOperationContext opCtx = new CacheOperationContext(true, null, false, null, false, null);
 
         return new GridCacheProxyImpl<>(ctx, this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Override public <K1, V1> GridCacheProxyImpl<K1, V1> keepBinary() {
+    @Override public final <K1, V1> GridCacheProxyImpl<K1, V1> keepBinary() {
         CacheOperationContext opCtx = new CacheOperationContext(false, null, true, null, false, null);
 
         return new GridCacheProxyImpl<>((GridCacheContext<K1, V1>)ctx, (GridCacheAdapter<K1, V1>)this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public ExpiryPolicy expiry() {
+    @Nullable @Override public final ExpiryPolicy expiry() {
         return null;
     }
 
     /** {@inheritDoc} */
-    @Override public GridCacheProxyImpl<K, V> withExpiryPolicy(ExpiryPolicy plc) {
+    @Override public final GridCacheProxyImpl<K, V> withExpiryPolicy(ExpiryPolicy plc) {
         assert !CU.isUtilityCache(ctx.name());
         assert !CU.isAtomicsCache(ctx.name());
         assert !CU.isMarshallerCache(ctx.name());
@@ -524,14 +509,14 @@ public boolean isDht() {
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalCache<K, V> withNoRetries() {
+    @Override public final IgniteInternalCache<K, V> withNoRetries() {
         CacheOperationContext opCtx = new CacheOperationContext(false, null, false, null, true, null);
 
         return new GridCacheProxyImpl<>(ctx, this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Override public CacheConfiguration configuration() {
+    @Override public final CacheConfiguration configuration() {
         return ctx.config();
     }
 
@@ -630,7 +615,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean isEmpty() {
+    @Override public final boolean isEmpty() {
         try {
             return localSize(CachePeekModes.ONHEAP_ONLY) == 0;
         }
@@ -640,7 +625,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean containsKey(K key) {
+    @Override public final boolean containsKey(K key) {
         try {
             return containsKeyAsync(key).get();
         }
@@ -667,7 +652,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean containsKeys(Collection<? extends K> keys) {
+    @Override public final boolean containsKeys(Collection<? extends K> keys) {
         try {
             return containsKeysAsync(keys).get();
         }
@@ -677,7 +662,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> containsKeysAsync(final Collection<? extends K> keys) {
+    @Override public final IgniteInternalFuture<Boolean> containsKeysAsync(final Collection<? extends K> keys) {
         A.notNull(keys, "keys");
 
         return getAllAsync(
@@ -708,7 +693,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public Iterable<Cache.Entry<K, V>> localEntries(CachePeekMode[] peekModes) throws IgniteCheckedException {
+    @Override public final Iterable<Cache.Entry<K, V>> localEntries(CachePeekMode[] peekModes) throws IgniteCheckedException {
         assert peekModes != null;
 
         ctx.checkSecurity(SecurityPermission.CACHE_READ);
@@ -765,7 +750,7 @@ public String toString() {
 
     /** {@inheritDoc} */
     @SuppressWarnings("ForLoopReplaceableByForEach")
-    @Nullable @Override public V localPeek(K key,
+    @Nullable @Override public final V localPeek(K key,
         CachePeekMode[] peekModes,
         @Nullable IgniteCacheExpiryPolicy plc)
         throws IgniteCheckedException {
@@ -907,7 +892,7 @@ public String toString() {
      *
      * @param ldr Class loader to undeploy.
      */
-    public void onUndeploy(ClassLoader ldr) {
+    public final void onUndeploy(ClassLoader ldr) {
         ctx.deploy().onUndeploy(ldr, context());
     }
 
@@ -916,7 +901,7 @@ public void onUndeploy(ClassLoader ldr) {
      * @param key Entry key.
      * @return Entry or <tt>null</tt>.
      */
-    @Nullable public GridCacheEntryEx peekEx(KeyCacheObject key) {
+    @Nullable public final GridCacheEntryEx peekEx(KeyCacheObject key) {
         return entry0(key, ctx.affinity().affinityTopologyVersion(), false, false);
     }
 
@@ -925,7 +910,7 @@ public void onUndeploy(ClassLoader ldr) {
      * @param key Entry key.
      * @return Entry or <tt>null</tt>.
      */
-    @Nullable public GridCacheEntryEx peekEx(Object key) {
+    @Nullable public final GridCacheEntryEx peekEx(Object key) {
         return entry0(ctx.toCacheKeyObject(key), ctx.affinity().affinityTopologyVersion(), false, false);
     }
 
@@ -933,7 +918,7 @@ public void onUndeploy(ClassLoader ldr) {
      * @param key Entry key.
      * @return Entry (never {@code null}).
      */
-    public GridCacheEntryEx entryEx(Object key) {
+    public final GridCacheEntryEx entryEx(Object key) {
         return entryEx(ctx.toCacheKeyObject(key), false);
     }
 
@@ -941,7 +926,7 @@ public GridCacheEntryEx entryEx(Object key) {
      * @param key Entry key.
      * @return Entry (never {@code null}).
      */
-    public GridCacheEntryEx entryEx(KeyCacheObject key) {
+    public final GridCacheEntryEx entryEx(KeyCacheObject key) {
         return entryEx(key, false);
     }
 
@@ -996,24 +981,24 @@ public GridCacheEntryEx entryEx(KeyCacheObject key, AffinityTopologyVersion topV
     /**
      * @return Set of internal cached entry representations.
      */
-    public Iterable<? extends GridCacheEntryEx> entries() {
+    public final Iterable<? extends GridCacheEntryEx> entries() {
         return allEntries();
     }
 
     /**
      * @return Set of internal cached entry representations.
      */
-    public Iterable<? extends GridCacheEntryEx> allEntries() {
+    public final Iterable<? extends GridCacheEntryEx> allEntries() {
         return map.entries();
     }
 
     /** {@inheritDoc} */
-    @Override public Set<Cache.Entry<K, V>> entrySet() {
+    @Override public final Set<Cache.Entry<K, V>> entrySet() {
         return entrySet((CacheEntryPredicate[])null);
     }
 
     /** {@inheritDoc} */
-    @Override public Set<Cache.Entry<K, V>> entrySetx(final CacheEntryPredicate... filter) {
+    @Override public final Set<Cache.Entry<K, V>> entrySetx(final CacheEntryPredicate... filter) {
         boolean keepBinary = ctx.keepBinary();
 
         return new EntrySet(map.entrySet(filter), keepBinary);
@@ -1025,17 +1010,17 @@ public Iterable<? extends GridCacheEntryEx> allEntries() {
     }
 
     /** {@inheritDoc} */
-    @Override public Set<K> keySet() {
+    @Override public final Set<K> keySet() {
         return new KeySet(map.entrySet());
     }
 
     /** {@inheritDoc} */
-    @Override public Set<K> keySetx() {
+    @Override public final Set<K> keySetx() {
         return keySet();
     }
 
     /** {@inheritDoc} */
-    @Override public Set<K> primaryKeySet() {
+    @Override public final Set<K> primaryKeySet() {
         return new KeySet(map.entrySet(CU.cachePrimary(ctx.grid().affinity(ctx.name()), ctx.localNode())));
     }
 
@@ -1057,7 +1042,7 @@ public Iterable<? extends GridCacheEntryEx> allEntries() {
      * @param filter Filters.
      * @return Collection of cached values.
      */
-    public Iterable<V> values(final CacheEntryPredicate... filter) {
+    public final Iterable<V> values(final CacheEntryPredicate... filter) {
         return new Iterable<V>() {
             @Override public Iterator<V> iterator() {
                 return new Iterator<V>() {
@@ -1083,12 +1068,12 @@ public Iterable<V> values(final CacheEntryPredicate... filter) {
      *
      * @param key Entry key.
      */
-    public void removeIfObsolete(KeyCacheObject key) {
+    public final void removeIfObsolete(KeyCacheObject key) {
         assert key != null;
 
         GridCacheMapEntry entry = map.getEntry(key);
 
-        if (entry.obsolete())
+        if (entry != null && entry.obsolete())
             removeEntry(entry);
     }
 
@@ -1272,11 +1257,11 @@ public void clearLocally(Collection<KeyCacheObject> keys, boolean readers) {
     /**
      * @param entry Removes entry from cache if currently mapped value is the same as passed.
      */
-    public void removeEntry(GridCacheEntryEx entry) {
-        boolean removed = map.removeEntry(entry);
+    public final void removeEntry(GridCacheEntryEx entry) {
+        boolean rmvd = map.removeEntry(entry);
 
         if (log.isDebugEnabled()) {
-            if (removed)
+            if (rmvd)
                 log.debug("Removed entry from cache: " + entry);
             else
                 log.debug("Remove will not be done for key (entry got replaced or removed): " + entry.key());
@@ -1311,7 +1296,7 @@ private boolean evictx(K key, GridCacheVersion ver,
     }
 
     /** {@inheritDoc} */
-    @Override public V getForcePrimary(K key) throws IgniteCheckedException {
+    @Override public final V getForcePrimary(K key) throws IgniteCheckedException {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(
@@ -1328,7 +1313,7 @@ private boolean evictx(K key, GridCacheVersion ver,
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getForcePrimaryAsync(final K key) {
+    @Override public final IgniteInternalFuture<V> getForcePrimaryAsync(final K key) {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(
@@ -1349,7 +1334,7 @@ private boolean evictx(K key, GridCacheVersion ver,
     }
 
     /** {@inheritDoc} */
-    public V getTopologySafe(K key) throws IgniteCheckedException {
+    public final V getTopologySafe(K key) throws IgniteCheckedException {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(
@@ -1366,12 +1351,12 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public Map<K, V> getAllOutTx(Set<? extends K> keys) throws IgniteCheckedException {
+    @Nullable @Override public final Map<K, V> getAllOutTx(Set<? extends K> keys) throws IgniteCheckedException {
         return getAllOutTxAsync(keys).get();
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Map<K, V>> getAllOutTxAsync(Set<? extends K> keys) {
+    @Override public final IgniteInternalFuture<Map<K, V>> getAllOutTxAsync(Set<? extends K> keys) {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(keys,
@@ -1385,15 +1370,6 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
             false);
     }
 
-    /**
-     * @param key Key.
-     * @param topVer Topology version.
-     * @return Entry.
-     */
-    @Nullable protected GridCacheEntryEx entryExSafe(KeyCacheObject key, AffinityTopologyVersion topVer) {
-        return entryEx(key);
-    }
-
     /** {@inheritDoc} */
     @Nullable @Override public V get(K key) throws IgniteCheckedException {
         A.notNull(key, "key");
@@ -1533,14 +1509,14 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
     }
 
     /** {@inheritDoc} */
-    @Override public Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
+    @Override public final Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
         boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         long start = statsEnabled ? System.nanoTime() : 0L;
 
-        Map<K, V> map = getAll(keys, !ctx.keepBinary(), false);
+        Map<K, V> map = getAll0(keys, !ctx.keepBinary(), false);
 
         if (ctx.config().getInterceptor() != null)
             map = interceptGet(keys, map);
@@ -1560,7 +1536,7 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
 
         long start = statsEnabled ? System.nanoTime() : 0L;
 
-        Map<K, T2<V, GridCacheVersion>> map = (Map<K, T2<V, GridCacheVersion>>)getAll(keys, !ctx.keepBinary(), true);
+        Map<K, T2<V, GridCacheVersion>> map = (Map<K, T2<V, GridCacheVersion>>)getAll0(keys, !ctx.keepBinary(), true);
 
         Collection<CacheEntry<K, V>> res = new HashSet<>();
 
@@ -1875,7 +1851,7 @@ public final IgniteInternalFuture<Map<K, V>> getAllAsync(@Nullable final Collect
      * @param needVer If {@code true} returns values as tuples containing value and version.
      * @return Future.
      */
-    public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
+    protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
         @Nullable final Collection<KeyCacheObject> keys,
         final boolean readThrough,
         boolean checkTx,
@@ -2138,7 +2114,7 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndPut(K key, V val) throws IgniteCheckedException {
+    @Override public final V getAndPut(K key, V val) throws IgniteCheckedException {
         return getAndPut(key, val, null);
     }
 
@@ -2160,7 +2136,24 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
         if (keyCheck)
             validateCacheKey(key);
 
-        V prevVal = syncOp(new SyncOp<V>(true) {
+        V prevVal = getAndPut0(key, val, filter);
+
+        if (statsEnabled)
+            metrics0().addPutAndGetTimeNanos(System.nanoTime() - start);
+
+        return prevVal;
+    }
+
+    /**
+     * @param key Key.
+     * @param val Value.
+     * @param filter Optional filter.
+     * @return Previous value.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected V getAndPut0(final K key, final V val, @Nullable final CacheEntryPredicate filter)
+        throws IgniteCheckedException {
+        return syncOp(new SyncOp<V>(true) {
             @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return (V)tx.putAsync(ctx, null, key, val, true, filter).get().value();
             }
@@ -2169,15 +2162,10 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                 return "put [key=" + key + ", val=" + val + ", filter=" + filter + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addPutAndGetTimeNanos(System.nanoTime() - start);
-
-        return prevVal;
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutAsync(K key, V val) {
+    @Override public final IgniteInternalFuture<V> getAndPutAsync(K key, V val) {
         return getAndPutAsync(key, val, null);
     }
 
@@ -2187,11 +2175,16 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
      * @param filter Filter.
      * @return Put operation future.
      */
-    public IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+    protected final IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
         final boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
 
+        A.notNull(key, "key", val, "val");
+
+        if (keyCheck)
+            validateCacheKey(key);
+
         IgniteInternalFuture<V> fut = getAndPutAsync0(key, val, filter);
 
         if (statsEnabled)
@@ -2206,13 +2199,10 @@ public IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntry
      * @param filter Optional filter.
      * @return Put operation future.
      */
-    public IgniteInternalFuture<V> getAndPutAsync0(final K key, final V val,
-        @Nullable final CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
+    public IgniteInternalFuture<V> getAndPutAsync0(final K key,
+        final V val,
+        @Nullable final CacheEntryPredicate filter)
+    {
         return asyncOp(new AsyncOp<V>() {
             @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAsync(ctx, readyTopVer, key, val, true, filter)
@@ -2226,7 +2216,7 @@ public IgniteInternalFuture<V> getAndPutAsync0(final K key, final V val,
     }
 
     /** {@inheritDoc} */
-    @Override public boolean put(final K key, final V val) throws IgniteCheckedException {
+    @Override public final boolean put(final K key, final V val) throws IgniteCheckedException {
         return put(key, val, null);
     }
 
@@ -2250,7 +2240,26 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
         if (keyCheck)
             validateCacheKey(key);
 
-        Boolean stored = syncOp(new SyncOp<Boolean>(true) {
+        boolean stored = put0(key, val, filter);
+
+        if (statsEnabled && stored)
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
+
+        return stored;
+    }
+
+    /**
+     * @param key Key.
+     * @param val Value.
+     * @param filter Filter.
+     * @return {@code True} if optional filter passed and value was stored in cache,
+     *      {@code false} otherwise. Note that this method will return {@code true} if filter is not
+     *      specified.
+     * @throws IgniteCheckedException If put operation failed.
+     */
+    protected boolean put0(final K key, final V val, final CacheEntryPredicate filter)
+        throws IgniteCheckedException {
+        Boolean res = syncOp(new SyncOp<Boolean>(true) {
             @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return tx.putAsync(ctx, null, key, val, false, filter).get().success();
             }
@@ -2260,10 +2269,9 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
             }
         });
 
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
+        assert res != null;
 
-        return stored;
+        return res;
     }
 
     /** {@inheritDoc} */
@@ -2305,7 +2313,7 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public <T> EntryProcessorResult<T> invoke(@Nullable AffinityTopologyVersion topVer,
+    @Nullable @Override public final <T> EntryProcessorResult<T> invoke(@Nullable AffinityTopologyVersion topVer,
         K key,
         EntryProcessor<K, V, T> entryProcessor,
         Object... args) throws IgniteCheckedException {
@@ -2538,7 +2546,7 @@ private <T> EntryProcessorResult<T> invoke0(
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putAsync(K key, V val) {
+    @Override public final IgniteInternalFuture<Boolean> putAsync(K key, V val) {
         return putAsync(key, val, null);
     }
 
@@ -2548,9 +2556,12 @@ private <T> EntryProcessorResult<T> invoke0(
      * @param filter Filter.
      * @return Put future.
      */
-    public IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+    public final IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
         A.notNull(key, "key", val, "val");
 
+        if (keyCheck)
+            validateCacheKey(key);
+
         final boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
@@ -2571,9 +2582,6 @@ public IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntry
      */
     public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         @Nullable final CacheEntryPredicate filter) {
-        if (keyCheck)
-            validateCacheKey(key);
-
         return asyncOp(new AsyncOp<Boolean>() {
             @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAsync(ctx,
@@ -2598,267 +2606,82 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public V getAndPutIfAbsent(final K key, final V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return syncOp(new SyncOp<V>(true) {
-            @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return (V)tx.putAsync(ctx, null, key, val, true, ctx.noVal()).get().value();
-            }
-
-            @Override public String toString() {
-                return "putIfAbsent [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Nullable @Override public final V getAndPutIfAbsent(final K key, final V val) throws IgniteCheckedException {
+        return getAndPut(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
-            @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx, readyTopVer, key, val, true, ctx.noVal())
-                    .chain((IgniteClosure<IgniteInternalFuture<GridCacheReturn>, V>)RET2VAL);
-            }
-
-            @Override public String toString() {
-                return "putIfAbsentAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
+    @Override public final IgniteInternalFuture<V> getAndPutIfAbsentAsync(final K key, final V val) {
+        return getAndPutAsync(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(final K key, final V val) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        Boolean stored = syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return tx.putAsync(ctx, null, key, val, false, ctx.noVal()).get().success();
-            }
-
-            @Override public String toString() {
-                return "putxIfAbsent [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled && stored)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
-
-        return stored;
+    @Override public final boolean putIfAbsent(final K key, final V val) throws IgniteCheckedException {
+        return put(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx,
-                    readyTopVer,
-                    key,
-                    val,
-                    false,
-                    ctx.noVal()).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "putxIfAbsentAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+    @Override public final IgniteInternalFuture<Boolean> putIfAbsentAsync(final K key, final V val) {
+        return putAsync(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public V getAndReplace(final K key, final V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return syncOp(new SyncOp<V>(true) {
-            @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return (V)tx.putAsync(ctx, null, key, val, true, ctx.hasVal()).get().value();
-            }
-
-            @Override public String toString() {
-                return "replace [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Nullable @Override public final V getAndReplace(final K key, final V val) throws IgniteCheckedException {
+        return getAndPut(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
-            @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx, readyTopVer, key, val, true, ctx.hasVal()).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, V>)RET2VAL);
-            }
-
-            @Override public String toString() {
-                return "replaceAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutAndGetTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
+    @Override public final IgniteInternalFuture<V> getAndReplaceAsync(final K key, final V val) {
+        return getAndPutAsync(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public boolean replace(final K key, final V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return tx.putAsync(ctx, null, key, val, false, ctx.hasVal()).get().success();
-            }
-
-            @Override public String toString() {
-                return "replacex [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Override public final boolean replace(final K key, final V val) throws IgniteCheckedException {
+        return put(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(final K key, final V val) {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx, readyTopVer, key, val, false, ctx.hasVal()).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>) RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "replacexAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Override public final IgniteInternalFuture<Boolean> replaceAsync(final K key, final V val) {
+        return putAsync(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public boolean replace(final K key, final V oldVal, final V newVal) throws IgniteCheckedException {
-        A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
-        if (keyCheck)
-            validateCacheKey(key);
+    @Override public final boolean replace(final K key, final V oldVal, final V newVal) throws IgniteCheckedException {
+        A.notNull(oldVal, "oldVal");
 
-        return syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled())
-                    ctx.deploy().registerClass(oldVal);
-
-                return tx.putAsync(ctx, null, key, newVal, false, ctx.equalsVal(oldVal)).get()
-                    .success();
-            }
-
-            @Override public String toString() {
-                return "replace [key=" + key + ", oldVal=" + oldVal + ", newVal=" + newVal + ']';
-            }
-        });
+        return put(key, newVal, ctx.equalsVal(oldVal));
     }
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<Boolean> replaceAsync(final K key, final V oldVal, final V newVal) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
+        A.notNull(oldVal, "oldVal");
 
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled()) {
-                    try {
-                        ctx.deploy().registerClass(oldVal);
-                    }
-                    catch (IgniteCheckedException e) {
-                        return new GridFinishedFuture<>(e);
-                    }
-                }
-
-                return tx.putAsync(ctx, readyTopVer, key, newVal, false, ctx.equalsVal(oldVal)).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "replaceAsync [key=" + key + ", oldVal=" + oldVal + ", newVal=" + newVal + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutAndGetTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+        return putAsync(key, newVal, ctx.equalsVal(oldVal));
     }
 
     /** {@inheritDoc} */
     @Override public void putAll(@Nullable final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        if (F.isEmpty(m))
+            return;
+
         boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         long start = statsEnabled ? System.nanoTime() : 0L;
 
-        if (F.isEmpty(m))
-            return;
-
         if (keyCheck)
             validateCacheKeys(m.keySet());
 
+        putAll0(m);
+
+        if (statsEnabled)
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
+    }
+
+    /**
+     * @param m Map.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void putAll0(final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
         syncOp(new SyncInOp(m.size() == 1) {
             @Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 tx.putAllAsync(ctx, null, m, false).get();
@@ -2868,9 +2691,6 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "putAll [map=" + m + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
@@ -2881,6 +2701,14 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKeys(m.keySet());
 
+        return putAllAsync0(m);
+    }
+
+    /**
+     * @param m Map.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<?> putAllAsync0(final Map<? extends K, ? extends V> m) {
         return asyncOp(new AsyncOp(m.keySet()) {
             @Override public IgniteInternalFuture<?> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAllAsync(ctx,
@@ -2906,11 +2734,25 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKey(key);
 
+        V prevVal = getAndRemove0(key);
+
+        if (statsEnabled)
+            metrics0().addRemoveAndGetTimeNanos(System.nanoTime() - start);
+
+        return prevVal;
+    }
+
+    /**
+     * @param key Key.
+     * @return Previous value.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected V getAndRemove0(final K key) throws IgniteCheckedException {
         final boolean keepBinary = ctx.keepBinary();
 
-        V prevVal = syncOp(new SyncOp<V>(true) {
+        return syncOp(new SyncOp<V>(true) {
             @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                K key0 = keepBinary ? (K)ctx.toCacheKeyObject(key) : key;
+                K key0 = keepBinary ? (K) ctx.toCacheKeyObject(key) : key;
 
                 V ret = tx.removeAllAsync(ctx,
                     null,
@@ -2920,9 +2762,9 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                     /*singleRmv*/false).get().value();
 
                 if (ctx.config().getInterceptor() != null) {
-                    K key = keepBinary ? (K)ctx.unwrapBinaryIfNeeded(key0, true, false) : key0;
+                    K key = keepBinary ? (K) ctx.unwrapBinaryIfNeeded(key0, true, false) : key0;
 
-                    return (V)ctx.config().getInterceptor().onBeforeRemove(new CacheEntryImpl(key, ret)).get2();
+                    return (V) ctx.config().getInterceptor().onBeforeRemove(new CacheEntryImpl(key, ret)).get2();
                 }
 
                 return ret;
@@ -2932,11 +2774,6 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "remove [key=" + key + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addRemoveAndGetTimeNanos(System.nanoTime() - start);
-
-        return prevVal;
     }
 
     /** {@inheritDoc} */
@@ -2950,7 +2787,20 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
+        IgniteInternalFuture<V> fut = getAndRemoveAsync0(key);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
+    }
+
+    /**
+     * @param key Key.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
+        return asyncOp(new AsyncOp<V>() {
             @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 // TODO should we invoke interceptor here?
                 return tx.removeAllAsync(ctx,
@@ -2965,11 +2815,6 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "removeAsync [key=" + key + ']';
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
     }
 
     /** {@inheritDoc} */
@@ -3002,6 +2847,17 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKeys(keys);
 
+        removeAll0(keys);
+
+        if (statsEnabled)
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+    }
+
+    /**
+     * @param keys Keys.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheckedException {
         syncOp(new SyncInOp(keys.size() == 1) {
             @Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 tx.removeAllAsync(ctx,
@@ -3016,24 +2872,34 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "removeAll [keys=" + keys + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<?> removeAllAsync(@Nullable final Collection<? extends K> keys) {
+        if (F.isEmpty(keys))
+            return new GridFinishedFuture<Object>();
+
         final boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
 
-        if (F.isEmpty(keys))
-            return new GridFinishedFuture<Object>();
-
         if (keyCheck)
             validateCacheKeys(keys);
 
-        IgniteInternalFuture<Object> fut = asyncOp(new AsyncOp(keys) {
+        IgniteInternalFuture<Object> fut = removeAllAsync0(keys);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+
+        return fut;
+    }
+
+    /**
+     * @param keys Keys.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<Object> removeAllAsync0(final Collection<? extends K> keys) {
+        return asyncOp(new AsyncOp(keys) {
             @Override public IgniteInternalFuture<?> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.removeAllAsync(ctx,
                     readyTopVer,
@@ -3047,15 +2913,20 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "removeAllAsync [keys=" + keys + ']';
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
-
-        return fut;
     }
 
     /** {@inheritDoc} */
     @Override public boolean remove(final K key) throws IgniteCheckedException {
+        return remove(key, (CacheEntryPredicate)null);
+    }
+
+    /**
+     * @param key Key.
+     * @param filter Filter.
+     * @return {@code True} if entry was removed.
+     * @throws IgniteCheckedException If failed.
+     */
+    public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
         boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         long start = statsEnabled ? System.nanoTime() : 0L;
@@ -3065,13 +2936,27 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKey(key);
 
-        boolean rmv = syncOp(new SyncOp<Boolean>(true) {
+        boolean rmv = remove0(key, filter);
+
+        if (statsEnabled && rmv)
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+
+        return rmv;
+    }
+
+    /**
+     * @param key Key.
+     * @return {@code True} if entry was removed.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected boolean remove0(final K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
+        Boolean res = syncOp(new SyncOp<Boolean>(true) {
             @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return tx.removeAllAsync(ctx,
                     null,
                     Collections.singletonList(key),
                     /*retval*/false,
-                    null,
+                    filter,
                     /*singleRmv*/true).get().success();
             }
 
@@ -3080,10 +2965,9 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
             }
         });
 
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+        assert res != null;
 
-        return rmv;
+        return res;
     }
 
     /** {@inheritDoc} */
@@ -3108,7 +2992,21 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
+        IgniteInternalFuture<Boolean> fut = removeAsync0(key, filter);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
+
+        return fut;
+    }
+
+    /**
+     * @param key Key.
+     * @param filter Filter.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<Boolean> removeAsync0(final K key, @Nullable final CacheEntryPredicate filter) {
+        return asyncOp(new AsyncOp<Boolean>() {
             @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.removeAllAsync(ctx,
                     readyTopVer,
@@ -3123,11 +3021,6 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
                 return "removeAsync [key=" + key + ", filter=" + filter + ']';
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
     }
 
     /** {@inheritDoc} */
@@ -3169,86 +3062,21 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
     }
 
     /** {@inheritDoc} */
-    @Override public boolean remove(final K key, final V val) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        boolean rmv = syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled())
-                    ctx.deploy().registerClass(val);
-
-                return tx.removeAllAsync(ctx,
-                    null,
-                    Collections.singletonList(key),
-                    /*retval*/false,
-                    ctx.equalsVal(val),
-                    /*singleRmv*/false).get().success();
-            }
-
-            @Override public String toString() {
-                return "remove [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Override public final boolean remove(final K key, final V val) throws IgniteCheckedException {
+        A.notNull(val, "val");
 
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
-
-        return rmv;
+        return remove(key, ctx.equalsVal(val));
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled()) {
-                    try {
-                        ctx.deploy().registerClass(val);
-                    }
-                    catch (IgniteCheckedException e) {
-                        return new GridFinishedFuture<>(e);
-                    }
-                }
-
-                return tx.removeAllAsync(ctx,
-                    readyTopVer,
-                    Collections.singletonList(key),
-                    /*retval*/false,
-                    ctx.equalsVal(val),
-                    /*singleRmv*/false).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
-            }
+    @Override public final IgniteInternalFuture<Boolean> removeAsync(final K key, final V val) {
+        A.notNull(key, "val");
 
-            @Override public String toString() {
-                return "removeAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+        return removeAsync(key, ctx.equalsVal(val));
     }
 
     /** {@inheritDoc} */
-    @Override public CacheMetrics clusterMetrics() {
+    @Override public final CacheMetrics clusterMetrics() {
         return clusterMetrics(ctx.grid().cluster().forCacheNodes(ctx.name()));
     }
 
@@ -3277,7 +3105,7 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
 
     /** {@inheritDoc} */
     @Override public CacheMetricsMXBean localMxBean() {
-        return localMxBean;
+        return locMxBean;
     }
 
     /** {@inheritDoc} */
@@ -4607,9 +4435,6 @@ protected void saveFuture(final FutureHolder holder, IgniteInternalFuture<?> fut
      * @return Failed future if waiting was interrupted.
      */
     @Nullable protected <T> IgniteInternalFuture<T> asyncOpAcquire() {
-        if (!asyncToggled)
-            return null;
-
         try {
             if (asyncOpsSem != null)
                 asyncOpsSem.acquire();
@@ -4627,8 +4452,8 @@ protected void saveFuture(final FutureHolder holder, IgniteInternalFuture<?> fut
     /**
      * Releases asynchronous operations permit, if limited.
      */
-    protected void asyncOpRelease() {
-        if (asyncOpsSem != null && asyncToggled)
+    private void asyncOpRelease() {
+        if (asyncOpsSem != null)
             asyncOpsSem.release();
     }
 
@@ -4793,12 +4618,10 @@ public Set<Cache.Entry<K, V>> entrySet(@Nullable CacheEntryPredicate... filter)
      * @return Cached value.
      * @throws IgniteCheckedException If failed.
      */
-    @Nullable public V get(K key, boolean deserializeBinary, final boolean needVer) throws IgniteCheckedException {
-        checkJta();
-
+    @Nullable public final V get(K key, boolean deserializeBinary, final boolean needVer) throws IgniteCheckedException {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
-        return get(key, taskName, deserializeBinary, needVer);
+        return get0(key, taskName, deserializeBinary, needVer);
     }
 
     /**
@@ -4809,11 +4632,13 @@ public Set<Cache.Entry<K, V>> entrySet(@Nullable CacheEntryPredicate... filter)
      * @return Cached value.
      * @throws IgniteCheckedException If failed.
      */
-    protected V get(
+    protected V get0(
         final K key,
         String taskName,
         boolean deserializeBinary,
         boolean needVer) throws IgniteCheckedException {
+        checkJta();
+
         try {
             return getAsync(key,
                 !ctx.config().isReadFromBackup(),
@@ -4867,7 +4692,7 @@ public final IgniteInternalFuture<V> getAsync(final K key, boolean deserializeBi
      * @return Map of cached values.
      * @throws IgniteCheckedException If read failed.
      */
-    public Map<K, V> getAll(Collection<? extends K> keys, boolean deserializeBinary,
+    protected Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary,
         boolean needVer) throws IgniteCheckedException {
         checkJta();
 
@@ -4922,7 +4747,7 @@ public void forceKeyCheck() {
      * @param key Cache key.
      * @throws IllegalArgumentException If validation fails.
      */
-    protected void validateCacheKey(Object key) {
+    protected final void validateCacheKey(Object key) {
         if (keyCheck) {
             CU.validateCacheKey(key);
 
@@ -4937,7 +4762,7 @@ protected void validateCacheKey(Object key) {
      * @param keys Cache keys.
      * @throws IgniteException If validation fails.
      */
-    protected void validateCacheKeys(Iterable<?> keys) {
+    protected final void validateCacheKeys(Iterable<?> keys) {
         if (keys == null)
             return;
 
@@ -4958,7 +4783,7 @@ protected void validateCacheKeys(Iterable<?> keys) {
      * @param deserializeBinary Deserialize binary flag.
      * @return Public API iterator.
      */
-    protected Iterator<Cache.Entry<K, V>> iterator(final Iterator<? extends GridCacheEntryEx> it,
+    protected final Iterator<Cache.Entry<K, V>> iterator(final Iterator<? extends GridCacheEntryEx> it,
         final boolean deserializeBinary) {
         return new Iterator<Cache.Entry<K, V>>() {
             {
@@ -5276,7 +5101,7 @@ private static PeekModes parsePeekModes(CachePeekMode[] peekModes, boolean prima
      * @param plc Explicitly specified expiry policy for cache operation.
      * @return Expiry policy wrapper.
      */
-    @Nullable public IgniteCacheExpiryPolicy expiryPolicy(@Nullable ExpiryPolicy plc) {
+    @Nullable public final IgniteCacheExpiryPolicy expiryPolicy(@Nullable ExpiryPolicy plc) {
         if (plc == null)
             plc = ctx.expiry();
 
@@ -5401,7 +5226,7 @@ public IgniteInternalFuture<T> op(final IgniteTxLocalAdapter tx, CacheOperationC
          * @param opCtx Operation context.
          * @return Operation future.
          */
-        protected IgniteInternalFuture<T> waitTopologyFuture(IgniteInternalFuture<?> topFut,
+        private IgniteInternalFuture<T> waitTopologyFuture(IgniteInternalFuture<?> topFut,
             final AffinityTopologyVersion topVer,
             final IgniteTxLocalAdapter tx,
             final CacheOperationContext opCtx) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
index b9e6e828a94fc..f87fa1d5d815f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
@@ -333,14 +333,6 @@ public GridCacheGateway<K, V> gate() {
         }
     }
 
-    /** {@inheritDoc} */
-    @Override public IgniteCache<K, V> withAsync() {
-        if (delegate instanceof GridCacheAdapter)
-            ((GridCacheAdapter)delegate).toggleAsync();
-
-        return super.withAsync();
-    }
-
     /** {@inheritDoc} */
     @Override public IgniteCache<K, V> withSkipStore() {
         return skipStore();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 0e60ff49ba79d..a67a9034d0003 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -139,7 +139,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
     private CI2<GridNearAtomicAbstractUpdateRequest, GridNearAtomicUpdateResponse> updateReplyClos;
 
     /** Pending */
-    private GridDeferredAckMessageSender deferredUpdateMessageSender;
+    private GridDeferredAckMessageSender deferredUpdateMsgSnd;
 
     /** */
     private GridNearAtomicCache<K, V> near;
@@ -173,6 +173,11 @@ public GridDhtAtomicCache(GridCacheContext<K, V> ctx, GridCacheConcurrentMap map
         msgLog = ctx.shared().atomicMessageLogger();
     }
 
+    /** {@inheritDoc} */
+    @Override protected void checkJta() throws IgniteCheckedException {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isDhtAtomic() {
         return true;
@@ -235,7 +240,7 @@ else if (res.error() != null) {
     @Override public void start() throws IgniteCheckedException {
         super.start();
 
-        deferredUpdateMessageSender = new GridDeferredAckMessageSender(ctx.time(), ctx.closures()) {
+        deferredUpdateMsgSnd = new GridDeferredAckMessageSender(ctx.time(), ctx.closures()) {
             @Override public int getTimeout() {
                 return DEFERRED_UPDATE_RESPONSE_TIMEOUT;
             }
@@ -447,7 +452,7 @@ else if (res.error() != null) {
 
     /** {@inheritDoc} */
     @Override public void stop() {
-        deferredUpdateMessageSender.stop();
+        deferredUpdateMsgSnd.stop();
     }
 
     /**
@@ -463,7 +468,7 @@ public void near(GridNearAtomicCache<K, V> near) {
     }
 
     /** {@inheritDoc} */
-    @Override protected V get(K key, String taskName, boolean deserializeBinary, boolean needVer)
+    @Override protected V get0(K key, String taskName, boolean deserializeBinary, boolean needVer)
         throws IgniteCheckedException {
         ctx.checkSecurity(SecurityPermission.CACHE_READ);
 
@@ -539,6 +544,21 @@ public void near(GridNearAtomicCache<K, V> near) {
         });
     }
 
+    /** {@inheritDoc} */
+    @Override protected Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
+        throws IgniteCheckedException {
+        return getAllAsyncInternal(keys,
+            !ctx.config().isReadFromBackup(),
+            true,
+            null,
+            ctx.kernalContext().job().currentTaskName(),
+            deserializeBinary,
+            false,
+            true,
+            needVer,
+            false).get();
+    }
+
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<Map<K, V>> getAllAsync(
         @Nullable final Collection<? extends K> keys,
@@ -550,6 +570,43 @@ public void near(GridNearAtomicCache<K, V> near) {
         final boolean skipVals,
         final boolean canRemap,
         final boolean needVer
+    ) {
+        return getAllAsyncInternal(keys,
+            forcePrimary,
+            skipTx,
+            subjId,
+            taskName,
+            deserializeBinary,
+            skipVals,
+            canRemap,
+            needVer,
+            true);
+    }
+
+    /**
+     * @param keys Keys.
+     * @param forcePrimary Force primary flag.
+     * @param skipTx Skip tx flag.
+     * @param subjId Subject ID.
+     * @param taskName Task name.
+     * @param deserializeBinary Deserialize binary flag.
+     * @param skipVals Skip values flag.
+     * @param canRemap Can remap flag.
+     * @param needVer Need version flag.
+     * @param asyncOp Async operation flag.
+     * @return Future.
+     */
+    private IgniteInternalFuture<Map<K, V>> getAllAsyncInternal(
+        @Nullable final Collection<? extends K> keys,
+        final boolean forcePrimary,
+        boolean skipTx,
+        @Nullable UUID subjId,
+        final String taskName,
+        final boolean deserializeBinary,
+        final boolean skipVals,
+        final boolean canRemap,
+        final boolean needVer,
+        boolean asyncOp
     ) {
         ctx.checkSecurity(SecurityPermission.CACHE_READ);
 
@@ -561,7 +618,7 @@ public void near(GridNearAtomicCache<K, V> near) {
 
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
-        subjId = ctx.subjectIdPerCall(null, opCtx);
+        subjId = ctx.subjectIdPerCall(subjId, opCtx);
 
         final UUID subjId0 = subjId;
 
@@ -569,57 +626,91 @@ public void near(GridNearAtomicCache<K, V> near) {
 
         final boolean skipStore = opCtx != null && opCtx.skipStore();
 
-        return asyncOp(new CO<IgniteInternalFuture<Map<K, V>>>() {
-            @Override public IgniteInternalFuture<Map<K, V>> apply() {
-                return getAllAsync0(ctx.cacheKeysView(keys),
-                    forcePrimary,
-                    subjId0,
-                    taskName,
-                    deserializeBinary,
-                    expiryPlc,
-                    skipVals,
-                    skipStore,
-                    canRemap,
-                    needVer);
-            }
-        });
+        if (asyncOp) {
+            return asyncOp(new CO<IgniteInternalFuture<Map<K, V>>>() {
+                @Override public IgniteInternalFuture<Map<K, V>> apply() {
+                    return getAllAsync0(ctx.cacheKeysView(keys),
+                        forcePrimary,
+                        subjId0,
+                        taskName,
+                        deserializeBinary,
+                        expiryPlc,
+                        skipVals,
+                        skipStore,
+                        canRemap,
+                        needVer);
+                }
+            });
+        }
+        else {
+            return getAllAsync0(ctx.cacheKeysView(keys),
+                forcePrimary,
+                subjId0,
+                taskName,
+                deserializeBinary,
+                expiryPlc,
+                skipVals,
+                skipStore,
+                canRemap,
+                needVer);
+        }
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndPut(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
-        return getAndPutAsync0(key, val, filter).get();
+    @Override protected V getAndPut0(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
+        return (V)update0(
+            key,
+            val,
+            null,
+            null,
+            true,
+            filter,
+            true,
+            false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public boolean put(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
-        return putAsync(key, val, filter).get();
+    @Override protected boolean put0(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
+        Boolean res = (Boolean)update0(
+            key,
+            val,
+            null,
+            null,
+            false,
+            filter,
+            true,
+            false).get();
+
+        assert res != null;
+
+        return res;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<V> getAndPutAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
-        return updateAsync0(
+        return update0(
             key,
             val,
             null,
             null,
             true,
             filter,
+            true,
             true);
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        return updateAsync0(
+        return update0(
             key,
             val,
             null,
             null,
             false,
             filter,
+            true,
             true);
     }
 
@@ -627,84 +718,34 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public V tryGetAndPut(K key, V val) throws IgniteCheckedException {
         A.notNull(key, "key", val, "val");
 
-        return (V)updateAsync0(
+        return (V) update0(
             key,
             val,
             null,
             null,
             true,
             null,
+            false,
             false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
-        return getAndPutIfAbsentAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return getAndPutAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
-        return putIfAbsentAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return putAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
-        return getAndReplaceAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return getAndPutAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V val) throws IgniteCheckedException {
-        return replaceAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return putAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
-        return replaceAsync(key, oldVal, newVal).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
-        A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
-        return putAsync(key, newVal, ctx.equalsVal(oldVal));
-    }
-
-    /** {@inheritDoc} */
-    @Override public void putAll(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
-        putAllAsync(m).get();
+    @Override protected void putAll0(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        updateAll0(m,
+            null,
+            null,
+            null,
+            null,
+            false,
+            false,
+            true,
+            UPDATE,
+            false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> putAllAsync(Map<? extends K, ? extends V> m) {
-        return updateAllAsync0(m,
+    @Override public IgniteInternalFuture<?> putAllAsync0(Map<? extends K, ? extends V> m) {
+        return updateAll0(m,
             null,
             null,
             null,
@@ -712,7 +753,8 @@ public void near(GridNearAtomicCache<K, V> near) {
             false,
             false,
             true,
-            UPDATE).chain(RET2NULL);
+            UPDATE,
+            true).chain(RET2NULL);
     }
 
     /** {@inheritDoc} */
@@ -725,7 +767,7 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public IgniteInternalFuture<?> putAllConflictAsync(Map<KeyCacheObject, GridCacheDrInfo> conflictMap) {
         ctx.dr().onReceiveCacheEntriesReceived(conflictMap.size());
 
-        return updateAllAsync0(null,
+        return updateAll0(null,
             null,
             null,
             conflictMap,
@@ -733,57 +775,40 @@ public void near(GridNearAtomicCache<K, V> near) {
             false,
             false,
             true,
-            UPDATE);
+            UPDATE,
+            true);
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndRemove(K key) throws IgniteCheckedException {
-        return getAndRemoveAsync(key).get();
+    @Override public V getAndRemove0(K key) throws IgniteCheckedException {
+        return (V)remove0(key, true, null, false).get();
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<V> getAndRemoveAsync(K key) {
-        A.notNull(key, "key");
-
-        return removeAsync0(key, true, null);
+    @Override public IgniteInternalFuture<V> getAndRemoveAsync0(K key) {
+        return remove0(key, true, null, true);
     }
 
     /** {@inheritDoc} */
-    @Override public void removeAll(Collection<? extends K> keys) throws IgniteCheckedException {
-        removeAllAsync(keys).get();
+    @Override protected void removeAll0(Collection<? extends K> keys) throws IgniteCheckedException {
+        removeAllAsync0(keys, null, false, false, false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> removeAllAsync(Collection<? extends K> keys) {
-        A.notNull(keys, "keys");
-
-        return removeAllAsync0(keys, null, false, false).chain(RET2NULL);
+    @Override public IgniteInternalFuture<Object> removeAllAsync0(Collection<? extends K> keys) {
+        return removeAllAsync0(keys, null, false, false, true).chain(RET2NULL);
     }
 
     /** {@inheritDoc} */
-    @Override public boolean remove(K key) throws IgniteCheckedException {
-        return removeAsync(key, (CacheEntryPredicate)null).get();
+    @Override protected boolean remove0(K key, CacheEntryPredicate filter) throws IgniteCheckedException {
+        return (Boolean)remove0(key, false, filter, false).get();
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key");
-
-        return removeAsync0(key, false, filter);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean remove(K key, V val) throws IgniteCheckedException {
-        return removeAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return removeAsync(key, ctx.equalsVal(val));
+    @Override public IgniteInternalFuture<Boolean> removeAsync0(K key, @Nullable CacheEntryPredicate filter) {
+        return remove0(key, false, filter, true);
     }
 
     /** {@inheritDoc} */
@@ -796,7 +821,7 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public IgniteInternalFuture<?> removeAllConflictAsync(Map<KeyCacheObject, GridCacheVersion> conflictMap) {
         ctx.dr().onReceiveCacheEntriesReceived(conflictMap.size());
 
-        return removeAllAsync0(null, conflictMap, false, false);
+        return removeAllAsync0(null, conflictMap, false, false, true);
     }
 
     /**
@@ -811,10 +836,7 @@ private boolean writeThrough() {
      * @return Future.
      */
     @SuppressWarnings("unchecked")
-    protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>> op) {
-        if (!asyncToggled)
-            return op.apply();
-
+    private <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>> op) {
         IgniteInternalFuture<T> fail = asyncOpAcquire();
 
         if (fail != null)
@@ -871,7 +893,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     /** {@inheritDoc} */
     @Override public <T> EntryProcessorResult<T> invoke(K key, EntryProcessor<K, V, T> entryProcessor, Object... args)
         throws IgniteCheckedException {
-        IgniteInternalFuture<EntryProcessorResult<T>> invokeFut = invokeAsync(key, entryProcessor, args);
+        IgniteInternalFuture<EntryProcessorResult<T>> invokeFut = invoke0(false, key, entryProcessor, args);
 
         EntryProcessorResult<T> res = invokeFut.get();
 
@@ -881,14 +903,28 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     /** {@inheritDoc} */
     @Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(Set<? extends K> keys,
         EntryProcessor<K, V, T> entryProcessor,
-        Object... args)
-        throws IgniteCheckedException {
-        return invokeAllAsync(keys, entryProcessor, args).get();
+        Object... args) throws IgniteCheckedException
+    {
+        return invokeAll0(false, keys, entryProcessor, args).get();
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
     @Override public <T> IgniteInternalFuture<EntryProcessorResult<T>> invokeAsync(K key,
+        EntryProcessor<K, V, T> entryProcessor,
+        Object... args) {
+        return invoke0(true, key, entryProcessor, args);
+    }
+
+    /**
+     * @param async Async operation flag.
+     * @param key Key.
+     * @param entryProcessor Entry processor.
+     * @param args Entry processor arguments.
+     * @return Future.
+     */
+    private <T> IgniteInternalFuture<EntryProcessorResult<T>> invoke0(
+        boolean async,
+        K key,
         EntryProcessor<K, V, T> entryProcessor,
         Object... args) {
         A.notNull(key, "key", entryProcessor, "entryProcessor");
@@ -900,14 +936,15 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut = updateAsync0(
+        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut = update0(
             key,
             null,
             entryProcessor,
             args,
             false,
             null,
-            true);
+            true,
+            async);
 
         return fut.chain(new CX1<IgniteInternalFuture<Map<K, EntryProcessorResult<T>>>, EntryProcessorResult<T>>() {
             @Override public EntryProcessorResult<T> applyx(IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut)
@@ -938,6 +975,21 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public <T> IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> invokeAllAsync(Set<? extends K> keys,
+        final EntryProcessor<K, V, T> entryProcessor,
+        Object... args) {
+        return invokeAll0(true, keys, entryProcessor, args);
+    }
+
+    /**
+     * @param async Async operation flag.
+     * @param keys Keys.
+     * @param entryProcessor Entry processor.
+     * @param args Entry processor arguments.
+     * @return Future.
+     */
+    private <T> IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> invokeAll0(
+        boolean async,
+        Set<? extends K> keys,
         final EntryProcessor<K, V, T> entryProcessor,
         Object... args) {
         A.notNull(keys, "keys", entryProcessor, "entryProcessor");
@@ -955,7 +1007,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> resFut = updateAllAsync0(null,
+        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> resFut = updateAll0(null,
             invokeMap,
             args,
             null,
@@ -963,7 +1015,8 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
             false,
             false,
             true,
-            TRANSFORM);
+            TRANSFORM,
+            async);
 
         return resFut.chain(
             new CX1<IgniteInternalFuture<Map<K, EntryProcessorResult<T>>>, Map<K, EntryProcessorResult<T>>>() {
@@ -981,7 +1034,21 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     @Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(
         Map<? extends K, ? extends EntryProcessor<K, V, T>> map,
         Object... args) throws IgniteCheckedException {
-        return invokeAllAsync(map, args).get();
+        A.notNull(map, "map");
+
+        if (keyCheck)
+            validateCacheKeys(map.keySet());
+
+        return (Map<K, EntryProcessorResult<T>>)updateAll0(null,
+            map,
+            args,
+            null,
+            null,
+            false,
+            false,
+            true,
+            TRANSFORM,
+            false).get();
     }
 
     /** {@inheritDoc} */
@@ -994,7 +1061,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
         if (keyCheck)
             validateCacheKeys(map.keySet());
 
-        return updateAllAsync0(null,
+        return updateAll0(null,
             map,
             args,
             null,
@@ -1002,7 +1069,8 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
             false,
             false,
             true,
-            TRANSFORM);
+            TRANSFORM,
+            true);
     }
 
     /**
@@ -1016,10 +1084,11 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
      * @param retval Return value required flag.
      * @param rawRetval Return {@code GridCacheReturn} instance.
      * @param waitTopFut Whether to wait for topology future.
+     * @param async Async operation flag.
      * @return Completion future.
      */
     @SuppressWarnings("ConstantConditions")
-    private IgniteInternalFuture updateAllAsync0(
+    private IgniteInternalFuture updateAll0(
         @Nullable Map<? extends K, ? extends V> map,
         @Nullable Map<? extends K, ? extends EntryProcessor> invokeMap,
         @Nullable Object[] invokeArgs,
@@ -1028,7 +1097,8 @@ private IgniteInternalFuture updateAllAsync0(
         final boolean retval,
         final boolean rawRetval,
         final boolean waitTopFut,
-        final GridCacheOperation op
+        final GridCacheOperation op,
+        boolean async
     ) {
         assert ctx.updatesAllowed();
 
@@ -1105,13 +1175,20 @@ else if (op == GridCacheOperation.DELETE) {
             opCtx != null && opCtx.noRetries() ? 1 : MAX_RETRIES,
             waitTopFut);
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-                return updateFut;
-            }
-        });
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
+
+            return updateFut;
+        }
     }
 
     /**
@@ -1124,16 +1201,18 @@ else if (op == GridCacheOperation.DELETE) {
      * @param retval Return value flag.
      * @param filter Filter.
      * @param waitTopFut Whether to wait for topology future.
+     * @param async Async operation flag.
      * @return Future.
      */
-    private IgniteInternalFuture updateAsync0(
+    private IgniteInternalFuture update0(
         K key,
         @Nullable V val,
         @Nullable EntryProcessor proc,
         @Nullable Object[] invokeArgs,
         final boolean retval,
         @Nullable final CacheEntryPredicate filter,
-        final boolean waitTopFut
+        final boolean waitTopFut,
+        boolean async
     ) {
         assert val == null || proc == null;
 
@@ -1146,13 +1225,20 @@ private IgniteInternalFuture updateAsync0(
         final GridNearAtomicAbstractUpdateFuture updateFut =
             createSingleUpdateFuture(key, val, proc, invokeArgs, retval, filter, waitTopFut);
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-                return updateFut;
-            }
-        });
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
+
+            return updateFut;
+        }
     }
 
     /**
@@ -1161,33 +1247,38 @@ private IgniteInternalFuture updateAsync0(
      * @param key Key.
      * @param retval Whether to return
      * @param filter Filter.
+     * @param async Async operation flag.
      * @return Future.
      */
-    private IgniteInternalFuture removeAsync0(K key, final boolean retval,
-        @Nullable CacheEntryPredicate filter) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
+    private IgniteInternalFuture remove0(K key, final boolean retval,
+        @Nullable CacheEntryPredicate filter,
+        boolean async) {
         assert ctx.updatesAllowed();
 
-        validateCacheKey(key);
-
         ctx.checkSecurity(SecurityPermission.CACHE_REMOVE);
 
-        final GridNearAtomicAbstractUpdateFuture updateFut =
-            createSingleUpdateFuture(key, null, null, null, retval, filter, true);
+        final GridNearAtomicAbstractUpdateFuture updateFut = createSingleUpdateFuture(key,
+            null,
+            null,
+            null,
+            retval,
+            filter,
+            true);
 
-        if (statsEnabled)
-            updateFut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
 
-                return updateFut;
-            }
-        });
+            return updateFut;
+        }
     }
 
     /**
@@ -1326,14 +1417,11 @@ private IgniteInternalFuture removeAllAsync0(
         @Nullable Collection<? extends K> keys,
         @Nullable Map<KeyCacheObject, GridCacheVersion> conflictMap,
         final boolean retval,
-        boolean rawRetval
+        boolean rawRetval,
+        boolean async
     ) {
         assert ctx.updatesAllowed();
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         assert keys != null || conflictMap != null;
 
         if (keyCheck)
@@ -1380,16 +1468,20 @@ private IgniteInternalFuture removeAllAsync0(
             opCtx != null && opCtx.noRetries() ? 1 : MAX_RETRIES,
             true);
 
-        if (statsEnabled)
-            updateFut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
 
-                return updateFut;
-            }
-        });
+            return updateFut;
+        }
     }
 
     /**
@@ -3248,7 +3340,7 @@ && writeThrough() && !req.skipStore(),
      * @param ver Version to ack.
      */
     private void sendDeferredUpdateResponse(UUID nodeId, GridCacheVersion ver) {
-        deferredUpdateMessageSender.sendDeferredAckMessage(nodeId, ver);
+        deferredUpdateMsgSnd.sendDeferredAckMessage(nodeId, ver);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 176a90f5894db..9cf400d8967df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -327,19 +327,6 @@ public GridDistributedCacheEntry entryExx(
             needVer);
     }
 
-    /** {@inheritDoc} */
-    @Override protected GridCacheEntryEx entryExSafe(
-        KeyCacheObject key,
-        AffinityTopologyVersion topVer
-    ) {
-        try {
-            return ctx.affinity().localNode(key, topVer) ? entryEx(key) : null;
-        }
-        catch (GridDhtInvalidPartitionException ignored) {
-            return null;
-        }
-    }
-
     /**
      * @param keys Keys to load.
      * @param readThrough Read through flag.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index a8219b08cefdc..4350b3e9f46a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -449,61 +449,11 @@ public void processDhtAtomicUpdateRequest(
         return dht.putAsync0(key, val, filter);
     }
 
-    /** {@inheritDoc} */
-    @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
-        return dht.getAndPutIfAbsent(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
-        return dht.getAndPutIfAbsentAsync(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
-        return dht.putIfAbsent(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
-        return dht.putIfAbsentAsync(key, val);
-    }
-
     /** {@inheritDoc} */
     @Nullable @Override public V tryGetAndPut(K key, V val) throws IgniteCheckedException {
         return dht.tryGetAndPut(key, val);
     }
 
-    /** {@inheritDoc} */
-    @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
-        return dht.getAndReplace(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
-        return dht.getAndReplaceAsync(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V val) throws IgniteCheckedException {
-        return dht.replace(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
-        return dht.replaceAsync(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
-        return dht.replace(key, oldVal, newVal);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
-        return dht.replaceAsync(key, oldVal, newVal);
-    }
-
     /** {@inheritDoc} */
     @Override public void putAll(Map<? extends K, ? extends V> m)
         throws IgniteCheckedException {
@@ -568,6 +518,11 @@ public void processDhtAtomicUpdateRequest(
         return dht.invokeAllAsync(keys, entryProcessor, args);
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean remove(K key, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
+        return dht.remove(key, filter);
+    }
+
     /** {@inheritDoc} */
     @Override public V getAndRemove(K key) throws IgniteCheckedException {
         return dht.getAndRemove(key);
@@ -601,16 +556,6 @@ public void processDhtAtomicUpdateRequest(
         return dht.removeAsync(key, filter);
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean remove(K key, V val) throws IgniteCheckedException {
-        return dht.remove(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
-        return dht.removeAsync(key, val);
-    }
-
     /** {@inheritDoc} */
     @Override public void removeAll() throws IgniteCheckedException {
         dht.removeAll();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index bc16ff4fd0b8a..a26d2f3fdde02 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -58,7 +58,6 @@
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.resource.GridResourceIoc;
-import org.apache.ignite.internal.processors.resource.GridResourceProcessor;
 import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
@@ -107,6 +106,11 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
         preldr = new GridCachePreloaderAdapter(ctx);
     }
 
+    /** {@inheritDoc} */
+    @Override protected void checkJta() throws IgniteCheckedException {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isLocal() {
         return true;
@@ -119,9 +123,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public V getAndPut(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
+    @Override protected V getAndPut0(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         return (V)updateAllInternal(UPDATE,
@@ -138,16 +140,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean put(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
+    @Override protected boolean put0(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
-        boolean res = (Boolean)updateAllInternal(UPDATE,
+        Boolean res = (Boolean)updateAllInternal(UPDATE,
             Collections.singleton(key),
             Collections.singleton(val),
             null,
@@ -159,8 +155,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
             ctx.readThrough(),
             opCtx != null && opCtx.isKeepBinary());
 
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
+        assert res != null;
 
         return res;
     }
@@ -168,8 +163,6 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<V> getAndPutAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
         return updateAllAsync0(F0.asMap(key, val),
             null,
             null,
@@ -181,8 +174,6 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
         return updateAllAsync0(F0.asMap(key, val),
             null,
             null,
@@ -192,65 +183,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
-        return getAndPut(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
-        return getAndPutAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
-        return put(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
-        return putAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
-        return getAndPut(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
-        return getAndPutAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V val) throws IgniteCheckedException {
-        return put(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
-        return putAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
-        A.notNull(oldVal, "oldVal");
-
-        return put(key, newVal, ctx.equalsVal(oldVal));
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
-        return putAsync(key, newVal, ctx.equalsVal(oldVal));
-    }
-
-    /** {@inheritDoc} */
-    @Override public void putAll(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
+    @Override protected void putAll0(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         updateAllInternal(UPDATE,
@@ -264,13 +197,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
             ctx.writeThrough(),
             ctx.readThrough(),
             opCtx != null && opCtx.isKeepBinary());
-
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> putAllAsync(Map<? extends K, ? extends V> m) {
+    @Override public IgniteInternalFuture<?> putAllAsync0(Map<? extends K, ? extends V> m) {
         return updateAllAsync0(m,
             null,
             null,
@@ -280,8 +210,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public V getAndRemove(K key) throws IgniteCheckedException {
+    @Override protected V getAndRemove0(K key) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         return (V)updateAllInternal(DELETE,
@@ -299,13 +228,13 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<V> getAndRemoveAsync(K key) {
+    @Override public IgniteInternalFuture<V> getAndRemoveAsync0(K key) {
         return removeAllAsync0(Collections.singletonList(key), true, false, null);
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public void removeAll(Collection<? extends K> keys) throws IgniteCheckedException {
+    @Override public void removeAll0(Collection<? extends K> keys) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         updateAllInternal(DELETE,
@@ -322,19 +251,13 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> removeAllAsync(Collection<? extends K> keys) {
+    @Override public IgniteInternalFuture<Object> removeAllAsync0(Collection<? extends K> keys) {
         return removeAllAsync0(keys, false, false, null).chain(RET2NULL);
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public boolean remove(K key) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key");
-
+    @Override public boolean remove0(K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         Boolean rmv = (Boolean)updateAllInternal(DELETE,
@@ -344,49 +267,22 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
             expiryPerCall(),
             false,
             false,
-            null,
+            filter,
             ctx.writeThrough(),
             ctx.readThrough(),
             opCtx != null && opCtx.isKeepBinary());
 
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+        assert rmv != null;
 
         return rmv;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key");
-
+    @Override public IgniteInternalFuture<Boolean> removeAsync0(K key, @Nullable CacheEntryPredicate filter) {
         return removeAllAsync0(Collections.singletonList(key), false, false, filter);
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean remove(K key, V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        CacheOperationContext opCtx = ctx.operationContextPerCall();
-
-        return (Boolean)updateAllInternal(DELETE,
-            Collections.singleton(key),
-            null,
-            null,
-            expiryPerCall(),
-            false,
-            false,
-            ctx.equalsVal(val),
-            ctx.writeThrough(),
-            ctx.readThrough(),
-            opCtx != null && opCtx.isKeepBinary());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
-        return removeAsync(key, ctx.equalsVal(val));
-    }
-
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<?> removeAllAsync() {
         return ctx.closures().callLocalSafe(new Callable<Void>() {
@@ -399,11 +295,13 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-
     @SuppressWarnings("unchecked")
-    @Override @Nullable public V get(K key, boolean deserializeBinary, boolean needVer) throws IgniteCheckedException {
-        String taskName = ctx.kernalContext().job().currentTaskName();
-
+    @Override protected V get0(
+        final K key,
+        String taskName,
+        boolean deserializeBinary,
+        boolean needVer) throws IgniteCheckedException
+    {
         Map<K, V> m = getAllInternal(Collections.singleton(key),
             ctx.isSwapOrOffheapEnabled(),
             ctx.readThrough(),
@@ -419,7 +317,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public final Map<K, V> getAll(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
+    @Override public final Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
         throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
@@ -794,7 +692,7 @@ private IgniteInternalFuture updateAllAsync0(
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture fut = asyncOp(new Callable<Object>() {
+        return asyncOp(new Callable<Object>() {
             @Override public Object call() throws Exception {
                 return updateAllInternal(op,
                     keys,
@@ -809,11 +707,6 @@ private IgniteInternalFuture updateAllAsync0(
                     keepBinary);
             }
         });
-
-        if (ctx.config().isStatisticsEnabled())
-            fut.listen(new UpdatePutTimeStatClosure(metrics0(), System.nanoTime()));
-
-        return fut;
     }
 
     /**
@@ -835,17 +728,13 @@ private IgniteInternalFuture removeAllAsync0(
 
         final boolean readThrough = ctx.readThrough();
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         final ExpiryPolicy expiryPlc = expiryPerCall();
 
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture fut = asyncOp(new Callable<Object>() {
+        return asyncOp(new Callable<Object>() {
             @Override public Object call() throws Exception {
                 return updateAllInternal(DELETE,
                     keys,
@@ -860,11 +749,6 @@ private IgniteInternalFuture removeAllAsync0(
                     keepBinary);
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
-
-        return fut;
     }
 
     /**
@@ -1584,10 +1468,7 @@ private static CachePartialUpdateCheckedException partialUpdateException() {
      * @return Future.
      */
     @SuppressWarnings("unchecked")
-    protected IgniteInternalFuture asyncOp(final Callable<?> op) {
-        if (!asyncToggled)
-            return ctx.closures().callLocalSafe(op);
-
+    private IgniteInternalFuture asyncOp(final Callable<?> op) {
         IgniteInternalFuture fail = asyncOpAcquire();
 
         if (fail != null)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
index 3e3b84e4403bf..648134e498c85 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
@@ -34,7 +34,7 @@ public class CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest extend
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
-        cfg.getTransactionConfiguration().setDefaultTxTimeout(Long.MAX_VALUE);
+        cfg.getTransactionConfiguration().setDefaultTxTimeout(5 * 60_000);
 
         return cfg;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
index ec3b8089cae6e..57c709baba12d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteFileSystem;
@@ -39,14 +41,11 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.transactions.TransactionConcurrency;
-import org.apache.ignite.transactions.TransactionIsolation;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
 /**
  * Test to check for system pool starvation due to {@link IgfsBlocksMessage}.
@@ -125,8 +124,7 @@ public void testStarvation() throws Exception {
             @Override public Void call() throws Exception {
                 GridCacheAdapter dataCache = dataCache(attacker);
 
-                try (IgniteInternalTx tx =
-                         dataCache.txStartEx(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+                try (IgniteInternalTx tx = dataCache.txStartEx(PESSIMISTIC, REPEATABLE_READ)) {
                     dataCache.put(DATA_KEY, 0);
 
                     txStartLatch.countDown();
@@ -185,6 +183,7 @@ private static boolean awaitFuture(final IgniteInternalFuture fut) throws Except
      * Create IGFS file asynchronously.
      *
      * @param path Path.
+     * @param writeStartLatch Write start latch.
      * @return Future.
      */
     private IgniteInternalFuture<Void> createFileAsync(final IgfsPath path, final CountDownLatch writeStartLatch) {
@@ -265,6 +264,7 @@ private IgniteConfiguration config(String name, TcpDiscoveryVmIpFinder ipFinder)
         cfg.setLocalHost("127.0.0.1");
         cfg.setConnectorConfiguration(null);
 
+        cfg.setStripedPoolSize(0);
         cfg.setSystemThreadPoolSize(2);
         cfg.setRebalanceThreadPoolSize(1);
         cfg.setPublicThreadPoolSize(1);

From c1fde05d05251353f0e75125aca44ab145798c8e Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 16 Dec 2016 19:15:48 +0300
Subject: [PATCH 057/446] ignite-4371 Avoid synchronous 'rollback' call from
 system threads

(cherry picked from commit 0c782b0)
---
 .../GridCachePartitionExchangeManager.java    |  37 +++---
 .../processors/cache/GridCacheProcessor.java  |   2 +-
 .../processors/cache/GridCacheUtils.java      |  10 +-
 .../GridDistributedTxRemoteAdapter.java       |   2 +-
 .../dht/GridDhtTxFinishFuture.java            |  74 +++++------
 .../cache/distributed/dht/GridDhtTxLocal.java | 125 ++++++------------
 .../dht/GridDhtTxLocalAdapter.java            |   7 +-
 .../near/GridNearTxFinishFuture.java          |  44 +++---
 .../distributed/near/GridNearTxLocal.java     |  48 ++++---
 .../cache/transactions/IgniteTxHandler.java   |  13 +-
 .../transactions/IgniteTxLocalAdapter.java    |   3 +-
 .../cache/transactions/IgniteTxManager.java   |   7 +-
 ...GridCacheMissingCommitVersionSelfTest.java |   6 +-
 ...gniteTxStoreExceptionAbstractSelfTest.java |   1 +
 14 files changed, 161 insertions(+), 218 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 503b3348e51b4..d26031ceb5978 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
@@ -854,24 +855,30 @@ else if (node.version().compareToIgnoreTimestamp(GridDhtPartitionsAbstractMessag
                     ready = cacheCtx.started();
 
                 if (ready) {
-                    GridDhtPartitionFullMap locMap = cacheCtx.topology().partitionMap(true);
+                    GridAffinityAssignmentCache affCache = cacheCtx.affinity().affinityCache();
 
-                    if (useOldApi) {
-                        locMap = new GridDhtPartitionFullMap(locMap.nodeId(),
-                            locMap.nodeOrder(),
-                            locMap.updateSequence(),
-                            locMap);
-                    }
+                    if (affCache != null) {
+                        GridDhtPartitionFullMap locMap = cacheCtx.topology().partitionMap(true);
+
+                        if (useOldApi) {
+                            locMap = new GridDhtPartitionFullMap(locMap.nodeId(),
+                                locMap.nodeOrder(),
+                                locMap.updateSequence(),
+                                locMap);
+                        }
 
-                    addFullPartitionsMap(m,
-                        dupData,
-                        compress,
-                        cacheCtx.cacheId(),
-                        locMap,
-                        cacheCtx.affinity().affinityCache().similarAffinityKey());
+                        addFullPartitionsMap(m,
+                            dupData,
+                            compress,
+                            cacheCtx.cacheId(),
+                            locMap,
+                            affCache.similarAffinityKey());
 
-                    if (exchId != null)
-                        m.addPartitionUpdateCounters(cacheCtx.cacheId(), cacheCtx.topology().updateCounters(true));
+                        if (exchId != null)
+                            m.addPartitionUpdateCounters(cacheCtx.cacheId(), cacheCtx.topology().updateCounters(true));
+                    }
+                    else
+                        assert cctx.cacheContext(cacheCtx.cacheId()) == null : cacheCtx.name();
                 }
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 0e0d76941ae49..cb9622514817e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1761,7 +1761,7 @@ private void stopGateway(DynamicCacheChangeRequest req) {
     /**
      * @param req Stop request.
      */
-    public void prepareCacheStop(DynamicCacheChangeRequest req) {
+    private void prepareCacheStop(DynamicCacheChangeRequest req) {
         assert req.stop() || req.close() : req;
 
         GridCacheAdapter<?, ?> cache = caches.remove(maskNull(req.cacheName()));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 317820362ff7b..fef6ddd5c40c3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -928,9 +928,13 @@ public static String txString(@Nullable IgniteInternalTx tx) {
         if (tx == null)
             return "null";
 
-        return tx.getClass().getSimpleName() + "[id=" + tx.xid() + ", concurrency=" + tx.concurrency() +
-            ", isolation=" + tx.isolation() + ", state=" + tx.state() + ", invalidate=" + tx.isInvalidate() +
-            ", rollbackOnly=" + tx.isRollbackOnly() + ", nodeId=" + tx.nodeId() +
+        return tx.getClass().getSimpleName() + "[id=" + tx.xid() +
+            ", concurrency=" + tx.concurrency() +
+            ", isolation=" + tx.isolation() +
+            ", state=" + tx.state() +
+            ", invalidate=" + tx.isInvalidate() +
+            ", rollbackOnly=" + tx.isRollbackOnly() +
+            ", nodeId=" + tx.nodeId() +
             ", duration=" + (U.currentTimeMillis() - tx.startTime()) + ']';
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index 4adfa8bf04f52..68c0e575d59f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -386,7 +386,7 @@ private void doneRemote(IgniteTxEntry txEntry,
         // If another thread is doing prepare or rollback.
         if (!state(PREPARING)) {
             // In optimistic mode prepare may be called multiple times.
-            if(state() != PREPARING || !optimistic()) {
+            if (state() != PREPARING || !optimistic()) {
                 if (log.isDebugEnabled())
                     log.debug("Invalid transaction state for prepare: " + this);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index ac2ab41049fe7..147cbea72077d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -33,8 +33,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
-import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
-import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.util.future.GridCompoundIdentityFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -94,9 +92,6 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
     /** Near mappings. */
     private Map<UUID, GridDistributedTxMapping> nearMap;
 
-    /** Trackable flag. */
-    private boolean trackable = true;
-
     /**
      * @param cctx Context.
      * @param tx Transaction.
@@ -151,46 +146,24 @@ public GridDhtTxLocalAdapter tx() {
 
     /** {@inheritDoc} */
     @Override public boolean trackable() {
-        return trackable;
+        return true;
     }
 
     /** {@inheritDoc} */
     @Override public void markNotTrackable() {
-        trackable = false;
+        assert false;
     }
 
     /**
      * @param e Error.
      */
-    public void onError(Throwable e) {
-        if (ERR_UPD.compareAndSet(this, null, e)) {
-            boolean marked = tx.setRollbackOnly();
-
-            if (e instanceof IgniteTxRollbackCheckedException) {
-                if (marked) {
-                    try {
-                        tx.rollback();
-                    }
-                    catch (IgniteCheckedException ex) {
-                        U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
-                    }
-                }
-            }
-            else if (tx.isSystemInvalidate()) { // Invalidate remote transactions on heuristic error.
-                finish();
+    public void rollbackOnError(Throwable e) {
+        assert e != null;
 
-                try {
-                    get();
-                }
-                catch (IgniteTxHeuristicCheckedException ignore) {
-                    // Future should complete with GridCacheTxHeuristicException.
-                }
-                catch (IgniteCheckedException err) {
-                    U.error(log, "Failed to invalidate transaction: " + tx, err);
-                }
-            }
+        if (ERR_UPD.compareAndSet(this, null, e)) {
+            tx.setRollbackOnly();
 
-            onComplete();
+            finish(false);
         }
     }
 
@@ -240,12 +213,21 @@ public void onResult(UUID nodeId, GridDhtTxFinishResponse res) {
     /** {@inheritDoc} */
     @Override public boolean onDone(IgniteInternalTx tx, Throwable err) {
         if (initialized() || err != null) {
-            if (this.tx.onePhaseCommit() && (this.tx.state() == COMMITTING))
-                this.tx.tmFinish(err == null);
-
             Throwable e = this.err;
 
-            if (e == null && commit)
+            if (this.tx.onePhaseCommit() && (this.tx.state() == COMMITTING)) {
+                try {
+                    this.tx.tmFinish(err == null);
+                }
+                catch (IgniteCheckedException finishErr) {
+                    U.error(log, "Failed to finish tx: " + tx, e);
+
+                    if (e == null)
+                        e = finishErr;
+                }
+            }
+
+            if (commit && e == null)
                 e = this.tx.commitError();
 
             Throwable finishErr = e != null ? e : err;
@@ -255,7 +237,7 @@ public void onResult(UUID nodeId, GridDhtTxFinishResponse res) {
                     finishErr = this.tx.commitError();
 
                 if (this.tx.syncMode() != PRIMARY_SYNC)
-                    this.tx.sendFinishReply(commit, finishErr);
+                    this.tx.sendFinishReply(finishErr);
 
                 // Don't forget to clean up.
                 cctx.mvcc().removeFuture(futId);
@@ -284,13 +266,15 @@ private void onComplete() {
 
     /**
      * Initializes future.
+     *
+     * @param commit Commit flag.
      */
     @SuppressWarnings({"SimplifiableIfStatement", "IfMayBeConditional"})
-    public void finish() {
+    public void finish(boolean commit) {
         boolean sync;
 
         if (!F.isEmpty(dhtMap) || !F.isEmpty(nearMap))
-            sync = finish(dhtMap, nearMap);
+            sync = finish(commit, dhtMap, nearMap);
         else if (!commit && !F.isEmpty(tx.lockTransactionNodes()))
             sync = rollbackLockTransactions(tx.lockTransactionNodes());
         else
@@ -308,7 +292,6 @@ else if (!commit && !F.isEmpty(tx.lockTransactionNodes()))
      * @return {@code True} in case there is at least one synchronous {@code MiniFuture} to wait for.
      */
     private boolean rollbackLockTransactions(Collection<ClusterNode> nodes) {
-        assert !commit;
         assert !F.isEmpty(nodes);
 
         if (tx.onePhaseCommit())
@@ -337,7 +320,7 @@ private boolean rollbackLockTransactions(Collection<ClusterNode> nodes) {
                 tx.commitVersion(),
                 tx.threadId(),
                 tx.isolation(),
-                commit,
+                false,
                 tx.isInvalidate(),
                 tx.system(),
                 tx.ioPolicy(),
@@ -390,11 +373,14 @@ private boolean rollbackLockTransactions(Collection<ClusterNode> nodes) {
     }
 
     /**
+     * @param commit Commit flag.
      * @param dhtMap DHT map.
      * @param nearMap Near map.
      * @return {@code True} in case there is at least one synchronous {@code MiniFuture} to wait for.
      */
-    private boolean finish(Map<UUID, GridDistributedTxMapping> dhtMap, Map<UUID, GridDistributedTxMapping> nearMap) {
+    private boolean finish(boolean commit,
+        Map<UUID, GridDistributedTxMapping> dhtMap,
+        Map<UUID, GridDistributedTxMapping> nearMap) {
         if (tx.onePhaseCommit())
             return false;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
index b659abb8ef679..4e39e9b54bf05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
@@ -503,52 +503,57 @@ public IgniteInternalFuture<GridNearTxPrepareResponse> prepareAsync(
     }
 
     /**
+     * @param commit Commit flag.
      * @param prepFut Prepare future.
      * @param fut Finish future.
      */
-    private void finishCommit(@Nullable IgniteInternalFuture prepFut, GridDhtTxFinishFuture fut) {
+    private void finishTx(boolean commit, @Nullable IgniteInternalFuture prepFut, GridDhtTxFinishFuture fut) {
+        assert prepFut == null || prepFut.isDone();
+
         boolean primarySync = syncMode() == PRIMARY_SYNC;
 
         IgniteCheckedException err = null;
 
-        try {
-            if (prepFut != null)
-                prepFut.get(); // Check for errors.
-
-            if (finish(true)) {
-                if (primarySync)
-                    sendFinishReply(true, null);
-
-                fut.finish();
+        if (!commit && prepFut != null) {
+            try {
+                prepFut.get();
             }
-            else {
-                err = new IgniteCheckedException("Failed to commit transaction: " + CU.txString(this));
-
-                fut.onError(err);
+            catch (IgniteCheckedException e) {
+                if (log.isDebugEnabled())
+                    log.debug("Failed to prepare transaction [tx=" + this + ", e=" + e + ']');
+            }
+            finally {
+                prepFut = null;
             }
         }
-        catch (IgniteTxOptimisticCheckedException e) {
-            if (log.isDebugEnabled())
-                log.debug("Failed to optimistically prepare transaction [tx=" + this + ", e=" + e + ']');
 
-            err = e;
+        try {
+            if (prepFut != null)
+                prepFut.get(); // Check for errors.
 
-            fut.onError(e);
+            boolean finished = finish(commit);
+
+            if (!finished)
+                err = new IgniteCheckedException("Failed to finish transaction [commit=" + commit +
+                    ", tx=" + CU.txString(this) + ']');
         }
         catch (IgniteCheckedException e) {
-            U.error(log, "Failed to prepare transaction: " + this, e);
+            U.error(log, "Failed to finish transaction [commit=" + commit + ", tx=" + this + ']', e);
 
             err = e;
-
-            fut.onError(e);
         }
 
-        if (primarySync && err != null)
-            sendFinishReply(true, err);
+        if (primarySync)
+            sendFinishReply(err);
+
+        if (err != null)
+            fut.rollbackOnError(err);
+        else
+            fut.finish(commit);
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings({"ThrowableInstanceNeverThrown"})
+    @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<IgniteInternalTx> commitAsync() {
         if (log.isDebugEnabled())
             log.debug("Committing dht local tx: " + this);
@@ -557,7 +562,7 @@ private void finishCommit(@Nullable IgniteInternalFuture prepFut, GridDhtTxFinis
         if (pessimistic())
             prepareAsync();
 
-        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, /*commit*/true);
+        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, true);
 
         cctx.mvcc().addFuture(fut, fut.futureId());
 
@@ -565,11 +570,11 @@ private void finishCommit(@Nullable IgniteInternalFuture prepFut, GridDhtTxFinis
 
         if (prep != null) {
             if (prep.isDone())
-                finishCommit(prep, fut);
+                finishTx(true, prep, fut);
             else {
                 prep.listen(new CI1<IgniteInternalFuture<?>>() {
                     @Override public void apply(IgniteInternalFuture<?> f) {
-                        finishCommit(f, fut);
+                        finishTx(true, f, fut);
                     }
                 });
             }
@@ -577,7 +582,7 @@ private void finishCommit(@Nullable IgniteInternalFuture prepFut, GridDhtTxFinis
         else {
             assert optimistic();
 
-            finishCommit(null, fut);
+            finishTx(true, null, fut);
         }
 
         return fut;
@@ -590,70 +595,26 @@ private void finishCommit(@Nullable IgniteInternalFuture prepFut, GridDhtTxFinis
         PREP_FUT_UPD.compareAndSet(this, fut, null);
     }
 
-    /**
-     * @param prepFut Prepare future.
-     * @param fut Finish future.
-     */
-    private void finishRollback(@Nullable IgniteInternalFuture prepFut, GridDhtTxFinishFuture fut) {
-        try {
-            if (prepFut != null)
-                prepFut.get();
-        }
-        catch (IgniteCheckedException e) {
-            if (log.isDebugEnabled())
-                log.debug("Failed to prepare or rollback transaction [tx=" + this + ", e=" + e + ']');
-        }
-
-        boolean primarySync = syncMode() == PRIMARY_SYNC;
-
-        IgniteCheckedException err = null;
-
-        try {
-            if (finish(false) || state() == UNKNOWN) {
-                if (primarySync)
-                    sendFinishReply(false, null);
-
-                fut.finish();
-            }
-            else {
-                err = new IgniteCheckedException("Failed to rollback transaction: " +
-                    CU.txString(GridDhtTxLocal.this));
-
-                fut.onError(err);
-            }
-        }
-        catch (IgniteCheckedException e) {
-            U.error(log, "Failed to gracefully rollback transaction: " + CU.txString(GridDhtTxLocal.this),
-                e);
-
-            err = e;
-
-            fut.onError(e);
-        }
-
-        if (primarySync && err != null)
-            sendFinishReply(false, err);
-    }
-
     /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<IgniteInternalTx> rollbackAsync() {
-        GridDhtTxPrepareFuture prepFut = this.prepFut;
-
-        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, /*rollback*/false);
+        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, false);
 
         cctx.mvcc().addFuture(fut, fut.futureId());
 
+        GridDhtTxPrepareFuture prepFut = this.prepFut;
+
         if (prepFut != null) {
             prepFut.complete();
 
             prepFut.listen(new CI1<IgniteInternalFuture<?>>() {
                 @Override public void apply(IgniteInternalFuture<?> f) {
-                    finishRollback(f, fut);
+                    finishTx(false, f, fut);
                 }
             });
         }
         else
-            finishRollback(null, fut);
+            finishTx(false, null, fut);
 
         return fut;
     }
@@ -672,7 +633,7 @@ private void finishRollback(@Nullable IgniteInternalFuture prepFut, GridDhtTxFin
     }
 
     /** {@inheritDoc} */
-    @Override protected void sendFinishReply(boolean commit, @Nullable Throwable err) {
+    @Override protected void sendFinishReply(@Nullable Throwable err) {
         if (nearFinFutId != null) {
             if (nearNodeId.equals(cctx.localNodeId())) {
                 if (log.isDebugEnabled())
@@ -701,8 +662,8 @@ private void finishRollback(@Nullable IgniteInternalFuture prepFut, GridDhtTxFin
                 }
             }
             catch (Throwable ex) {
-                U.error(log, "Failed to send finish response to node (transaction was " +
-                    (commit ? "committed" : "rolledback") + ") [txId=" + nearXidVersion() +
+                U.error(log, "Failed to send finish response to node [txId=" + nearXidVersion() +
+                    ", txState=" + state() +
                     ", dhtTxId=" + xidVersion() +
                     ", node=" + nearNodeId +
                     ", res=" + res + ']', ex);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
index 35dfb62c5fc98..1d88d84a13528 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
@@ -236,10 +236,9 @@ public void explicitLock(boolean explicitLock) {
         AffinityTopologyVersion topVer);
 
     /**
-     * @param commit Commit flag.
      * @param err Error, if any.
      */
-    protected abstract void sendFinishReply(boolean commit, @Nullable Throwable err);
+    protected abstract void sendFinishReply(@Nullable Throwable err);
 
     /** {@inheritDoc} */
     @Override public boolean needsCompletedVersions() {
@@ -249,7 +248,7 @@ public void explicitLock(boolean explicitLock) {
     /**
      * @return Versions for all pending locks that were in queue before tx locks were released.
      */
-    public Collection<GridCacheVersion> pendingVersions() {
+    Collection<GridCacheVersion> pendingVersions() {
         return pendingVers == null ? Collections.<GridCacheVersion>emptyList() : pendingVers;
     }
 
@@ -726,7 +725,7 @@ private IgniteInternalFuture<GridCacheReturn> obtainLockAsync(
                         /*read*/read,
                         accessTtl,
                         filter == null ? CU.empty0() : filter,
-                        /**computeInvoke*/false);
+                        /*computeInvoke*/false);
 
                     return ret;
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 54bd5436c636b..9acab56ae2d8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -298,34 +298,16 @@ else if (fut.getClass() == CheckRemoteTxMiniFuture.class) {
             if (isDone())
                 return false;
 
-            if (err != null) {
-                tx.commitError(err);
-
-                boolean marked = tx.setRollbackOnly();
-
-                if (err instanceof IgniteTxRollbackCheckedException) {
-                    if (marked) {
-                        try {
-                            tx.rollback();
-                        }
-                        catch (IgniteCheckedException ex) {
-                            U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
-                        }
-                    }
-                }
-                else if (tx.implicit() && tx.isSystemInvalidate()) { // Finish implicit transaction on heuristic error.
-                    try {
-                        tx.close();
-                    }
-                    catch (IgniteCheckedException ex) {
-                        U.error(log, "Failed to invalidate transaction: " + tx, ex);
-                    }
-                }
+            if (err != null)
+                tx.setRollbackOnly();
+
+            if (commit) {
+                if (tx.commitError() != null)
+                    err = tx.commitError();
+                else if (err != null)
+                    tx.commitError(err);
             }
 
-            if (commit && tx.commitError() != null)
-                err = tx.commitError();
-
             if (initialized() || err != null) {
                 if (tx.needCheckBackup()) {
                     assert tx.onePhaseCommit();
@@ -349,7 +331,15 @@ else if (tx.implicit() && tx.isSystemInvalidate()) { // Finish implicit transact
 
                     finishOnePhase(commit);
 
-                    tx.tmFinish(commit);
+                    try {
+                        tx.tmFinish(commit);
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to finish tx: " + tx, e);
+
+                        if (err == null)
+                            err = e;
+                    }
                 }
 
                 if (super.onDone(tx0, err)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index ed37059b20cb8..0730300732da1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -231,7 +231,7 @@ public GridNearTxLocal(
     }
 
     /** {@inheritDoc} */
-    @Override protected void sendFinishReply(boolean commit, @Nullable Throwable err) {
+    @Override protected void sendFinishReply(@Nullable Throwable err) {
         // We are in near transaction, do not send finish reply to local node.
     }
 
@@ -1062,50 +1062,48 @@ public IgniteInternalFuture<IgniteInternalTx> commitAsyncLocal() {
             return new GridFinishedFuture<IgniteInternalTx>(this);
         }
 
-        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, /*commit*/true);
+        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, true);
 
         cctx.mvcc().addFuture(fut, fut.futureId());
 
         if (prep == null || prep.isDone()) {
             assert prep != null || optimistic();
 
+            IgniteCheckedException err = null;
+
             try {
                 if (prep != null)
                     prep.get(); // Check for errors of a parent future.
-
-                fut.finish();
-            }
-            catch (IgniteTxOptimisticCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Failed optimistically to prepare transaction [tx=" + this + ", e=" + e + ']');
-
-                fut.onError(e);
             }
             catch (IgniteCheckedException e) {
-                U.error(log, "Failed to prepare transaction: " + this, e);
+                err = e;
 
-                fut.onError(e);
+                U.error(log, "Failed to prepare transaction: " + this, e);
             }
+
+            if (err != null)
+                fut.rollbackOnError(err);
+            else
+                fut.finish(true);
         }
         else
             prep.listen(new CI1<IgniteInternalFuture<?>>() {
                 @Override public void apply(IgniteInternalFuture<?> f) {
+                    IgniteCheckedException err = null;
+
                     try {
                         f.get(); // Check for errors of a parent future.
-
-                        fut.finish();
-                    }
-                    catch (IgniteTxOptimisticCheckedException e) {
-                        if (log.isDebugEnabled())
-                            log.debug("Failed optimistically to prepare transaction [tx=" + this + ", e=" + e + ']');
-
-                        fut.onError(e);
                     }
                     catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to prepare transaction: " + this, e);
+                        err = e;
 
-                        fut.onError(e);
+                        U.error(log, "Failed to prepare transaction: " + this, e);
                     }
+
+                    if (err != null)
+                        fut.rollbackOnError(err);
+                    else
+                        fut.finish(true);
                 }
             });
 
@@ -1121,7 +1119,7 @@ public IgniteInternalFuture<IgniteInternalTx> rollbackAsyncLocal() {
         if (log.isDebugEnabled())
             log.debug("Rolling back colocated tx locally: " + this);
 
-        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, /*commit*/false);
+        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, false);
 
         cctx.mvcc().addFuture(fut, fut.futureId());
 
@@ -1138,7 +1136,7 @@ public IgniteInternalFuture<IgniteInternalTx> rollbackAsyncLocal() {
                         e.getMessage() + ']');
             }
 
-            fut.finish();
+            fut.finish(false);
         }
         else
             prep.listen(new CI1<IgniteInternalFuture<?>>() {
@@ -1151,7 +1149,7 @@ public IgniteInternalFuture<IgniteInternalTx> rollbackAsyncLocal() {
                             e.getMessage() + ']');
                     }
 
-                    fut.finish();
+                    fut.finish(false);
                 }
             });
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 4b9907967d273..b6e3c48d18aac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -472,13 +472,8 @@ private IgniteInternalFuture<GridNearTxPrepareResponse> prepareNearTx(
                 req.last());
 
             if (tx.isRollbackOnly() && !tx.commitOnPrepare()) {
-                try {
-                    if (tx.state() != TransactionState.ROLLED_BACK && tx.state() != TransactionState.ROLLING_BACK)
-                        tx.rollback();
-                }
-                catch (IgniteCheckedException e) {
-                    U.error(log, "Failed to rollback transaction: " + tx, e);
-                }
+                if (tx.state() != TransactionState.ROLLED_BACK && tx.state() != TransactionState.ROLLING_BACK)
+                    tx.rollbackAsync();
             }
 
             final GridDhtTxLocal tx0 = tx;
@@ -872,7 +867,7 @@ private IgniteInternalFuture<IgniteInternalTx> finishDhtLocal(UUID nodeId, @Null
 
             U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e);
 
-            IgniteInternalFuture<IgniteInternalTx> res = null;
+            IgniteInternalFuture<IgniteInternalTx> res;
 
             IgniteInternalFuture<IgniteInternalTx> rollbackFut = tx.rollbackAsync();
 
@@ -884,7 +879,7 @@ private IgniteInternalFuture<IgniteInternalTx> finishDhtLocal(UUID nodeId, @Null
             if (e instanceof Error)
                 throw (Error)e;
 
-            return res == null ? new GridFinishedFuture<IgniteInternalTx>(e) : res;
+            return res;
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 393fb1ac4b958..8d0a2b105bf89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1000,8 +1000,9 @@ assert ownsLock(txEntry.cached()):
      * Commits transaction to transaction manager. Used for one-phase commit transactions only.
      *
      * @param commit If {@code true} commits transaction, otherwise rollbacks.
+     * @throws IgniteCheckedException If failed.
      */
-    public void tmFinish(boolean commit) {
+    public void tmFinish(boolean commit) throws IgniteCheckedException {
         assert onePhaseCommit();
 
         if (DONE_FLAG_UPD.compareAndSet(this, 0, 1)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index e2e9868093d38..036fb0a562d44 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -1191,8 +1191,9 @@ private boolean unlockReadEntries(IgniteInternalTx tx) {
      * Commits a transaction.
      *
      * @param tx Transaction to commit.
+     * @throws IgniteCheckedException If failed.
      */
-    public void commitTx(IgniteInternalTx tx) {
+    public void commitTx(IgniteInternalTx tx) throws IgniteCheckedException {
         assert tx != null;
         assert tx.state() == COMMITTING : "Invalid transaction state for commit from tm [state=" + tx.state() +
             ", expected=COMMITTING, tx=" + tx + ']';
@@ -1210,12 +1211,12 @@ public void commitTx(IgniteInternalTx tx) {
         Boolean committed = committed0 != null && !committed0.equals(Boolean.FALSE);
 
         // 1. Make sure that committed version has been recorded.
-        if (!((committed != null && committed) || tx.writeSet().isEmpty() || tx.isSystemInvalidate())) {
+        if (!(committed || tx.writeSet().isEmpty() || tx.isSystemInvalidate())) {
             uncommitTx(tx);
 
             tx.errorWhenCommitting();
 
-            throw new IgniteException("Missing commit version (consider increasing " +
+            throw new IgniteCheckedException("Missing commit version (consider increasing " +
                 IGNITE_MAX_COMPLETED_TX_COUNT + " system property) [ver=" + tx.xidVersion() +
                 ", tx=" + tx.getClass().getSimpleName() + ']');
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMissingCommitVersionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMissingCommitVersionSelfTest.java
index 19e49f3995797..ac56d18acc72f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMissingCommitVersionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMissingCommitVersionSelfTest.java
@@ -43,7 +43,7 @@ public class GridCacheMissingCommitVersionSelfTest extends GridCommonAbstractTes
     private volatile boolean putFailed;
 
     /** */
-    private String maxCompletedTxCount;
+    private String maxCompletedTxCnt;
 
     /**
      */
@@ -53,7 +53,7 @@ public GridCacheMissingCommitVersionSelfTest() {
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration() throws Exception {
-        maxCompletedTxCount = System.getProperty(IGNITE_MAX_COMPLETED_TX_COUNT);
+        maxCompletedTxCnt = System.getProperty(IGNITE_MAX_COMPLETED_TX_COUNT);
 
         System.setProperty(IGNITE_MAX_COMPLETED_TX_COUNT, String.valueOf(5));
 
@@ -78,7 +78,7 @@ public GridCacheMissingCommitVersionSelfTest() {
 
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
-        System.setProperty(IGNITE_MAX_COMPLETED_TX_COUNT, maxCompletedTxCount != null ? maxCompletedTxCount : "");
+        System.setProperty(IGNITE_MAX_COMPLETED_TX_COUNT, maxCompletedTxCnt != null ? maxCompletedTxCnt : "");
 
         super.afterTest();
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java
index b65b4419ef2eb..795ab81c3763f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java
@@ -357,6 +357,7 @@ private void checkPutTx(boolean putBefore, TransactionConcurrency concurrency,
 
     /**
      * @param key Key.
+     * @param putBefore If {@code true} expects non-null values.
      * @throws Exception If failed.
      */
     private void checkValue(final Integer key, boolean putBefore) throws Exception {

From 147277d89fcdbd134425b74fc0b63596572a4af1 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Mon, 19 Dec 2016 14:49:25 +0300
Subject: [PATCH 058/446] ignite-4371 Neat TX finish request processing may
 fall into sync wait of dht finish response - enabled striped pool usage for
 GridDistributedTxFinishResponse - convert exceptions from
 CacheStoreSessionListener.onSessionStart into IgniteCheckedException - should
 not use IgniteTxLocalAdapter.commitErr for rollback exceptions (otherwise
 cyclic reference in exception->cause is possible, this leads to infinite loop
 in 'X.hasCause')

(cherry picked from commit cf7b6a0)
---
 .../GridDistributedTxFinishResponse.java      |  6 ----
 .../cache/store/CacheStoreManager.java        |  4 ++-
 .../store/GridCacheStoreManagerAdapter.java   | 34 +++++++++++--------
 .../store/GridCacheWriteBehindStore.java      |  6 ++--
 .../transactions/IgniteTxLocalAdapter.java    | 27 ++++++---------
 5 files changed, 36 insertions(+), 41 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
index c5cf33242b9b3..109d665704dda 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
@@ -85,12 +85,6 @@ public IgniteUuid futureId() {
         return ctx.txFinishMessageLogger();
     }
 
-    /** {@inheritDoc} */
-    @Override public int partition() {
-        // TODO https://issues.apache.org/jira/browse/IGNITE-4371
-        return Integer.MIN_VALUE;
-    }
-
     /** {@inheritDoc} */
     @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
         writer.setBuffer(buf);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
index 67c9334fa6ed2..8d6b63dce3c8a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
@@ -171,8 +171,10 @@ public boolean removeAll(@Nullable IgniteInternalTx tx, Collection<Object> keys)
 
     /**
      * End session initiated by write-behind store.
+     *
+     * @throws IgniteCheckedException If failed.
      */
-    public void writeBehindSessionInit();
+    public void writeBehindSessionInit() throws IgniteCheckedException;
 
     /**
      * End session initiated by write-behind store.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 024375e75c477..8b93afa431c9e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -288,7 +288,7 @@ private CacheStore cacheStoreWrapper(GridKernalContext ctx,
     }
 
     /** {@inheritDoc} */
-    @Override @Nullable public Object load(@Nullable IgniteInternalTx tx, KeyCacheObject key)
+    @Override @Nullable public final Object load(@Nullable IgniteInternalTx tx, KeyCacheObject key)
         throws IgniteCheckedException {
         return loadFromStore(tx, key, true);
     }
@@ -377,7 +377,7 @@ private Object convert(Object val) {
     }
 
     /** {@inheritDoc} */
-    @Override public void localStoreLoadAll(@Nullable IgniteInternalTx tx, Collection keys, GridInClosure3 vis)
+    @Override public final void localStoreLoadAll(@Nullable IgniteInternalTx tx, Collection keys, GridInClosure3 vis)
         throws IgniteCheckedException {
         assert store != null;
         assert locStore;
@@ -386,7 +386,7 @@ private Object convert(Object val) {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean loadAll(@Nullable IgniteInternalTx tx, Collection keys, IgniteBiInClosure vis)
+    @Override public final boolean loadAll(@Nullable IgniteInternalTx tx, Collection keys, IgniteBiInClosure vis)
         throws IgniteCheckedException {
         if (store != null) {
             loadAllFromStore(tx, keys, vis, null);
@@ -499,7 +499,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public boolean loadCache(final GridInClosure3 vis, Object[] args) throws IgniteCheckedException {
+    @Override public final boolean loadCache(final GridInClosure3 vis, Object[] args) throws IgniteCheckedException {
         if (store != null) {
             if (log.isDebugEnabled())
                 log.debug("Loading all values from store.");
@@ -554,7 +554,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public boolean put(@Nullable IgniteInternalTx tx, Object key, Object val, GridCacheVersion ver)
+    @Override public final boolean put(@Nullable IgniteInternalTx tx, Object key, Object val, GridCacheVersion ver)
         throws IgniteCheckedException {
         if (store != null) {
             // Never persist internal keys.
@@ -599,7 +599,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public boolean putAll(@Nullable IgniteInternalTx tx, Map map) throws IgniteCheckedException {
+    @Override public final boolean putAll(@Nullable IgniteInternalTx tx, Map map) throws IgniteCheckedException {
         if (F.isEmpty(map))
             return true;
 
@@ -658,7 +658,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public boolean remove(@Nullable IgniteInternalTx tx, Object key) throws IgniteCheckedException {
+    @Override public final boolean remove(@Nullable IgniteInternalTx tx, Object key) throws IgniteCheckedException {
         if (store != null) {
             // Never remove internal key from store as it is never persisted.
             if (key instanceof GridCacheInternal)
@@ -701,7 +701,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public boolean removeAll(@Nullable IgniteInternalTx tx, Collection keys) throws IgniteCheckedException {
+    @Override public final boolean removeAll(@Nullable IgniteInternalTx tx, Collection keys) throws IgniteCheckedException {
         if (F.isEmpty(keys))
             return true;
 
@@ -763,7 +763,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException {
+    @Override public final void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException {
         assert store != null;
 
         sessionInit0(tx);
@@ -807,7 +807,7 @@ private void handleClassCastException(ClassCastException e) throws IgniteChecked
     }
 
     /** {@inheritDoc} */
-    @Override public void writeBehindSessionInit() {
+    @Override public void writeBehindSessionInit() throws IgniteCheckedException {
         sessionInit0(null);
     }
 
@@ -818,8 +818,9 @@ private void handleClassCastException(ClassCastException e) throws IgniteChecked
 
     /**
      * @param tx Current transaction.
+     * @throws IgniteCheckedException If failed.
      */
-    private void sessionInit0(@Nullable IgniteInternalTx tx) {
+    private void sessionInit0(@Nullable IgniteInternalTx tx) throws IgniteCheckedException {
         assert sesHolder != null;
 
         SessionData ses;
@@ -841,9 +842,14 @@ private void sessionInit0(@Nullable IgniteInternalTx tx) {
 
         sesHolder.set(ses);
 
-        if (sesLsnrs != null && !ses.started(this)) {
-            for (CacheStoreSessionListener lsnr : sesLsnrs)
-                lsnr.onSessionStart(locSes);
+        try {
+            if (sesLsnrs != null && !ses.started(this)) {
+                for (CacheStoreSessionListener lsnr : sesLsnrs)
+                    lsnr.onSessionStart(locSes);
+            }
+        }
+        catch (Exception e) {
+            throw new IgniteCheckedException("Failed to start store session: " + e, e);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
index 858d9a7498510..f5c0c12741420 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
@@ -673,10 +673,10 @@ private boolean updateStore(StoreOperation operation,
         Map<K, Entry<? extends K, ? extends  V>> vals,
         boolean initSes) {
 
-        if (initSes && storeMgr != null)
-            storeMgr.writeBehindSessionInit();
-
         try {
+            if (initSes && storeMgr != null)
+                storeMgr.writeBehindSessionInit();
+
             boolean threwEx = true;
 
             try {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 8d0a2b105bf89..9f794b4fa6285 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1063,7 +1063,7 @@ public Collection<GridCacheVersion> rolledbackVersions() {
             setRollbackOnly();
 
             throw new IgniteCheckedException("Invalid transaction state for rollback [state=" + state +
-                ", tx=" + this + ']', commitErr);
+                ", tx=" + this + ']');
         }
 
         if (near()) {
@@ -1074,28 +1074,21 @@ public Collection<GridCacheVersion> rolledbackVersions() {
         }
 
         if (DONE_FLAG_UPD.compareAndSet(this, 0, 1)) {
-            try {
-                cctx.tm().rollbackTx(this);
+            cctx.tm().rollbackTx(this);
 
-                if (!internal()) {
-                    Collection<CacheStoreManager> stores = txState.stores(cctx);
+            if (!internal()) {
+                Collection<CacheStoreManager> stores = txState.stores(cctx);
 
-                    if (stores != null && !stores.isEmpty()) {
-                        assert isWriteToStoreFromDhtValid(stores) :
-                            "isWriteToStoreFromDht can't be different within one transaction";
+                if (stores != null && !stores.isEmpty()) {
+                    assert isWriteToStoreFromDhtValid(stores) :
+                        "isWriteToStoreFromDht can't be different within one transaction";
 
-                        boolean isWriteToStoreFromDht = F.first(stores).isWriteToStoreFromDht();
+                    boolean isWriteToStoreFromDht = F.first(stores).isWriteToStoreFromDht();
 
-                        if (stores != null && !stores.isEmpty() && (near() || isWriteToStoreFromDht))
-                            sessionEnd(stores, false);
-                    }
+                    if (!stores.isEmpty() && (near() || isWriteToStoreFromDht))
+                        sessionEnd(stores, false);
                 }
             }
-            catch (Error | IgniteCheckedException | RuntimeException e) {
-                U.addLastCause(e, commitErr, log);
-
-                throw e;
-            }
         }
     }
 

From 09c634e28200e0dd94c3c2e2271b5cdf60304888 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Mon, 19 Dec 2016 15:55:02 +0300
Subject: [PATCH 059/446] Merge remote-tracking branch
 'remotes/community/ignite-1.7.4' into ignite-1.8.2

# Conflicts:
#	modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
#	modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
#	modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
#	modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
#	modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
#	modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
#	modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
#	modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
---
 .../store/cassandra/CassandraCacheStore.java     |  3 +++
 .../org/apache/ignite/internal/IgniteKernal.java | 16 ----------------
 2 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
index b4bed0da874e1..70d798bae4daf 100644
--- a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
+++ b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
@@ -43,6 +43,9 @@
 import org.apache.ignite.cache.store.cassandra.session.ExecutionAssistant;
 import org.apache.ignite.cache.store.cassandra.session.GenericBatchExecutionAssistant;
 import org.apache.ignite.cache.store.cassandra.session.LoadCacheCustomQueryWorker;
+import org.apache.ignite.cache.store.cassandra.session.transaction.DeleteMutation;
+import org.apache.ignite.cache.store.cassandra.session.transaction.Mutation;
+import org.apache.ignite.cache.store.cassandra.session.transaction.WriteMutation;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiInClosure;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index e4a61153826f9..4972d1f306e2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2877,22 +2877,6 @@ public IgniteInternalFuture<?> destroyCachesAsync(Collection<String> cacheNames,
         }
     }
 
-    /**
-     * @param cacheNames Collection of cache names.
-     * @param checkThreadTx If {@code true} checks that current thread does not have active transactions.
-     * @return Ignite future.
-     */
-    public IgniteInternalFuture<?> destroyCachesAsync(Collection<String> cacheNames, boolean checkThreadTx) {
-        guard();
-
-        try {
-            return ctx.cache().dynamicDestroyCaches(cacheNames, checkThreadTx);
-        }
-        finally {
-            unguard();
-        }
-    }
-
     /** {@inheritDoc} */
     @Override public <K, V> IgniteCache<K, V> getOrCreateCache(String cacheName) {
         guard();

From e82eefe7a7d7f541e6d2fc4b166843b87ec9ef87 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 20 Dec 2016 12:35:51 +0300
Subject: [PATCH 060/446] Null argument check for cache.putAll.

(cherry picked from commit c0f1a21)
---
 .../ignite/internal/processors/cache/GridCacheAdapter.java      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 8187e8f256c15..f235f6a9e76d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2661,6 +2661,8 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
 
     /** {@inheritDoc} */
     @Override public void putAll(@Nullable final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        A.notNull(m, "map");
+
         if (F.isEmpty(m))
             return;
 

From 800579c9ca193f7660e35f241c255904c573eef1 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 20 Dec 2016 15:15:04 +0300
Subject: [PATCH 061/446] Fixed 'singleRmv' flag for cache.remove(k, v)
 operation in tx cache.

(cherry picked from commit 126ab60)
---
 .../ignite/internal/processors/cache/GridCacheAdapter.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index f235f6a9e76d2..3b4557d57daca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2948,6 +2948,7 @@ public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws
 
     /**
      * @param key Key.
+     * @param filter Filter.
      * @return {@code True} if entry was removed.
      * @throws IgniteCheckedException If failed.
      */
@@ -2959,7 +2960,7 @@ protected boolean remove0(final K key, final CacheEntryPredicate filter) throws
                     Collections.singletonList(key),
                     /*retval*/false,
                     filter,
-                    /*singleRmv*/true).get().success();
+                    /*singleRmv*/filter == null).get().success();
             }
 
             @Override public String toString() {

From e8fd1657de7847922feec3d3e58250ad116e4e47 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 20 Dec 2016 15:27:04 +0300
Subject: [PATCH 062/446] ignite-4379: fixed issue after merge with latest DML
 changes

---
 .../ignite/internal/processors/query/h2/IgniteH2Indexing.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index c54118530a9c0..903794310c3c4 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -801,8 +801,6 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
         Prepared p = GridSqlQueryParser.prepared((JdbcPreparedStatement)stmt);
 
         if (!p.isQuery()) {
-            GridH2QueryContext.clearThreadLocal();
-
             SqlFieldsQuery fldsQry = new SqlFieldsQuery(qry);
 
             if (params != null)

From ce83715123d9ac3c1939e154318fce25d387bfed Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Tue, 20 Dec 2016 15:32:39 +0300
Subject: [PATCH 063/446] IGNITE-4359 .NET: Support DateTime properties in LINQ

---
 .../Cache/Query/CacheLinqTest.cs              | 24 +++++++++----
 .../Impl/CacheQueryExpressionVisitor.cs       | 12 ++-----
 .../Apache.Ignite.Linq/Impl/MethodVisitor.cs  | 34 +++++++++++++++++--
 3 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheLinqTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheLinqTest.cs
index 1ac7fa75cfbe7..46979efc65092 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheLinqTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheLinqTest.cs
@@ -63,7 +63,7 @@ public class CacheLinqTest
         private bool _runDbConsole;
 
         /** */
-        private static readonly DateTime StartDateTime = new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+        private static readonly DateTime StartDateTime = new DateTime(2000, 5, 17, 15, 4, 5, DateTimeKind.Utc);
 
         /// <summary>
         /// Fixture set up.
@@ -111,7 +111,7 @@ public void FixtureSetUp()
 
             roleCache[new RoleKey(1, 101)] = new Role {Name = "Role_1", Date = StartDateTime};
             roleCache[new RoleKey(2, 102)] = new Role {Name = "Role_2", Date = StartDateTime.AddYears(1)};
-            roleCache[new RoleKey(3, 103)] = new Role {Name = null, Date = StartDateTime.AddYears(2)};
+            roleCache[new RoleKey(3, 103)] = new Role {Name = null, Date = StartDateTime.AddHours(5432)};
         }
 
         /// <summary>
@@ -123,7 +123,8 @@ private static IgniteConfiguration GetConfig(string gridName = null)
             {
                 BinaryConfiguration = new BinaryConfiguration(typeof(Person),
                     typeof(Organization), typeof(Address), typeof(Role), typeof(RoleKey), typeof(Numerics)),
-                GridName = gridName
+                GridName = gridName,
+                JvmOptions = { "-Duser.timezone=UTC" }
             };
         }
 
@@ -735,7 +736,7 @@ public void TestDateTime()
 
             // Test retrieval
             var dates = roles.OrderBy(x => x.Value.Date).Select(x => x.Value.Date);
-            var expDates = new[] {StartDateTime, StartDateTime.AddYears(1), StartDateTime.AddYears(2)};
+            var expDates = GetRoleCache().Select(x => x.Value.Date).OrderBy(x => x).ToArray();
             Assert.AreEqual(expDates, dates.ToArray());
 
             // Filtering
@@ -748,10 +749,21 @@ public void TestDateTime()
                 join person in persons on role.Value.Date equals person.Value.Birthday
                 select person;
 
-            Assert.AreEqual(RoleCount, join.Count());
+            Assert.AreEqual(2, join.Count());
 
             // Functions
-            Assert.AreEqual("01 01 2000 00:00:00", dates.Select(x => x.ToString("DD MM YYYY HH:mm:ss")).First());
+            var strings = dates.Select(x => x.ToString("dd MM YYYY HH:mm:ss")).ToArray();
+            Assert.AreEqual(new[] {"17 05 2000 15:04:05", "29 12 2000 23:04:05", "17 05 2001 15:04:05"}, strings);
+
+            // Properties
+            Assert.AreEqual(new[] {2000, 2000, 2001}, dates.Select(x => x.Year).ToArray());
+            Assert.AreEqual(new[] {5, 12, 5}, dates.Select(x => x.Month).ToArray());
+            Assert.AreEqual(new[] {17, 29, 17}, dates.Select(x => x.Day).ToArray());
+            Assert.AreEqual(expDates.Select(x => x.DayOfYear).ToArray(), dates.Select(x => x.DayOfYear).ToArray());
+            Assert.AreEqual(expDates.Select(x => x.DayOfWeek).ToArray(), dates.Select(x => x.DayOfWeek).ToArray());
+            Assert.AreEqual(new[] {15, 23, 15}, dates.Select(x => x.Hour).ToArray());
+            Assert.AreEqual(new[] { 4, 4, 4 }, dates.Select(x => x.Minute).ToArray());
+            Assert.AreEqual(new[] { 5, 5, 5 }, dates.Select(x => x.Second).ToArray());
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
index 8231053611682..1f9da1ceb0483 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
@@ -276,17 +276,9 @@ protected override Expression VisitMember(MemberExpression expression)
         {
             // Field hierarchy is flattened (Person.Address.Street is just Street), append as is, do not call Visit.
 
-            // Special case: string.Length
-            if (expression.Member == MethodVisitor.StringLength)
-            {
-                ResultBuilder.Append("length(");
-
-                VisitMember((MemberExpression) expression.Expression);
-
-                ResultBuilder.Append(")");
-
+            // Property call (string.Length, DateTime.Month, etc).
+            if (MethodVisitor.VisitPropertyCall(expression, this))
                 return expression;
-            }
 
             // Special case: grouping
             if (VisitGroupByMember(expression.Expression))
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/MethodVisitor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/MethodVisitor.cs
index 3c8acc71d2f70..e83c448a0992b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/MethodVisitor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/MethodVisitor.cs
@@ -29,8 +29,19 @@ namespace Apache.Ignite.Linq.Impl
     /// </summary>
     internal static class MethodVisitor
     {
-        /// <summary> The string length method. </summary>
-        public static readonly MemberInfo StringLength = typeof (string).GetProperty("Length");
+        /// <summary> Property visitors. </summary>
+        private static readonly Dictionary<MemberInfo, string> Properties = new Dictionary<MemberInfo, string>
+        {
+            {typeof(string).GetProperty("Length"), "length"},
+            {typeof(DateTime).GetProperty("Year"), "year"},
+            {typeof(DateTime).GetProperty("Month"), "month"},
+            {typeof(DateTime).GetProperty("Day"), "day_of_month"},
+            {typeof(DateTime).GetProperty("DayOfYear"), "day_of_year"},
+            {typeof(DateTime).GetProperty("DayOfWeek"), "-1 + day_of_week"},
+            {typeof(DateTime).GetProperty("Hour"), "hour"},
+            {typeof(DateTime).GetProperty("Minute"), "minute"},
+            {typeof(DateTime).GetProperty("Second"), "second"}
+        };
 
         /// <summary> Method visit delegate. </summary>
         private delegate void VisitMethodDelegate(MethodCallExpression expression, CacheQueryExpressionVisitor visitor);
@@ -102,6 +113,25 @@ internal static class MethodVisitor
             GetMathMethod("Truncate", typeof (decimal)),
         }.ToDictionary(x => x.Key, x => x.Value);
 
+        /// <summary>
+        /// Visits the property call expression.
+        /// </summary>
+        public static bool VisitPropertyCall(MemberExpression expression, CacheQueryExpressionVisitor visitor)
+        {
+            string funcName;
+
+            if (!Properties.TryGetValue(expression.Member, out funcName))
+                return false;
+
+            visitor.ResultBuilder.Append(funcName).Append('(');
+
+            visitor.Visit(expression.Expression);
+
+            visitor.ResultBuilder.Append(')');
+
+            return true;
+        }
+
         /// <summary>
         /// Visits the method call expression.
         /// </summary>

From d92b88f12fa2949a7ab0b205c8d2fcbfd1042f11 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Tue, 20 Dec 2016 16:01:22 +0300
Subject: [PATCH 064/446] IGNITE-4359 .NET: Fix Java timezone setting in tests

---
 .../Apache.Ignite.Core.Tests/Cache/Query/CacheLinqTest.cs      | 3 +--
 modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.cs | 3 ++-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheLinqTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheLinqTest.cs
index 46979efc65092..e74f09f53a71b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheLinqTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheLinqTest.cs
@@ -123,8 +123,7 @@ private static IgniteConfiguration GetConfig(string gridName = null)
             {
                 BinaryConfiguration = new BinaryConfiguration(typeof(Person),
                     typeof(Organization), typeof(Address), typeof(Role), typeof(RoleKey), typeof(Numerics)),
-                GridName = gridName,
-                JvmOptions = { "-Duser.timezone=UTC" }
+                GridName = gridName
             };
         }
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.cs
index 7d483bb60947d..ad2739802a61b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.cs
@@ -55,7 +55,8 @@ public static class TestUtils
                 "-Xms1g",
                 "-Xmx4g",
                 "-ea",
-                "-DIGNITE_QUIET=true"
+                "-DIGNITE_QUIET=true",
+                "-Duser.timezone=UTC"
             }
             : new List<string>
             {

From 072b9c26dda4eb7a39d095d33c08d1611c891444 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Wed, 21 Dec 2016 13:08:30 +0300
Subject: [PATCH 065/446] IGNITE-4469: Hadoop: set default value of offheap
 memory page and shuffle messages to 1Mb.

---
 .../ignite/internal/processors/hadoop/HadoopJobProperty.java | 4 ++--
 .../internal/processors/hadoop/shuffle/HadoopShuffleJob.java | 2 +-
 .../hadoop/shuffle/collections/HadoopMultimapBase.java       | 5 ++++-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index 9dd430b902ec3..4398acd0be5fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -71,7 +71,7 @@ public enum HadoopJobProperty {
     /**
      * Size in bytes of single memory page which will be allocated for data structures in shuffle.
      * <p>
-     * By default is {@code 32 * 1024}.
+     * Defaults to {@code 1Mb}.
      */
     SHUFFLE_OFFHEAP_PAGE_SIZE("ignite.shuffle.offheap.page.size"),
 
@@ -98,7 +98,7 @@ public enum HadoopJobProperty {
     /**
      * Defines approximate size in bytes of shuffle message which will be passed over wire from mapper to reducer.
      * <p>
-     * Defaults to 128Kb.
+     * Defaults to {@code 1Mb}.
      */
     SHUFFLE_MSG_SIZE("ignite.shuffle.message.size"),
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 214a335432df6..1c546a1037d60 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -76,7 +76,7 @@
  */
 public class HadoopShuffleJob<T> implements AutoCloseable {
     /** */
-    private static final int DFLT_SHUFFLE_MSG_SIZE = 128 * 1024;
+    private static final int DFLT_SHUFFLE_MSG_SIZE = 1024 * 1024;
 
     /** */
     private final HadoopJob job;
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
index 39b7c51d486ac..d7a4541a44670 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
@@ -40,6 +40,9 @@
  * Base class for all multimaps.
  */
 public abstract class HadoopMultimapBase implements HadoopMultimap {
+    /** Default offheap page size. */
+    private static final int DFLT_OFFHEAP_PAGE_SIZE = 1024 * 1024;
+
     /** */
     protected final GridUnsafeMemory mem;
 
@@ -59,7 +62,7 @@ protected HadoopMultimapBase(HadoopJobInfo jobInfo, GridUnsafeMemory mem) {
 
         this.mem = mem;
 
-        pageSize = get(jobInfo, SHUFFLE_OFFHEAP_PAGE_SIZE, 32 * 1024);
+        pageSize = get(jobInfo, SHUFFLE_OFFHEAP_PAGE_SIZE, DFLT_OFFHEAP_PAGE_SIZE);
     }
 
     /**

From ce46c10ea767254a60b80abecf45be8a2d9921b4 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 21 Dec 2016 13:09:26 +0300
Subject: [PATCH 066/446] TcpCommunicationSpi: fixed typo in warning.

---
 .../ignite/spi/communication/tcp/TcpCommunicationSpi.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 0c90414de365a..c35b5efb302f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -1872,7 +1872,7 @@ public CommunicationListener getListener() {
         if (msgQueueLimit == 0)
             U.quietAndWarn(log, "Message queue limit is set to 0 which may lead to " +
                 "potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes " +
-                "due to message queues growth on sender and reciever sides.");
+                "due to message queues growth on sender and receiver sides.");
 
         registerMBean(gridName, this, TcpCommunicationSpiMBean.class);
 

From c103ac33d590492577bb687b29969550cdb532cf Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Dec 2016 16:53:18 +0300
Subject: [PATCH 067/446] IGNITE-4248: Fixed: Some loggers ignore IGNITE_QUIET
 system property. This closes #1367.

---
 .../src/main/java/org/apache/ignite/IgniteLogger.java |  4 ++--
 .../hadoop/impl/igfs/HadoopIgfsJclLogger.java         |  9 ++++++++-
 .../java/org/apache/ignite/logger/jcl/JclLogger.java  |  9 ++++++++-
 .../org/apache/ignite/logger/slf4j/Slf4jLogger.java   | 11 +++++++++--
 4 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java b/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
index f3afa995f0ca4..8d814fd78b39e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
@@ -146,9 +146,9 @@ public interface IgniteLogger {
     public boolean isInfoEnabled();
 
     /**
-     * Tests whether {@code info} and {@code debug} levels are turned off.
+     * Tests whether Logger is in "Quiet mode".
      *
-     * @return Whether {@code info} and {@code debug} levels are turned off.
+     * @return {@code true} "Quiet mode" is enabled, {@code false} otherwise
      */
     public boolean isQuiet();
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsJclLogger.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsJclLogger.java
index 0ae8a9fe1a421..64752040fe446 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsJclLogger.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsJclLogger.java
@@ -24,6 +24,8 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_QUIET;
+
 /**
  * JCL logger wrapper for Hadoop.
  */
@@ -32,6 +34,9 @@ public class HadoopIgfsJclLogger implements IgniteLogger {
     @GridToStringInclude
     private Log impl;
 
+    /** Quiet flag. */
+    private final boolean quiet;
+
     /**
      * Constructor.
      *
@@ -41,6 +46,8 @@ public class HadoopIgfsJclLogger implements IgniteLogger {
         assert impl != null;
 
         this.impl = impl;
+
+        quiet = Boolean.valueOf(System.getProperty(IGNITE_QUIET, "true"));
     }
 
     /** {@inheritDoc} */
@@ -81,7 +88,7 @@ public class HadoopIgfsJclLogger implements IgniteLogger {
 
     /** {@inheritDoc} */
     @Override public boolean isQuiet() {
-        return !isInfoEnabled() && !isDebugEnabled();
+        return quiet;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/jcl/src/main/java/org/apache/ignite/logger/jcl/JclLogger.java b/modules/jcl/src/main/java/org/apache/ignite/logger/jcl/JclLogger.java
index a13caa104c559..c75197a02977f 100644
--- a/modules/jcl/src/main/java/org/apache/ignite/logger/jcl/JclLogger.java
+++ b/modules/jcl/src/main/java/org/apache/ignite/logger/jcl/JclLogger.java
@@ -22,6 +22,8 @@
 import org.apache.ignite.IgniteLogger;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_QUIET;
+
 /**
  * This logger wraps any JCL (<a target=_blank href="http://jakarta.apache.org/commons/logging/">Jakarta Commons Logging</a>)
  * loggers. Implementation simply delegates to underlying JCL logger. This logger
@@ -77,6 +79,9 @@ public class JclLogger implements IgniteLogger {
     /** JCL implementation proxy. */
     private Log impl;
 
+    /** Quiet flag. */
+    private final boolean quiet;
+
     /**
      * Creates new logger.
      */
@@ -93,6 +98,8 @@ public JclLogger(Log impl) {
         assert impl != null;
 
         this.impl = impl;
+
+        quiet = Boolean.valueOf(System.getProperty(IGNITE_QUIET, "true"));
     }
 
     /** {@inheritDoc} */
@@ -133,7 +140,7 @@ public JclLogger(Log impl) {
 
     /** {@inheritDoc} */
     @Override public boolean isQuiet() {
-        return !isInfoEnabled() && !isDebugEnabled();
+        return quiet;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/slf4j/src/main/java/org/apache/ignite/logger/slf4j/Slf4jLogger.java b/modules/slf4j/src/main/java/org/apache/ignite/logger/slf4j/Slf4jLogger.java
index 51e56693c10c5..2b0e98041db39 100644
--- a/modules/slf4j/src/main/java/org/apache/ignite/logger/slf4j/Slf4jLogger.java
+++ b/modules/slf4j/src/main/java/org/apache/ignite/logger/slf4j/Slf4jLogger.java
@@ -22,6 +22,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_QUIET;
+
 /**
  * SLF4J-based implementation for logging. This logger should be used
  * by loaders that have prefer slf4j-based logging.
@@ -41,11 +43,14 @@ public class Slf4jLogger implements IgniteLogger {
     /** SLF4J implementation proxy. */
     private final Logger impl;
 
+    /** Quiet flag. */
+    private final boolean quiet;
+
     /**
      * Creates new logger.
      */
     public Slf4jLogger() {
-        impl = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
+        this(LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME));
     }
 
     /**
@@ -57,6 +62,8 @@ public Slf4jLogger(Logger impl) {
         assert impl != null;
 
         this.impl = impl;
+
+        quiet = Boolean.valueOf(System.getProperty(IGNITE_QUIET, "true"));
     }
 
     /** {@inheritDoc} */
@@ -129,7 +136,7 @@ public Slf4jLogger(Logger impl) {
 
     /** {@inheritDoc} */
     @Override public boolean isQuiet() {
-        return !isInfoEnabled() && !isDebugEnabled();
+        return quiet;
     }
 
     /** {@inheritDoc} */

From 8372e69dd01972a01dbd59689475f2d45c2b6d94 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 22 Dec 2016 10:30:05 +0300
Subject: [PATCH 068/446] GridTopic.TOPIC_HADOOP_MSG: new topic should be added
 at the end for compatibility with previous releases.

---
 .../main/java/org/apache/ignite/internal/GridTopic.java   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 24ddcd226ae7d..2962540721ec9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -90,9 +90,6 @@ public enum GridTopic {
     /** */
     TOPIC_HADOOP,
 
-    /** */
-    TOPIC_HADOOP_MSG,
-
     /** */
     TOPIC_QUERY,
 
@@ -100,7 +97,10 @@ public enum GridTopic {
     TOPIC_TX,
 
     /** */
-    TOPIC_IO_TEST;
+    TOPIC_IO_TEST,
+
+    /** */
+    TOPIC_HADOOP_MSG;
 
     /** Enum values. */
     private static final GridTopic[] VALS = values();

From 708cc8c6849b21063a555895671f6f820d92184a Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 22 Dec 2016 12:48:58 +0300
Subject: [PATCH 069/446] IGNITE-4408: Allow BinaryObjects pass to IndexingSpi.
 This closes #1353.

---
 .../apache/ignite/IgniteSystemProperties.java |   8 +
 .../processors/cache/IgniteCacheProxy.java    |   2 +-
 .../cache/query/GridCacheQueryAdapter.java    |   2 +-
 .../processors/query/GridQueryProcessor.java  |  36 +++-
 .../ignite/spi/indexing/IndexingSpi.java      |   3 +
 .../cache/query/IndexingSpiQuerySelfTest.java | 199 +++++++++++++++++-
 6 files changed, 229 insertions(+), 21 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index de6cbed1fb3a5..fe78d889b178f 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -499,6 +499,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_UNALIGNED_MEMORY_ACCESS = "IGNITE_UNALIGNED_MEMORY_ACCESS";
 
+    /**
+     * When set to {@code true} BinaryObject will be unwrapped before passing to IndexingSpi to preserve
+     * old behavior query processor with IndexingSpi.
+     * <p>
+     * @deprecated Should be removed in Apache Ignite 2.0.
+     */
+    public static final String IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI = "IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
index f87fa1d5d815f..b9737c62dcbf3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
@@ -848,7 +848,7 @@ private boolean isReplicatedDataNode() {
      */
     private void validate(Query qry) {
         if (!GridQueryProcessor.isEnabled(ctx.config()) && !(qry instanceof ScanQuery) &&
-            !(qry instanceof ContinuousQuery))
+            !(qry instanceof ContinuousQuery) && !(qry instanceof SpiQuery))
             throw new CacheException("Indexing is disabled for cache: " + ctx.cache().name() +
                 ". Use setIndexedTypes or setTypeMetadata methods on CacheConfiguration to enable.");
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
index 2355591def2f1..b29e5e7a62ebd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
@@ -430,7 +430,7 @@ public ClusterGroup projection() {
      * @throws IgniteCheckedException If query is invalid.
      */
     public void validate() throws IgniteCheckedException {
-        if ((type != SCAN && type != SET) && !GridQueryProcessor.isEnabled(cctx.config()))
+        if ((type != SCAN && type != SET && type != SPI) && !GridQueryProcessor.isEnabled(cctx.config()))
             throw new IgniteCheckedException("Indexing is disabled for cache: " + cctx.cache().name());
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 8befa0e784e1e..6c093ee7b541c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -44,7 +44,7 @@
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryField;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryType;
@@ -160,6 +160,9 @@ public class GridQueryProcessor extends GridProcessorAdapter {
     /** */
     private static final ThreadLocal<AffinityTopologyVersion> requestTopVer = new ThreadLocal<>();
 
+    /** Default is @{true} */
+    private final boolean isIndexingSpiAllowsBinary = !IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI);
+
     /**
      * @param ctx Kernal context.
      */
@@ -680,7 +683,11 @@ public void store(final String space, final CacheObject key, final CacheObject v
         if (ctx.indexing().enabled()) {
             coctx = cacheObjectContext(space);
 
-            ctx.indexing().store(space, key.value(coctx, false), val.value(coctx, false), expirationTime);
+            Object key0 = unwrap(key, coctx);
+
+            Object val0 = unwrap(val, coctx);
+
+            ctx.indexing().store(space, key0, val0, expirationTime);
         }
 
         if (idx == null)
@@ -735,6 +742,13 @@ public void store(final String space, final CacheObject key, final CacheObject v
         }
     }
 
+    /**
+     * Unwrap CacheObject if needed.
+     */
+    private Object unwrap(CacheObject obj, CacheObjectContext coctx) {
+        return isIndexingSpiAllowsBinary && ctx.cacheObjects().isBinaryObject(obj) ? obj : obj.value(coctx, false);
+    }
+
     /**
      * @throws IgniteCheckedException If failed.
      */
@@ -1025,7 +1039,9 @@ public void remove(String space, CacheObject key, CacheObject val) throws Ignite
         if (ctx.indexing().enabled()) {
             CacheObjectContext coctx = cacheObjectContext(space);
 
-            ctx.indexing().remove(space, key.value(coctx, false));
+            Object key0 = unwrap(key, coctx);
+
+            ctx.indexing().remove(space, key0);
         }
 
         if (idx == null)
@@ -1168,11 +1184,9 @@ public void onSwap(String spaceName, CacheObject key) throws IgniteCheckedExcept
         if (ctx.indexing().enabled()) {
             CacheObjectContext coctx = cacheObjectContext(spaceName);
 
-            ctx.indexing().onSwap(
-                spaceName,
-                key.value(
-                    coctx,
-                    false));
+            Object key0 = unwrap(key, coctx);
+
+            ctx.indexing().onSwap(spaceName, key0);
         }
 
         if (idx == null)
@@ -1207,7 +1221,11 @@ public void onUnswap(String spaceName, CacheObject key, CacheObject val)
         if (ctx.indexing().enabled()) {
             CacheObjectContext coctx = cacheObjectContext(spaceName);
 
-            ctx.indexing().onUnswap(spaceName, key.value(coctx, false), val.value(coctx, false));
+            Object key0 = unwrap(key, coctx);
+
+            Object val0 = unwrap(val, coctx);
+
+            ctx.indexing().onUnswap(spaceName, key0, val0);
         }
 
         if (idx == null)
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingSpi.java
index a3ea33ed26ff2..bbe27c0719b2a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingSpi.java
@@ -35,6 +35,9 @@
  * methods. Note again that calling methods from this interface on the obtained instance can lead
  * to undefined behavior and explicitly not supported.
  *
+ * <b>NOTE:</b> Key and value arguments of IgniteSpi methods can be {@link org.apache.ignite.binary.BinaryObject} instances.
+ * BinaryObjects can be deserialized manually if original objects needed.
+ *
  * Here is a Java example on how to configure SPI.
  * <pre name="code" class="java">
  * IndexingSpi spi = new MyIndexingSpi();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
index 94b0c8aedbb2b..f66b99ef5b1c7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
@@ -17,11 +17,22 @@
 
 package org.apache.ignite.internal.processors.cache.query;
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import javax.cache.Cache;
 import junit.framework.TestCase;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.IgniteTransactions;
 import org.apache.ignite.Ignition;
+import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SpiQuery;
@@ -40,17 +51,9 @@
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.apache.ignite.transactions.TransactionState;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.Callable;
-import javax.cache.Cache;
-
 /**
  * Indexing Spi query test
  */
@@ -85,6 +88,94 @@ public void testSimpleIndexingSpi() throws Exception {
             System.out.println(entry);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testIndexingSpiWithDisabledQueryProcessor() throws Exception {
+        IgniteConfiguration cfg = configuration();
+
+        cfg.setIndexingSpi(new MyIndexingSpi());
+
+        Ignite ignite = Ignition.start(cfg);
+
+        CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>("test-cache");
+
+        IgniteCache<Integer, Integer> cache = ignite.createCache(ccfg);
+
+        for (int i = 0; i < 10; i++)
+            cache.put(i, i);
+
+        QueryCursor<Cache.Entry<Integer, Integer>> cursor = cache.query(new SpiQuery<Integer, Integer>().setArgs(2, 5));
+
+        for (Cache.Entry<Integer, Integer> entry : cursor)
+            System.out.println(entry);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBinaryIndexingSpi() throws Exception {
+        IgniteConfiguration cfg = configuration();
+
+        cfg.setIndexingSpi(new MyBinaryIndexingSpi());
+
+        Ignite ignite = Ignition.start(cfg);
+
+        CacheConfiguration<PersonKey, Person> ccfg = new CacheConfiguration<>("test-binary-cache");
+
+        ccfg.setIndexedTypes(PersonKey.class, Person.class);
+
+        IgniteCache<PersonKey, Person> cache = ignite.createCache(ccfg);
+
+        for (int i = 0; i < 10; i++) {
+            PersonKey key = new PersonKey(i);
+
+            cache.put(key, new Person("John Doe " + i));
+        }
+
+        QueryCursor<Cache.Entry<PersonKey, Person>> cursor = cache.query(
+            new SpiQuery<PersonKey, Person>().setArgs(new PersonKey(2), new PersonKey(5)));
+
+        for (Cache.Entry<PersonKey, Person> entry : cursor)
+            System.out.println(entry);
+
+        cache.remove(new PersonKey(9));
+    }
+
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNonBinaryIndexingSpi() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI, "true");
+
+        IgniteConfiguration cfg = configuration();
+
+        cfg.setIndexingSpi(new MyIndexingSpi());
+
+        Ignite ignite = Ignition.start(cfg);
+
+        CacheConfiguration<PersonKey, Person> ccfg = new CacheConfiguration<>("test-binary-cache");
+
+        ccfg.setIndexedTypes(PersonKey.class, Person.class);
+
+        IgniteCache<PersonKey, Person> cache = ignite.createCache(ccfg);
+
+        for (int i = 0; i < 10; i++) {
+            PersonKey key = new PersonKey(i);
+
+            cache.put(key, new Person("John Doe " + i));
+        }
+
+        QueryCursor<Cache.Entry<PersonKey, Person>> cursor = cache.query(
+            new SpiQuery<PersonKey, Person>().setArgs(new PersonKey(2), new PersonKey(5)));
+
+        for (Cache.Entry<PersonKey, Person> entry : cursor)
+            System.out.println(entry);
+
+        cache.remove(new PersonKey(9));
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -173,6 +264,9 @@ private static class MyIndexingSpi extends IgniteSpiAdapter implements IndexingS
             Object from = paramsIt.next();
             Object to = paramsIt.next();
 
+            from = from instanceof BinaryObject ? ((BinaryObject)from).deserialize() : from;
+            to = to instanceof BinaryObject ? ((BinaryObject)to).deserialize() : to;
+
             SortedMap<Object, Object> map = idx.subMap(from, to);
 
             Collection<Cache.Entry<?, ?>> res = new ArrayList<>(map.size());
@@ -186,6 +280,9 @@ private static class MyIndexingSpi extends IgniteSpiAdapter implements IndexingS
         /** {@inheritDoc} */
         @Override public void store(@Nullable String spaceName, Object key, Object val, long expirationTime)
             throws IgniteSpiException {
+            assertFalse(key instanceof BinaryObject);
+            assertFalse(val instanceof BinaryObject);
+
             idx.put(key, val);
         }
 
@@ -205,14 +302,96 @@ private static class MyIndexingSpi extends IgniteSpiAdapter implements IndexingS
         }
     }
 
+    /**
+     * Indexing Spi implementation for test. Accepts binary objects only
+     */
+    private static class MyBinaryIndexingSpi extends MyIndexingSpi {
+
+        /** {@inheritDoc} */
+        @Override public void store(@Nullable String spaceName, Object key, Object val,
+            long expirationTime) throws IgniteSpiException {
+            assertTrue(key instanceof BinaryObject);
+
+            assertTrue(val instanceof BinaryObject);
+
+            super.store(spaceName, ((BinaryObject)key).deserialize(), ((BinaryObject)val).deserialize(), expirationTime);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void remove(@Nullable String spaceName, Object key) throws IgniteSpiException {
+            assertTrue(key instanceof BinaryObject);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onSwap(@Nullable String spaceName, Object key) throws IgniteSpiException {
+            assertTrue(key instanceof BinaryObject);
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public void onUnswap(@Nullable String spaceName, Object key, Object val) throws IgniteSpiException {
+            assertTrue(key instanceof BinaryObject);
+
+            assertTrue(val instanceof BinaryObject);
+        }
+    }
+
     /**
      * Broken Indexing Spi implementation for test
      */
-    private class MyBrokenIndexingSpi extends MyIndexingSpi {
+    private static class MyBrokenIndexingSpi extends MyIndexingSpi {
         /** {@inheritDoc} */
         @Override public void store(@Nullable String spaceName, Object key, Object val,
             long expirationTime) throws IgniteSpiException {
             throw new IgniteSpiException("Test exception");
         }
     }
+
+    /**
+     *
+     */
+    private static class PersonKey implements Serializable, Comparable<PersonKey> {
+        /** */
+        private int id;
+
+        /** */
+        public PersonKey(int id) {
+            this.id = id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int compareTo(@NotNull PersonKey o) {
+            return Integer.compare(id, o.id);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            PersonKey key = (PersonKey)o;
+
+            return id == key.id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return id;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Person implements Serializable {
+        /** */
+        private String name;
+
+        /** */
+        Person(String name) {
+            this.name = name;
+        }
+    }
 }
\ No newline at end of file

From 6e71ef26d8e3c6f86d1f0b9f4bec97b7e33d0b2e Mon Sep 17 00:00:00 2001
From: Valentin Kulichenko <valentin.lulichenko@gmail.com>
Date: Thu, 22 Dec 2016 13:05:35 -0800
Subject: [PATCH 070/446] IGNITE-4439 - Attribute based node filter

---
 .../ignite/util/AttributeNodeFilter.java      | 105 ++++++++++
 .../testsuites/IgniteBasicTestSuite.java      |   3 +
 .../util/AttributeNodeFilterSelfTest.java     | 184 ++++++++++++++++++
 3 files changed, 292 insertions(+)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/util/AttributeNodeFilter.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/util/AttributeNodeFilterSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/util/AttributeNodeFilter.java b/modules/core/src/main/java/org/apache/ignite/util/AttributeNodeFilter.java
new file mode 100644
index 0000000000000..e2b972be462fb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/util/AttributeNodeFilter.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.util.Collections;
+import java.util.Map;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Implementation of {@code IgnitePredicate<ClusterNode>} based on
+ * {@link IgniteConfiguration#getUserAttributes() user attributes}.
+ * This filter can be used in methods like {@link ClusterGroup#forPredicate(IgnitePredicate)},
+ * {@link CacheConfiguration#setNodeFilter(IgnitePredicate)},
+ * {@link ServiceConfiguration#setNodeFilter(IgnitePredicate)}, etc.
+ * <p>
+ * The filter will evaluate to true if a node has <b>all</b> provided attributes set to
+ * corresponding values. Here is an example of how you can configure node filter for a
+ * cache or a service so that it's deployed only on nodes that have {@code group}
+ * attribute set to value {@code data}:
+ * <pre name="code" class="xml">
+ * &lt;property name=&quot;nodeFilter&quot;&gt;
+ *     &lt;bean class=&quot;org.apache.ignite.util.ClusterAttributeNodeFilter&quot;&gt;
+ *         &lt;constructor-arg value="group"/&gt;
+ *         &lt;constructor-arg value="data"/&gt;
+ *     &lt;/bean&gt;
+ * &lt;/property&gt;
+ * </pre>
+ * You can also specify multiple attributes for the filter:
+ * <pre name="code" class="xml">
+ * &lt;property name=&quot;nodeFilter&quot;&gt;
+ *     &lt;bean class=&quot;org.apache.ignite.util.ClusterAttributeNodeFilter&quot;&gt;
+ *         &lt;constructor-arg&gt;
+ *             &lt;map&gt;
+ *                 &lt;entry key=&quot;cpu-group&quot; value=&quot;high&quot;/&gt;
+ *                 &lt;entry key=&quot;memory-group&quot; value=&quot;high&quot;/&gt;
+ *             &lt;/map&gt;
+ *         &lt;/constructor-arg&gt;
+ *     &lt;/bean&gt;
+ * &lt;/property&gt;
+ * </pre>
+ * With this configuration a cache or a service will deploy only on nodes that have both
+ * {@code cpu-group} and {@code memory-group} attributes set to value {@code high}.
+ */
+public class AttributeNodeFilter implements IgnitePredicate<ClusterNode> {
+    /** Attributes. */
+    private final Map<String, Object> attrs;
+
+    /**
+     * Creates new node filter with a single attribute value.
+     *
+     * @param attrName Attribute name.
+     * @param attrVal Attribute value.
+     */
+    public AttributeNodeFilter(String attrName, @Nullable Object attrVal) {
+        A.notNull(attrName, "attrName");
+
+        attrs = Collections.singletonMap(attrName, attrVal);
+    }
+
+    /**
+     * Creates new node filter with a set of attributes.
+     *
+     * @param attrs Attributes.
+     */
+    public AttributeNodeFilter(Map<String, Object> attrs) {
+        A.notNull(attrs, "attrs");
+
+        this.attrs = attrs;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean apply(ClusterNode node) {
+        Map<String, Object> nodeAttrs = node.attributes();
+
+        for (Map.Entry<String, Object> attr : attrs.entrySet()) {
+            if (!F.eq(nodeAttrs.get(attr.getKey()), attr.getValue()))
+                return false;
+        }
+
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index 1c1fcf7b494a9..b2fafe20a56c7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -20,6 +20,7 @@
 import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.GridSuppressedExceptionSelfTest;
+import org.apache.ignite.util.AttributeNodeFilterSelfTest;
 import org.apache.ignite.internal.ClusterGroupHostsSelfTest;
 import org.apache.ignite.internal.ClusterGroupSelfTest;
 import org.apache.ignite.internal.GridFailFastNodeFailureDetectionSelfTest;
@@ -148,6 +149,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
 
         suite.addTestSuite(SecurityPermissionSetBuilderTest.class);
 
+        suite.addTestSuite(AttributeNodeFilterSelfTest.class);
+
         return suite;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/AttributeNodeFilterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/AttributeNodeFilterSelfTest.java
new file mode 100644
index 0000000000000..ac3800f19545c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/AttributeNodeFilterSelfTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for {@link AttributeNodeFilter}.
+ */
+public class AttributeNodeFilterSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private Map<String, ?> attrs;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        if (attrs != null)
+            cfg.setUserAttributes(attrs);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        attrs = null;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSingleAttribute() throws Exception {
+        IgnitePredicate<ClusterNode> filter = new AttributeNodeFilter("attr", "value");
+
+        assertTrue(filter.apply(nodeProxy(F.asMap("attr", "value"))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr", "wrong"))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr", null))));
+        assertFalse(filter.apply(nodeProxy(Collections.<String, Object>emptyMap())));
+        assertFalse(filter.apply(nodeProxy(F.asMap("wrong", "value"))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("null", "value"))));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSingleAttributeNullValue() throws Exception {
+        IgnitePredicate<ClusterNode> filter = new AttributeNodeFilter("attr", null);
+
+        assertTrue(filter.apply(nodeProxy(F.asMap("attr", null))));
+        assertTrue(filter.apply(nodeProxy(Collections.<String, Object>emptyMap())));
+        assertTrue(filter.apply(nodeProxy(F.asMap("wrong", "value"))));
+        assertTrue(filter.apply(nodeProxy(F.asMap("wrong", null))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr", "value"))));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMultipleAttributes() throws Exception {
+        IgnitePredicate<ClusterNode> filter =
+            new AttributeNodeFilter(F.<String, Object>asMap("attr1", "value1", "attr2", "value2"));
+
+        assertTrue(filter.apply(nodeProxy(F.asMap("attr1", "value1", "attr2", "value2"))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr1", "wrong", "attr2", "value2"))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr1", "value1", "attr2", "wrong"))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr1", "wrong", "attr2", "wrong"))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr1", "value1"))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr2", "value2"))));
+        assertFalse(filter.apply(nodeProxy(Collections.<String, Object>emptyMap())));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMultipleAttributesNullValues() throws Exception {
+        IgnitePredicate<ClusterNode> filter = new AttributeNodeFilter(F.asMap("attr1", null, "attr2", null));
+
+        assertTrue(filter.apply(nodeProxy(F.asMap("attr1", null, "attr2", null))));
+        assertTrue(filter.apply(nodeProxy(F.asMap("attr1", null))));
+        assertTrue(filter.apply(nodeProxy(F.asMap("attr2", null))));
+        assertTrue(filter.apply(nodeProxy(Collections.<String, Object>emptyMap())));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr1", "value1"))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr2", "value2"))));
+        assertFalse(filter.apply(nodeProxy(F.asMap("attr1", "value1", "attr2", "value2"))));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClusterGroup() throws Exception {
+        Ignite group1 = startGridsMultiThreaded(3);
+
+        attrs = F.asMap("group", "data");
+
+        Ignite group2 = startGridsMultiThreaded(3, 2);
+
+        assertEquals(2, group1.cluster().forPredicate(new AttributeNodeFilter("group", "data")).nodes().size());
+        assertEquals(2, group2.cluster().forPredicate(new AttributeNodeFilter("group", "data")).nodes().size());
+
+        assertEquals(3, group1.cluster().forPredicate(new AttributeNodeFilter("group", null)).nodes().size());
+        assertEquals(3, group2.cluster().forPredicate(new AttributeNodeFilter("group", null)).nodes().size());
+
+        assertEquals(0, group1.cluster().forPredicate(new AttributeNodeFilter("group", "wrong")).nodes().size());
+        assertEquals(0, group2.cluster().forPredicate(new AttributeNodeFilter("group", "wrong")).nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCacheFilter() throws Exception {
+        Ignite group1 = startGridsMultiThreaded(3);
+
+        attrs = F.asMap("group", "data");
+
+        Ignite group2 = startGridsMultiThreaded(3, 2);
+
+        group1.createCache(new CacheConfiguration<>("test-cache").
+            setNodeFilter(new AttributeNodeFilter("group", "data")));
+
+        assertEquals(2, group1.cluster().forDataNodes("test-cache").nodes().size());
+        assertEquals(2, group2.cluster().forDataNodes("test-cache").nodes().size());
+
+        assertEquals(0, group1.cluster().forDataNodes("wrong").nodes().size());
+        assertEquals(0, group2.cluster().forDataNodes("wrong").nodes().size());
+    }
+
+    /**
+     * @param attrs Attributes.
+     * @return Node proxy.
+     */
+    private static ClusterNode nodeProxy(final Map<String, ?> attrs) {
+        return (ClusterNode)Proxy.newProxyInstance(
+            ClusterNode.class.getClassLoader(),
+            new Class[] { ClusterNode.class },
+            new InvocationHandler() {
+                @SuppressWarnings("SuspiciousMethodCalls")
+                @Override public Object invoke(Object proxy, Method mtd, Object[] args) throws Throwable {
+                    if ("attributes".equals(mtd.getName()))
+                        return attrs;
+
+                    throw new UnsupportedOperationException();
+                }
+            });
+    }
+}

From babfc2f051f8471f541bd054650a47cceb3cc09e Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 23 Dec 2016 12:02:24 +0300
Subject: [PATCH 071/446] AttributeNodeFilter: added serialVersionUID.

---
 .../main/java/org/apache/ignite/util/AttributeNodeFilter.java  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/util/AttributeNodeFilter.java b/modules/core/src/main/java/org/apache/ignite/util/AttributeNodeFilter.java
index e2b972be462fb..fed0d43f26019 100644
--- a/modules/core/src/main/java/org/apache/ignite/util/AttributeNodeFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/util/AttributeNodeFilter.java
@@ -65,6 +65,9 @@
  * {@code cpu-group} and {@code memory-group} attributes set to value {@code high}.
  */
 public class AttributeNodeFilter implements IgnitePredicate<ClusterNode> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
     /** Attributes. */
     private final Map<String, Object> attrs;
 

From 2b3a180ff7692c0253da3ff7c32d65c09f9488d2 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 23 Dec 2016 16:34:10 +0700
Subject: [PATCH 072/446] Web console beta-7.

(cherry picked from commit 8e7c852)
---
 modules/web-console/backend/app/agent.js      |   15 +
 modules/web-console/backend/app/browser.js    |   13 +
 modules/web-console/backend/app/mongo.js      |   24 +-
 modules/web-console/backend/routes/demo.js    |   17 +-
 modules/web-console/backend/routes/profile.js |    3 +-
 .../web-console/backend/services/notebooks.js |   14 +-
 .../web-console/backend/services/sessions.js  |    6 +-
 .../web-console/backend/services/spaces.js    |   15 +
 modules/web-console/frontend/app/app.js       |    5 -
 .../controllers/reset-password.controller.js  |   14 +-
 .../{event-types.json => event-groups.json}   |    0
 .../frontend/app/data/pom-dependencies.json   |   12 +-
 .../ui-ace-docker/ui-ace-docker.controller.js |    2 +-
 .../ui-ace-docker/ui-ace-docker.jade          |    2 +-
 .../ui-ace-pojos/ui-ace-pojos.controller.js   |   12 +-
 .../ui-ace-pom/ui-ace-pom.controller.js       |    4 +-
 .../jade/form/form-field-dropdown.jade        |    5 +-
 .../helpers/jade/form/form-field-number.jade  |    3 +-
 .../helpers/jade/form/form-field-text.jade    |   19 +-
 .../frontend/app/helpers/jade/mixins.jade     |   52 +-
 .../frontend/app/modules/Demo/Demo.module.js  |    6 +-
 .../modules/configuration/Version.service.js  |    6 +-
 .../configuration/configuration.module.js     |   63 +-
 .../generator/AbstractTransformer.js          |   17 +
 .../modules/configuration/generator/Beans.js  |    5 +
 .../generator/ConfigurationGenerator.js       | 2795 ++++++-------
 ...enerator-optional.js => Custom.service.js} |   12 +-
 .../configuration/generator/Docker.service.js |    4 +-
 .../generator/JavaTransformer.service.js      | 2318 +++++------
 .../{Pom.service.js => Maven.service.js}      |    7 +-
 .../generator/Properties.service.js           |    2 +-
 .../configuration/generator/Readme.service.js |    2 +-
 .../generator/SharpTransformer.service.js     |  437 +-
 .../generator/SpringTransformer.service.js    |  497 ++-
 ....provider.js => Cache.platform.service.js} |   12 +-
 .../{cache.provider.js => Cache.service.js}   |   14 +-
 ...rovider.js => Cluster.platform.service.js} |   14 +-
 ...cluster.provider.js => Cluster.service.js} |   12 +-
 .../defaults/Event-groups.service.js}         |   19 +-
 .../{igfs.provider.js => IGFS.service.js}     |   12 +-
 .../generator/generator-common.js             |  625 ---
 .../configuration/generator/generator-java.js | 3617 -----------------
 .../generator/generator-spring.js             | 2111 ----------
 .../frontend/app/modules/sql/Notebook.data.js |   11 +-
 .../app/modules/sql/Notebook.service.js       |    2 +-
 .../app/modules/sql/scan-filter-input.jade    |   39 -
 .../app/modules/sql/sql.controller.js         |  211 +-
 .../frontend/app/modules/sql/sql.module.js    |    2 -
 .../app/modules/states/configuration.state.js |    2 +
 .../configuration/caches/node-filter.jade     |    2 +-
 .../states/configuration/caches/query.jade    |    3 +
 .../states/configuration/caches/store.jade    |    4 +-
 .../configuration/clusters/checkpoint.jade    |   11 +-
 .../configuration/clusters/checkpoint/fs.jade |    8 +-
 .../clusters/checkpoint/jdbc.jade             |    8 +-
 .../configuration/clusters/checkpoint/s3.jade |   25 +-
 .../clusters/collision/custom.jade            |    2 +-
 .../clusters/collision/job-stealing.jade      |    2 +-
 .../configuration/clusters/deployment.jade    |  129 +-
 .../states/configuration/clusters/events.jade |    4 +-
 .../configuration/clusters/failover.jade      |    4 +-
 .../clusters/general/discovery/zookeeper.jade |    2 +-
 .../zookeeper/retrypolicy/custom.jade         |    2 +-
 .../clusters/load-balancing.jade              |   23 +-
 .../configuration/clusters/logger/custom.jade |    2 +-
 .../states/configuration/clusters/ssl.jade    |    2 +-
 .../summary/summary-zipper.service.js}        |   40 +-
 .../summary/summary.controller.js             |  103 +-
 .../configuration/summary/summary.worker.js   |  123 +
 .../frontend/app/modules/user/Auth.service.js |   11 +-
 .../app/services/JavaTypes.service.js         |   13 +-
 .../frontend/app/services/Messages.service.js |   17 +-
 .../frontend/controllers/admin-controller.js  |  211 +-
 .../frontend/controllers/caches-controller.js |   22 +-
 .../controllers/clusters-controller.js        |   42 +-
 .../controllers/domains-controller.js         |   32 +-
 .../frontend/controllers/igfs-controller.js   |   20 +-
 .../controllers/profile-controller.js         |    3 +-
 .../gulpfile.babel.js/webpack/common.js       |   17 +-
 .../webpack/environments/development.js       |   14 +-
 .../webpack/environments/production.js        |    3 +-
 .../webpack/plugins/progress.js               |   82 -
 modules/web-console/frontend/package.json     |  178 +-
 .../frontend/public/images/cache.png          |  Bin 23700 -> 24791 bytes
 .../frontend/public/images/domains.png        |  Bin 23828 -> 22131 bytes
 .../frontend/public/images/igfs.png           |  Bin 14307 -> 14139 bytes
 .../frontend/public/images/query-chart.png    |  Bin 16637 -> 17142 bytes
 .../frontend/public/images/query-metadata.png |  Bin 32298 -> 39361 bytes
 .../frontend/public/images/query-table.png    |  Bin 29189 -> 28065 bytes
 .../frontend/public/images/summary.png        |  Bin 31997 -> 33650 bytes
 .../stylesheets/_font-awesome-custom.scss     |   23 +-
 .../public/stylesheets/form-field.scss        |   37 +
 .../frontend/public/stylesheets/style.scss    |  111 +-
 .../frontend/test/unit/JavaTypes.test.js      |   17 +-
 .../frontend/test/unit/Version.test.js        |    8 +-
 .../views/configuration/domains-import.jade   |    5 +-
 .../frontend/views/configuration/summary.jade |   25 +-
 .../frontend/views/settings/admin.jade        |   85 +-
 .../frontend/views/sql/notebook-new.jade      |    2 +-
 .../web-console/frontend/views/sql/sql.jade   |  235 +-
 .../frontend/views/templates/alert.jade       |    2 +-
 .../frontend/views/templates/select.jade      |    2 +-
 102 files changed, 4554 insertions(+), 10273 deletions(-)
 rename modules/web-console/frontend/app/data/{event-types.json => event-groups.json} (100%)
 rename modules/web-console/frontend/app/modules/configuration/generator/{generator-optional.js => Custom.service.js} (82%)
 rename modules/web-console/frontend/app/modules/configuration/generator/{Pom.service.js => Maven.service.js} (97%)
 rename modules/web-console/frontend/app/modules/configuration/generator/defaults/{cache.platform.provider.js => Cache.platform.service.js} (88%)
 rename modules/web-console/frontend/app/modules/configuration/generator/defaults/{cache.provider.js => Cache.service.js} (95%)
 rename modules/web-console/frontend/app/modules/configuration/generator/defaults/{cluster.platform.provider.js => Cluster.platform.service.js} (84%)
 rename modules/web-console/frontend/app/modules/configuration/generator/defaults/{cluster.provider.js => Cluster.service.js} (98%)
 rename modules/web-console/frontend/app/modules/configuration/{EventGroups.provider.js => generator/defaults/Event-groups.service.js} (78%)
 rename modules/web-console/frontend/app/modules/configuration/generator/defaults/{igfs.provider.js => IGFS.service.js} (92%)
 delete mode 100644 modules/web-console/frontend/app/modules/configuration/generator/generator-common.js
 delete mode 100644 modules/web-console/frontend/app/modules/configuration/generator/generator-java.js
 delete mode 100644 modules/web-console/frontend/app/modules/configuration/generator/generator-spring.js
 delete mode 100644 modules/web-console/frontend/app/modules/sql/scan-filter-input.jade
 rename modules/web-console/frontend/app/modules/{sql/scan-filter-input.service.js => states/configuration/summary/summary-zipper.service.js} (51%)
 create mode 100644 modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
 delete mode 100644 modules/web-console/frontend/gulpfile.babel.js/webpack/plugins/progress.js

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index f74a3f2e4c245..791ea50ed58b9 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -312,6 +312,21 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo)
             return this.executeRest(cmd);
         }
 
+        /**
+         * @param {Boolean} demo Is need run command on demo node.
+         * @param {Array.<String>} nids Node ids.
+         * @returns {Promise}
+         */
+        queryResetDetailMetrics(demo, nids) {
+            const cmd = new Command(demo, 'exe')
+                .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
+                .addParam('p1', nids)
+                .addParam('p2', 'org.apache.ignite.internal.visor.cache.VisorCacheResetQueryDetailMetricsTask')
+                .addParam('p3', 'java.lang.Void');
+
+            return this.executeRest(cmd);
+        }
+
         /**
          * @param {Boolean} demo Is need run command on demo node.
          * @param {String} cacheName Cache name.
diff --git a/modules/web-console/backend/app/browser.js b/modules/web-console/backend/app/browser.js
index 271082910e2c1..499d84d8e990a 100644
--- a/modules/web-console/backend/app/browser.js
+++ b/modules/web-console/backend/app/browser.js
@@ -162,6 +162,19 @@ module.exports.factory = (_, socketio, agentMgr, configure) => {
                         .catch((err) => cb(_errorToJson(err)));
                 });
 
+                // Collect cache query metrics and return result to browser.
+                socket.on('node:query:reset:metrics', (nids, cb) => {
+                    agentMgr.findAgent(accountId())
+                        .then((agent) => agent.queryResetDetailMetrics(demo, nids))
+                        .then((data) => {
+                            if (data.finished)
+                                return cb(null, data.result);
+
+                            cb(_errorToJson(data.error));
+                        })
+                        .catch((err) => cb(_errorToJson(err)));
+                });
+
                 // Return cache metadata from all nodes in grid.
                 socket.on('node:cache:metadata', (cacheName, cb) => {
                     agentMgr.findAgent(accountId())
diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index 0f38eb2c3c899..58ab11983aca1 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -247,6 +247,7 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
         longQueryWarningTimeout: Number,
         sqlFunctionClasses: [String],
         snapshotableIndex: Boolean,
+        queryDetailMetricsSize: Number,
         statisticsEnabled: Boolean,
         managementEnabled: Boolean,
         readFromBackup: Boolean,
@@ -823,7 +824,24 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
             Custom: {
                 className: String
             }
-        }]
+        }],
+        deploymentSpi: {
+            kind: {type: String, enum: ['URI', 'Local', 'Custom']},
+            URI: {
+                uriList: [String],
+                temporaryDirectoryPath: String,
+                scanners: [String],
+                listener: String,
+                checkMd5: Boolean,
+                encodeUri: Boolean
+            },
+            Local: {
+                listener: String
+            },
+            Custom: {
+                className: String
+            }
+        }
     });
 
     ClusterSchema.index({name: 1, space: 1}, {unique: true});
@@ -843,13 +861,15 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
             result: {type: String, enum: ['none', 'table', 'bar', 'pie', 'line', 'area']},
             pageSize: Number,
             timeLineSpan: String,
+            maxPages: Number,
             hideSystemColumns: Boolean,
             cacheName: String,
             chartsOptions: {barChart: {stacked: Boolean}, areaChart: {style: String}},
             rate: {
                 value: Number,
                 unit: Number
-            }
+            },
+            qryType: String
         }]
     });
 
diff --git a/modules/web-console/backend/routes/demo.js b/modules/web-console/backend/routes/demo.js
index ad4be6e6a4e15..3f4166d883763 100644
--- a/modules/web-console/backend/routes/demo.js
+++ b/modules/web-console/backend/routes/demo.js
@@ -39,20 +39,17 @@ module.exports.factory = (_, express, settings, mongo, spacesService, errors) =>
         router.post('/reset', (req, res) => {
             spacesService.spaces(req.user._id, true)
                 .then((spaces) => {
-                    if (spaces.length) {
-                        const spaceIds = spaces.map((space) => space._id);
-
-                        return Promise.all([
-                            mongo.Cluster.remove({space: {$in: spaceIds}}).exec(),
-                            mongo.Cache.remove({space: {$in: spaceIds}}).exec(),
-                            mongo.DomainModel.remove({space: {$in: spaceIds}}).exec(),
-                            mongo.Igfs.remove({space: {$in: spaceIds}}).exec()
-                        ]).then(() => spaces[0]);
-                    }
+                    const spaceIds = _.map(spaces, '_id');
+
+                    return spacesService.cleanUp(spaceIds)
+                        .then(() => mongo.Space.remove({_id: {$in: _.tail(spaceIds)}}).exec())
+                        .then(() => _.head(spaces));
                 })
                 .catch((err) => {
                     if (err instanceof errors.MissingResourceException)
                         return spacesService.createDemoSpace(req.user._id);
+
+                    throw err;
                 })
                 .then((space) => {
                     return Promise.all(_.map(clusters, (cluster) => {
diff --git a/modules/web-console/backend/routes/profile.js b/modules/web-console/backend/routes/profile.js
index 4d01cda1e44ae..1d6fccb8213f3 100644
--- a/modules/web-console/backend/routes/profile.js
+++ b/modules/web-console/backend/routes/profile.js
@@ -45,7 +45,7 @@ module.exports.factory = function(_, express, mongo, usersService) {
 
             usersService.save(req.body)
                 .then((user) => {
-                    const becomeUsed = req.session.viewedUser && user.admin;
+                    const becomeUsed = req.session.viewedUser && req.user.admin;
 
                     if (becomeUsed) {
                         req.session.viewedUser = user;
@@ -64,6 +64,7 @@ module.exports.factory = function(_, express, mongo, usersService) {
                         });
                     });
                 })
+                .then(() => usersService.get(req.user, req.session.viewedUser))
                 .then(res.api.ok)
                 .catch(res.api.error);
         });
diff --git a/modules/web-console/backend/services/notebooks.js b/modules/web-console/backend/services/notebooks.js
index 8846d8e365dd7..9aa2c386d8abe 100644
--- a/modules/web-console/backend/services/notebooks.js
+++ b/modules/web-console/backend/services/notebooks.js
@@ -34,12 +34,14 @@ module.exports = {
 module.exports.factory = (_, mongo, spacesService, errors) => {
     /**
      * Convert remove status operation to own presentation.
+     *
      * @param {RemoveResult} result - The results of remove operation.
      */
     const convertRemoveStatus = ({result}) => ({rowsAffected: result.n});
 
     /**
-     * Update existing notebook
+     * Update existing notebook.
+     *
      * @param {Object} notebook - The notebook for updating
      * @returns {Promise.<mongo.ObjectId>} that resolves cache id
      */
@@ -53,6 +55,7 @@ module.exports.factory = (_, mongo, spacesService, errors) => {
 
     /**
      * Create new notebook.
+     *
      * @param {Object} notebook - The notebook for creation.
      * @returns {Promise.<mongo.ObjectId>} that resolves cache id.
      */
@@ -67,6 +70,7 @@ module.exports.factory = (_, mongo, spacesService, errors) => {
     class NotebooksService {
         /**
          * Create or update Notebook.
+         *
          * @param {Object} notebook - The Notebook
          * @returns {Promise.<mongo.ObjectId>} that resolves Notebook id of merge operation.
          */
@@ -78,16 +82,18 @@ module.exports.factory = (_, mongo, spacesService, errors) => {
         }
 
         /**
-         * Get caches by spaces.
+         * Get notebooks by spaces.
+         *
          * @param {mongo.ObjectId|String} spaceIds - The spaces ids that own caches.
-         * @returns {Promise.<mongo.Cache[]>} - contains requested caches.
+         * @returns {Promise.<mongo.Notebook[]>} - contains requested caches.
          */
         static listBySpaces(spaceIds) {
             return mongo.Notebook.find({space: {$in: spaceIds}}).sort('name').lean().exec();
         }
 
         /**
-         * Remove Notebook.
+         * Remove notebook.
+         *
          * @param {mongo.ObjectId|String} notebookId - The Notebook id for remove.
          * @returns {Promise.<{rowsAffected}>} - The number of affected rows.
          */
diff --git a/modules/web-console/backend/services/sessions.js b/modules/web-console/backend/services/sessions.js
index ff0e303068080..7f62a606c9369 100644
--- a/modules/web-console/backend/services/sessions.js
+++ b/modules/web-console/backend/services/sessions.js
@@ -38,11 +38,11 @@ module.exports.factory = (_, mongo, errors) => {
          * @param {mongo.ObjectId|String} viewedUserId - id of user to become.
          */
         static become(session, viewedUserId) {
+            if (!session.req.user.admin)
+                return Promise.reject(new errors.IllegalAccessError('Became this user is not permitted. Only administrators can perform this actions.'));
+
             return mongo.Account.findById(viewedUserId).lean().exec()
                 .then((viewedUser) => {
-                    if (!session.req.user.admin)
-                        throw new errors.IllegalAccessError('Became this user is not permitted. Only administrators can perform this actions.');
-
                     viewedUser.token = session.req.user.token;
 
                     session.viewedUser = viewedUser;
diff --git a/modules/web-console/backend/services/spaces.js b/modules/web-console/backend/services/spaces.js
index 863d57cc9c74e..85f346e7bc510 100644
--- a/modules/web-console/backend/services/spaces.js
+++ b/modules/web-console/backend/services/spaces.js
@@ -68,6 +68,21 @@ module.exports.factory = (mongo, errors) => {
         static createDemoSpace(userId) {
             return new mongo.Space({name: 'Demo space', owner: userId, demo: true}).save();
         }
+
+        /**
+         * Clean up spaces.
+         *
+         * @param {mongo.ObjectId|String} spaceIds - The space ids for clean up.
+         * @returns {Promise.<>}
+         */
+        static cleanUp(spaceIds) {
+            return Promise.all([
+                mongo.Cluster.remove({space: {$in: spaceIds}}).exec(),
+                mongo.Cache.remove({space: {$in: spaceIds}}).exec(),
+                mongo.DomainModel.remove({space: {$in: spaceIds}}).exec(),
+                mongo.Igfs.remove({space: {$in: spaceIds}}).exec()
+            ]);
+        }
     }
 
     return SpacesService;
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 3510743e129c3..4ecd9b5394483 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -99,11 +99,6 @@ import domainsValidation from './filters/domainsValidation.filter';
 import duration from './filters/duration.filter';
 import hasPojo from './filters/hasPojo.filter';
 
-// Generators
-import $generatorOptional from './modules/configuration/generator/generator-optional';
-
-window.$generatorOptional = $generatorOptional;
-
 // Controllers
 import admin from 'controllers/admin-controller';
 import caches from 'controllers/caches-controller';
diff --git a/modules/web-console/frontend/app/controllers/reset-password.controller.js b/modules/web-console/frontend/app/controllers/reset-password.controller.js
index da0c37b05c428..f84a8762af49c 100644
--- a/modules/web-console/frontend/app/controllers/reset-password.controller.js
+++ b/modules/web-console/frontend/app/controllers/reset-password.controller.js
@@ -21,10 +21,10 @@ export default ['resetPassword', [
     ($scope, $modal, $http, $state, Messages, Focus) => {
         if ($state.params.token) {
             $http.post('/api/v1/password/validate/token', {token: $state.params.token})
-                .success((res) => {
-                    $scope.email = res.email;
-                    $scope.token = res.token;
-                    $scope.error = res.error;
+                .then(({data}) => {
+                    $scope.email = data.email;
+                    $scope.token = data.token;
+                    $scope.error = data.error;
 
                     if ($scope.token && !$scope.error)
                         Focus.move('user_password');
@@ -34,16 +34,16 @@ export default ['resetPassword', [
         // Try to reset user password for provided token.
         $scope.resetPassword = (reset_info) => {
             $http.post('/api/v1/password/reset', reset_info)
-                .success(() => {
+                .then(() => {
                     $state.go('signin');
 
                     Messages.showInfo('Password successfully changed');
                 })
-                .error((err, state) => {
+                .catch(({data, state}) => {
                     if (state === 503)
                         $state.go('signin');
 
-                    Messages.showError(err);
+                    Messages.showError(data);
                 });
         };
     }
diff --git a/modules/web-console/frontend/app/data/event-types.json b/modules/web-console/frontend/app/data/event-groups.json
similarity index 100%
rename from modules/web-console/frontend/app/data/event-types.json
rename to modules/web-console/frontend/app/data/event-groups.json
diff --git a/modules/web-console/frontend/app/data/pom-dependencies.json b/modules/web-console/frontend/app/data/pom-dependencies.json
index acf2bc86d6249..7d2bed0de8172 100644
--- a/modules/web-console/frontend/app/data/pom-dependencies.json
+++ b/modules/web-console/frontend/app/data/pom-dependencies.json
@@ -10,11 +10,11 @@
     "HadoopIgfsJcl": {"artifactId": "ignite-hadoop"},
     "SLF4J": {"artifactId": "ignite-slf4j"},
 
-    "Generic": {"groupId": "com.mchange", "artifactId": "c3p0", "version": "0.9.5.1"},
-    "MySQL": {"groupId": "mysql", "artifactId": "mysql-connector-java", "version": "5.1.37"},
-    "PostgreSQL": {"groupId": "org.postgresql", "artifactId": "postgresql", "version": "9.4-1204-jdbc42"},
+    "Generic": {"groupId": "com.mchange", "artifactId": "c3p0", "version": "0.9.5.2"},
+    "MySQL": {"groupId": "mysql", "artifactId": "mysql-connector-java", "version": "5.1.40"},
+    "PostgreSQL": {"groupId": "org.postgresql", "artifactId": "postgresql", "version": "9.4.1212.jre7"},
     "H2": {"groupId": "com.h2database", "artifactId": "h2", "version": "1.4.191"},
-    "Oracle": {"groupId": "oracle", "artifactId": "jdbc", "version": "11.2", "jar": "ojdbc6.jar"},
-    "DB2": {"groupId": "ibm", "artifactId": "jdbc", "version": "4.19.26", "jar": "db2jcc4.jar"},
-    "SQLServer": {"groupId": "microsoft", "artifactId": "jdbc", "version": "4.1", "jar": "sqljdbc41.jar"}
+    "Oracle": {"groupId": "com.oracle.jdbc", "artifactId": "ojdbc7", "version": "12.1.0.2", "jar": "ojdbc7.jar"},
+    "DB2": {"groupId": "ibm", "artifactId": "jdbc", "version": "4.21.29", "jar": "db2jcc4.jar"},
+    "SQLServer": {"groupId": "microsoft", "artifactId": "jdbc", "version": "4.2", "jar": "sqljdbc41.jar"}
 }
diff --git a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.controller.js b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.controller.js
index 32feaf357c3a3..de335aefdcacb 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.controller.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.controller.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-export default ['$scope', 'GeneratorDocker', function($scope, docker) {
+export default ['$scope', 'IgniteDockerGenerator', function($scope, docker) {
     const ctrl = this;
 
     // Watchers definition.
diff --git a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.jade b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.jade
index 3b0e7b8e9f492..3a24cfbd956fd 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.jade
+++ b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.jade
@@ -20,7 +20,7 @@ mixin hard-link(ref, txt)
 .panel-details-noborder
     .details-row
         p
-            +hard-link('https://docs.docker.com/reference/builder', 'Docker')
+            +hard-link('https://docs.docker.com/engine/reference/builder/', 'Docker')
             | &nbsp;file is a text file with instructions to create Docker image.<br/>
             | To build image you have to store following Docker file with your Ignite XML configuration to the same directory.<br>
             | Also you could use predefined&nbsp;
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.controller.js b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.controller.js
index 4e1187447096c..61bf086bc741e 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.controller.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.controller.js
@@ -30,7 +30,7 @@ export default ['$scope', 'JavaTypes', 'JavaTransformer', function($scope, JavaT
     const updatePojos = () => {
         delete ctrl.pojos;
 
-        if (!ctrl.cluster || !ctrl.cluster.caches)
+        if (_.isNil(ctrl.cluster) || _.isEmpty(ctrl.cluster.caches))
             return;
 
         ctrl.pojos = generator.pojos(ctrl.cluster.caches, ctrl.useConstructor, ctrl.includeKeyFields);
@@ -46,7 +46,7 @@ export default ['$scope', 'JavaTypes', 'JavaTransformer', function($scope, JavaT
         const classes = ctrl.classes = [];
 
         _.forEach(ctrl.pojos, (pojo) => {
-            if (pojo.keyType && JavaTypes.nonBuiltInClass(pojo.keyType))
+            if (_.nonNil(pojo.keyClass))
                 classes.push(pojo.keyType);
 
             classes.push(pojo.valueType);
@@ -55,17 +55,17 @@ export default ['$scope', 'JavaTypes', 'JavaTransformer', function($scope, JavaT
 
     // Update pojos class.
     const updateClass = (value) => {
-        if (!value || !ctrl.pojos.length)
+        if (_.isEmpty(value))
             return;
 
-        const keyType = ctrl.pojos[0].keyType;
+        const pojo = value[0];
 
-        ctrl.class = ctrl.class || (JavaTypes.nonBuiltInClass(keyType) ? keyType : null) || ctrl.pojos[0].valueType;
+        ctrl.class = ctrl.class || (pojo.keyClass ? pojo.keyType : pojo.valueType);
     };
 
     // Update pojos data.
     const updatePojosData = (value) => {
-        if (!value)
+        if (_.isNil(value))
             return;
 
         _.forEach(ctrl.pojos, (pojo) => {
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
index 2bf78c359230d..477cf20999271 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-export default ['$scope', 'GeneratorPom', 'IgniteVersion', function($scope, pom, Version) {
+export default ['$scope', 'IgniteMavenGenerator', 'IgniteVersion', function($scope, maven, Version) {
     const ctrl = this;
 
     // Watchers definition.
@@ -25,7 +25,7 @@ export default ['$scope', 'GeneratorPom', 'IgniteVersion', function($scope, pom,
         if (!value)
             return;
 
-        ctrl.data = pom.generate($scope.cluster, Version.productVersion().ignite).asString();
+        ctrl.data = maven.generate($scope.cluster, Version.productVersion().ignite).asString();
     };
 
     // Setup watchers.
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.jade
index 298db52c89b1b..33af6d1d8785a 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.jade
@@ -28,7 +28,7 @@ mixin ignite-form-field-dropdown(label, model, name, disabled, required, multipl
             data-ng-disabled=disabled && '#{disabled}' || '!#{options}.length'
 
             bs-select
-            bs-options='item.value as item.label for item in #{options}' 
+            bs-options='item.value as item.label for item in #{options}'
 
             data-multiple=multiple ? '1' : false
             data-container='body > .wrapper'
@@ -41,7 +41,8 @@ mixin ignite-form-field-dropdown(label, model, name, disabled, required, multipl
     .ignite-form-field
         +ignite-form-field__label(label, name, required)
         .ignite-form-field__control
-            i.tipField.icon-help(bs-tooltip='' data-title=tip)
+            if tip
+                i.tipField.icon-help(bs-tooltip='' data-title=tip)
 
             if block
                 block
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-number.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-number.jade
index d48343c36d6f0..58b0dcd654291 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-number.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-number.jade
@@ -38,7 +38,8 @@ mixin ignite-form-field-number(label, model, name, disabled, required, placehold
     .ignite-form-field
         +ignite-form-field__label(label, name, required)
         .ignite-form-field__control
-            i.tipField.icon-help(bs-tooltip='' data-title=tip)
+            if tip
+                i.tipField.icon-help(bs-tooltip='' data-title=tip)
             
             +form-field-feedback(name, 'required', 'This field could not be empty')
             +form-field-feedback(name, 'min', 'Value is less than allowable minimum: '+ min || 0)
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.jade
index 136d23b2340a1..1f93d3b8ede50 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.jade
@@ -30,13 +30,30 @@ mixin ignite-form-field-input(name, model, disabled, required, placeholder)
         data-ignite-form-panel-field=''
     )&attributes(attributes ? attributes.attributes ? attributes.attributes : attributes: {})
 
+mixin ignite-form-field-url-input(name, model, disabled, required, placeholder)
+    input.form-control(
+        id='{{ #{name} }}Input'
+        name='{{ #{name} }}'
+        placeholder=placeholder
+        type='url'
+
+        data-ng-model=model
+
+        data-ng-required=required && '#{required}'
+        data-ng-disabled=disabled && '#{disabled}'
+        data-ng-focus='tableReset()'
+
+        data-ignite-form-panel-field=''
+    )&attributes(attributes ? attributes.attributes ? attributes.attributes : attributes: {})
+
 mixin ignite-form-field-text(label, model, name, disabled, required, placeholder, tip)
     -var errLbl = label.substring(0, label.length - 1)
 
     .ignite-form-field
         +ignite-form-field__label(label, name, required)
         .ignite-form-field__control
-            i.tipField.icon-help(bs-tooltip='' data-title=tip)
+            if tip
+                i.tipField.icon-help(bs-tooltip='' data-title=tip)
             
             if block
                 block
diff --git a/modules/web-console/frontend/app/helpers/jade/mixins.jade b/modules/web-console/frontend/app/helpers/jade/mixins.jade
index 92af1b02a01b3..6ca41f645c68c 100644
--- a/modules/web-console/frontend/app/helpers/jade/mixins.jade
+++ b/modules/web-console/frontend/app/helpers/jade/mixins.jade
@@ -183,6 +183,14 @@ mixin text-enabled(lbl, model, name, enabled, required, placeholder, tip)
         if  block
             block
 
+//- Mixin for text field with autofocus.
+mixin text-enabled-autofocus(lbl, model, name, enabled, required, placeholder, tip)
+    +ignite-form-field-text(lbl, model, name, enabledToDisabled(enabled), required, placeholder, tip)(
+        data-ignite-form-field-input-autofocus='true'
+    )
+        if  block
+            block
+
 //- Mixin for text field.
 mixin text(lbl, model, name, required, placeholder, tip)
     +ignite-form-field-text(lbl, model, name, false, required, placeholder, tip)
@@ -221,12 +229,28 @@ mixin dropdown-required-empty(lbl, model, name, enabled, required, placeholder,
         if  block
             block
 
+//- Mixin for required dropdown field with autofocus.
+mixin dropdown-required-empty-autofocus(lbl, model, name, enabled, required, placeholder, placeholderEmpty, options, tip)
+    +ignite-form-field-dropdown(lbl, model, name, enabledToDisabled(enabled), required, false, placeholder, placeholderEmpty, options, tip)(
+        data-ignite-form-field-input-autofocus='true'
+    )
+        if  block
+            block
+
 //- Mixin for required dropdown field.
 mixin dropdown-required(lbl, model, name, enabled, required, placeholder, options, tip)
     +ignite-form-field-dropdown(lbl, model, name, enabledToDisabled(enabled), required, false, placeholder, '', options, tip)
         if  block
             block
 
+//- Mixin for required dropdown field with autofocus.
+mixin dropdown-required-autofocus(lbl, model, name, enabled, required, placeholder, options, tip)
+    +ignite-form-field-dropdown(lbl, model, name, enabledToDisabled(enabled), required, false, placeholder, '', options, tip)(
+        data-ignite-form-field-input-autofocus='true'
+    )
+        if  block
+            block
+
 //- Mixin for dropdown field.
 mixin dropdown(lbl, model, name, enabled, placeholder, options, tip)
     +ignite-form-field-dropdown(lbl, model, name, enabledToDisabled(enabled), false, false, placeholder, '', options, tip)
@@ -324,6 +348,28 @@ mixin table-java-package-field(name, model, items, valid, save, newItem)
                 ignite-on-escape=onEscape
             )
 
+//- Mixin for table java package field.
+mixin table-url-field(name, model, items, valid, save, newItem)
+    -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
+    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+
+    -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
+
+    -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
+    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+
+    div(ignite-on-focus-out=onBlur)
+        if block
+            block
+
+        .input-tip
+            +ignite-form-field-url-input(name, model, false, 'true', 'Enter URL')(
+                data-ignite-unique=items
+                data-ignite-form-field-input-autofocus='true'
+
+                ignite-on-enter=onEnter
+                ignite-on-escape=onEscape
+            )
 
 //- Mixin for table address field.
 mixin table-address-field(name, model, items, valid, save, newItem, portRange)
@@ -393,17 +439,17 @@ mixin table-save-button(valid, save, newItem)
     )
 
 //- Mixin for table remove button.
-mixin table-remove-conditional-button(items, show, tip)
+mixin table-remove-conditional-button(items, show, tip, row)
     i.tipField.fa.fa-remove(
         ng-hide='!#{show} || field.edit'
         bs-tooltip
         data-title=tip
-        ng-click='#{items}.splice(#{items}.indexOf(model), 1)'
+        ng-click='#{items}.splice(#{items}.indexOf(#{row}), 1)'
     )
 
 //- Mixin for table remove button.
 mixin table-remove-button(items, tip)
-    +table-remove-conditional-button(items, 'true', tip)
+    +table-remove-conditional-button(items, 'true', tip, 'model')
 
 //- Mixin for cache mode.
 mixin cacheMode(lbl, model, name, placeholder)
diff --git a/modules/web-console/frontend/app/modules/Demo/Demo.module.js b/modules/web-console/frontend/app/modules/Demo/Demo.module.js
index 83d55ed596da7..a3700ca3ed141 100644
--- a/modules/web-console/frontend/app/modules/Demo/Demo.module.js
+++ b/modules/web-console/frontend/app/modules/Demo/Demo.module.js
@@ -41,11 +41,11 @@ angular
             url: '/demo/reset',
             controller: ['$state', '$http', 'IgniteMessages', ($state, $http, Messages) => {
                 $http.post('/api/v1/demo/reset')
-                    .success(() => $state.go('base.configuration.clusters'))
-                    .error((err) => {
+                    .then(() => $state.go('base.configuration.clusters'))
+                    .catch((res) => {
                         $state.go('base.configuration.clusters');
 
-                        Messages.showError(err);
+                        Messages.showError(res);
                     });
             }],
             metaTags: {}
diff --git a/modules/web-console/frontend/app/modules/configuration/Version.service.js b/modules/web-console/frontend/app/modules/configuration/Version.service.js
index 06efdda42b83b..f0e9c4c2a2a89 100644
--- a/modules/web-console/frontend/app/modules/configuration/Version.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/Version.service.js
@@ -22,7 +22,7 @@ const VERSION_MATCHER = /(\d+)\.(\d+)\.(\d+)([-.]([^0123456789][^-]+)(-SNAPSHOT)
 
 const numberComparator = (a, b) => a > b ? 1 : a < b ? -1 : 0;
 
-export default class Version {
+export default class IgniteVersion {
     /**
      * Tries to parse product version from it's string representation.
      *
@@ -70,7 +70,7 @@ export default class Version {
         if (res !== 0)
             return res;
 
-        return numberComparator(pa.revTs, pb.maintenance);
+        return numberComparator(pa.revTs, pb.revTs);
     }
 
     /**
@@ -79,7 +79,7 @@ export default class Version {
      */
     productVersion() {
         return {
-            ignite: '1.7.0'
+            ignite: '1.8.0'
         };
     }
 
diff --git a/modules/web-console/frontend/app/modules/configuration/configuration.module.js b/modules/web-console/frontend/app/modules/configuration/configuration.module.js
index 27f7befbeb4f5..4288ff79ea182 100644
--- a/modules/web-console/frontend/app/modules/configuration/configuration.module.js
+++ b/modules/web-console/frontend/app/modules/configuration/configuration.module.js
@@ -17,26 +17,28 @@
 
 import angular from 'angular';
 
-import igniteEventGroups from './EventGroups.provider';
+
 import igniteSidebar from './Sidebar.provider';
-import Version from './Version.service';
+import IgniteVersion from './Version.service';
 
-import clusterDefaults from './generator/defaults/cluster.provider';
-import clusterPlatformDefaults from './generator/defaults/cluster.platform.provider';
-import cacheDefaults from './generator/defaults/cache.provider';
-import cachePlatformDefaults from './generator/defaults/cache.platform.provider';
-import igfsDefaults from './generator/defaults/igfs.provider';
+import IgniteClusterDefaults from './generator/defaults/Cluster.service';
+import IgniteClusterPlatformDefaults from './generator/defaults/Cluster.platform.service';
+import IgniteCacheDefaults from './generator/defaults/Cache.service';
+import IgniteCachePlatformDefaults from './generator/defaults/Cache.platform.service';
+import IgniteIGFSDefaults from './generator/defaults/IGFS.service';
+import IgniteEventGroups from './generator/defaults/Event-groups.service';
 
-import ConfigurationGenerator from './generator/ConfigurationGenerator';
-import PlatformGenerator from './generator/PlatformGenerator';
+import IgniteConfigurationGenerator from './generator/ConfigurationGenerator';
+import IgnitePlatformGenerator from './generator/PlatformGenerator';
 
-import SpringTransformer from './generator/SpringTransformer.service';
-import JavaTransformer from './generator/JavaTransformer.service';
+import IgniteSpringTransformer from './generator/SpringTransformer.service';
+import IgniteJavaTransformer from './generator/JavaTransformer.service';
 import SharpTransformer from './generator/SharpTransformer.service';
-import GeneratorDocker from './generator/Docker.service';
-import GeneratorPom from './generator/Pom.service';
-import GeneratorProperties from './generator/Properties.service';
-import GeneratorReadme from './generator/Readme.service';
+import IgniteDockerGenerator from './generator/Docker.service';
+import IgniteMavenGenerator from './generator/Maven.service';
+import IgniteGeneratorProperties from './generator/Properties.service';
+import IgniteReadmeGenerator from './generator/Readme.service';
+import IgniteCustomGenerator from './generator/Custom.service';
 
 import igniteSidebarDirective from './sidebar.directive';
 
@@ -45,21 +47,22 @@ angular
 .module('ignite-console.configuration', [
 
 ])
-.provider('igniteClusterDefaults', clusterDefaults)
-.provider('igniteClusterPlatformDefaults', clusterPlatformDefaults)
-.provider('igniteCacheDefaults', cacheDefaults)
-.provider('igniteCachePlatformDefaults', cachePlatformDefaults)
-.provider('igniteIgfsDefaults', igfsDefaults)
-.provider(...igniteEventGroups)
 .provider(...igniteSidebar)
 .directive(...igniteSidebarDirective)
-.service('IgniteVersion', Version)
-.service('IgniteConfigurationGenerator', ConfigurationGenerator)
-.service('IgnitePlatformGenerator', PlatformGenerator)
-.service('SpringTransformer', SpringTransformer)
-.service('JavaTransformer', JavaTransformer)
+.service('IgniteConfigurationGenerator', IgniteConfigurationGenerator)
+.service('IgnitePlatformGenerator', IgnitePlatformGenerator)
+.service('SpringTransformer', IgniteSpringTransformer)
+.service('JavaTransformer', IgniteJavaTransformer)
 .service('IgniteSharpTransformer', SharpTransformer)
-.service('IgnitePropertiesGenerator', GeneratorProperties)
-.service('IgniteReadmeGenerator', GeneratorReadme)
-.service(...GeneratorDocker)
-.service(...GeneratorPom);
+.service('IgniteVersion', IgniteVersion)
+.service('IgniteEventGroups', IgniteEventGroups)
+.service('IgniteClusterDefaults', IgniteClusterDefaults)
+.service('IgniteClusterPlatformDefaults', IgniteClusterPlatformDefaults)
+.service('IgniteCacheDefaults', IgniteCacheDefaults)
+.service('IgniteCachePlatformDefaults', IgniteCachePlatformDefaults)
+.service('IgniteIGFSDefaults', IgniteIGFSDefaults)
+.service('IgnitePropertiesGenerator', IgniteGeneratorProperties)
+.service('IgniteReadmeGenerator', IgniteReadmeGenerator)
+.service('IgniteDockerGenerator', IgniteDockerGenerator)
+.service('IgniteMavenGenerator', IgniteMavenGenerator)
+.service('IgniteCustomGenerator', IgniteCustomGenerator);
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
index 6244a531ea85b..f5afe59805114 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
@@ -17,7 +17,24 @@
 
 import StringBuilder from './StringBuilder';
 
+import IgniteConfigurationGenerator from './ConfigurationGenerator';
+import IgniteEventGroups from './defaults/Event-groups.service';
+
+import IgniteClusterDefaults from './defaults/Cluster.service';
+import IgniteCacheDefaults from './defaults/Cache.service';
+import IgniteIGFSDefaults from './defaults/IGFS.service';
+
+import JavaTypes from '../../../services/JavaTypes.service';
+
+const clusterDflts = new IgniteClusterDefaults();
+const cacheDflts = new IgniteCacheDefaults();
+const igfsDflts = new IgniteIGFSDefaults();
+
 export default class AbstractTransformer {
+    static generator = IgniteConfigurationGenerator;
+    static javaTypes = new JavaTypes(clusterDflts, cacheDflts, igfsDflts);
+    static eventGroups = new IgniteEventGroups();
+
     // Append comment with time stamp.
     static mainComment(sb, ...lines) {
         lines.push(sb.generatedBy());
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Beans.js b/modules/web-console/frontend/app/modules/configuration/generator/Beans.js
index 2750626f4a74b..ca1934282510e 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Beans.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Beans.js
@@ -17,6 +17,11 @@
 
 import _ from 'lodash';
 
+_.mixin({
+    nonNil: _.negate(_.isNil),
+    nonEmpty: _.negate(_.isEmpty)
+});
+
 export class EmptyBean {
     /**
      * @param {String} clsName
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 5887832ff81df..8770bf6063c73 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -19,1776 +19,1825 @@ import DFLT_DIALECTS from 'app/data/dialects.json';
 
 import { EmptyBean, Bean } from './Beans';
 
-export default ['JavaTypes', 'igniteClusterDefaults', 'igniteCacheDefaults', 'igniteIgfsDefaults', (JavaTypes, clusterDflts, cacheDflts, igfsDflts) => {
-    class ConfigurationGenerator {
-        static igniteConfigurationBean(cluster) {
-            return new Bean('org.apache.ignite.configuration.IgniteConfiguration', 'cfg', cluster, clusterDflts);
-        }
+import IgniteClusterDefaults from './defaults/Cluster.service';
+import IgniteCacheDefaults from './defaults/Cache.service';
+import IgniteIGFSDefaults from './defaults/IGFS.service';
 
-        static igfsConfigurationBean(igfs) {
-            return new Bean('org.apache.ignite.configuration.FileSystemConfiguration', 'igfs', igfs, igfsDflts);
-        }
+import JavaTypes from '../../../services/JavaTypes.service';
 
-        static cacheConfigurationBean(cache) {
-            return new Bean('org.apache.ignite.configuration.CacheConfiguration', 'ccfg', cache, cacheDflts);
-        }
+const clusterDflts = new IgniteClusterDefaults();
+const cacheDflts = new IgniteCacheDefaults();
+const igfsDflts = new IgniteIGFSDefaults();
 
-        static domainConfigurationBean(domain) {
-            return new Bean('org.apache.ignite.cache.QueryEntity', 'qryEntity', domain, cacheDflts);
-        }
+const javaTypes = new JavaTypes(clusterDflts, cacheDflts, igfsDflts);
 
-        static discoveryConfigurationBean(discovery) {
-            return new Bean('org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi', 'discovery', discovery, clusterDflts.discovery);
-        }
+export default class IgniteConfigurationGenerator {
+    static igniteConfigurationBean(cluster) {
+        return new Bean('org.apache.ignite.configuration.IgniteConfiguration', 'cfg', cluster, clusterDflts);
+    }
 
-        /**
-         * Function to generate ignite configuration.
-         *
-         * @param {Object} cluster Cluster to process.
-         * @param {Boolean} client
-         * @return {Bean} Generated ignite configuration.
-         */
-        static igniteConfiguration(cluster, client) {
-            const cfg = this.igniteConfigurationBean(cluster);
-
-            this.clusterGeneral(cluster, cfg, client);
-            this.clusterAtomics(cluster.atomicConfiguration, cfg);
-            this.clusterBinary(cluster.binaryConfiguration, cfg);
-            this.clusterCacheKeyConfiguration(cluster.cacheKeyConfiguration, cfg);
-            this.clusterCheckpoint(cluster, cluster.caches, cfg);
-            this.clusterCollision(cluster.collision, cfg);
-            this.clusterCommunication(cluster, cfg);
-            this.clusterConnector(cluster.connector, cfg);
-            this.clusterDeployment(cluster, cfg);
-            this.clusterEvents(cluster, cfg);
-            this.clusterFailover(cluster, cfg);
-            this.clusterLoadBalancing(cluster, cfg);
-            this.clusterLogger(cluster.logger, cfg);
-            this.clusterODBC(cluster.odbc, cfg);
-            this.clusterMarshaller(cluster, cfg);
-            this.clusterMetrics(cluster, cfg);
-            this.clusterSwap(cluster, cfg);
-            this.clusterTime(cluster, cfg);
-            this.clusterPools(cluster, cfg);
-            this.clusterTransactions(cluster.transactionConfiguration, cfg);
-            this.clusterSsl(cluster, cfg);
-            this.clusterUserAttributes(cluster, cfg);
-
-            this.clusterCaches(cluster, cluster.caches, cluster.igfss, client, cfg);
-
-            if (!client)
-                this.clusterIgfss(cluster.igfss, cfg);
+    static igfsConfigurationBean(igfs) {
+        return new Bean('org.apache.ignite.configuration.FileSystemConfiguration', 'igfs', igfs, igfsDflts);
+    }
 
-            return cfg;
-        }
+    static cacheConfigurationBean(cache) {
+        return new Bean('org.apache.ignite.configuration.CacheConfiguration', 'ccfg', cache, cacheDflts);
+    }
 
-        static dialectClsName(dialect) {
-            return DFLT_DIALECTS[dialect] || 'Unknown database: ' + (dialect || 'Choose JDBC dialect');
-        }
+    static domainConfigurationBean(domain) {
+        return new Bean('org.apache.ignite.cache.QueryEntity', 'qryEntity', domain, cacheDflts);
+    }
 
-        static dataSourceBean(id, dialect) {
-            let dsBean;
+    static discoveryConfigurationBean(discovery) {
+        return new Bean('org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi', 'discovery', discovery, clusterDflts.discovery);
+    }
 
-            switch (dialect) {
-                case 'Generic':
-                    dsBean = new Bean('com.mchange.v2.c3p0.ComboPooledDataSource', id, {})
-                        .property('jdbcUrl', `${id}.jdbc.url`, 'jdbc:your_database');
+    /**
+     * Function to generate ignite configuration.
+     *
+     * @param {Object} cluster Cluster to process.
+     * @param {Boolean} client
+     * @return {Bean} Generated ignite configuration.
+     */
+    static igniteConfiguration(cluster, client) {
+        const cfg = this.igniteConfigurationBean(cluster);
+
+        this.clusterGeneral(cluster, cfg, client);
+        this.clusterAtomics(cluster.atomicConfiguration, cfg);
+        this.clusterBinary(cluster.binaryConfiguration, cfg);
+        this.clusterCacheKeyConfiguration(cluster.cacheKeyConfiguration, cfg);
+        this.clusterCheckpoint(cluster, cluster.caches, cfg);
+        this.clusterCollision(cluster.collision, cfg);
+        this.clusterCommunication(cluster, cfg);
+        this.clusterConnector(cluster.connector, cfg);
+        this.clusterDeployment(cluster, cfg);
+        this.clusterEvents(cluster, cfg);
+        this.clusterFailover(cluster, cfg);
+        this.clusterLoadBalancing(cluster, cfg);
+        this.clusterLogger(cluster.logger, cfg);
+        this.clusterODBC(cluster.odbc, cfg);
+        this.clusterMarshaller(cluster, cfg);
+        this.clusterMetrics(cluster, cfg);
+        this.clusterSwap(cluster, cfg);
+        this.clusterTime(cluster, cfg);
+        this.clusterPools(cluster, cfg);
+        this.clusterTransactions(cluster.transactionConfiguration, cfg);
+        this.clusterSsl(cluster, cfg);
+        this.clusterUserAttributes(cluster, cfg);
+
+        this.clusterCaches(cluster, cluster.caches, cluster.igfss, client, cfg);
+
+        if (!client)
+            this.clusterIgfss(cluster.igfss, cfg);
+
+        return cfg;
+    }
 
-                    break;
-                case 'Oracle':
-                    dsBean = new Bean('oracle.jdbc.pool.OracleDataSource', id, {})
-                        .property('URL', `${id}.jdbc.url`, 'jdbc:oracle:thin:@[host]:[port]:[database]');
+    static dialectClsName(dialect) {
+        return DFLT_DIALECTS[dialect] || 'Unknown database: ' + (dialect || 'Choose JDBC dialect');
+    }
 
-                    break;
-                case 'DB2':
-                    dsBean = new Bean('com.ibm.db2.jcc.DB2DataSource', id, {})
-                        .property('serverName', `${id}.jdbc.server_name`, 'YOUR_DATABASE_SERVER_NAME')
-                        .propertyInt('portNumber', `${id}.jdbc.port_number`, 'YOUR_JDBC_PORT_NUMBER')
-                        .property('databaseName', `${id}.jdbc.database_name`, 'YOUR_DATABASE_NAME')
-                        .propertyInt('driverType', `${id}.jdbc.driver_type`, 'YOUR_JDBC_DRIVER_TYPE');
+    static dataSourceBean(id, dialect) {
+        let dsBean;
+
+        switch (dialect) {
+            case 'Generic':
+                dsBean = new Bean('com.mchange.v2.c3p0.ComboPooledDataSource', id, {})
+                    .property('jdbcUrl', `${id}.jdbc.url`, 'jdbc:your_database');
+
+                break;
+            case 'Oracle':
+                dsBean = new Bean('oracle.jdbc.pool.OracleDataSource', id, {})
+                    .property('URL', `${id}.jdbc.url`, 'jdbc:oracle:thin:@[host]:[port]:[database]');
+
+                break;
+            case 'DB2':
+                dsBean = new Bean('com.ibm.db2.jcc.DB2DataSource', id, {})
+                    .property('serverName', `${id}.jdbc.server_name`, 'YOUR_DATABASE_SERVER_NAME')
+                    .propertyInt('portNumber', `${id}.jdbc.port_number`, 'YOUR_JDBC_PORT_NUMBER')
+                    .property('databaseName', `${id}.jdbc.database_name`, 'YOUR_DATABASE_NAME')
+                    .propertyInt('driverType', `${id}.jdbc.driver_type`, 'YOUR_JDBC_DRIVER_TYPE');
+
+                break;
+            case 'SQLServer':
+                dsBean = new Bean('com.microsoft.sqlserver.jdbc.SQLServerDataSource', id, {})
+                    .property('URL', `${id}.jdbc.url`, 'jdbc:sqlserver://[host]:[port][;databaseName=database]');
+
+                break;
+            case 'MySQL':
+                dsBean = new Bean('com.mysql.jdbc.jdbc2.optional.MysqlDataSource', id, {})
+                    .property('URL', `${id}.jdbc.url`, 'jdbc:mysql://[host]:[port]/[database]');
+
+                break;
+            case 'PostgreSQL':
+                dsBean = new Bean('org.postgresql.ds.PGPoolingDataSource', id, {})
+                    .property('url', `${id}.jdbc.url`, 'jdbc:postgresql://[host]:[port]/[database]');
+
+                break;
+            case 'H2':
+                dsBean = new Bean('org.h2.jdbcx.JdbcDataSource', id, {})
+                    .property('URL', `${id}.jdbc.url`, 'jdbc:h2:tcp://[host]/[database]');
+
+                break;
+            default:
+        }
 
-                    break;
-                case 'SQLServer':
-                    dsBean = new Bean('com.microsoft.sqlserver.jdbc.SQLServerDataSource', id, {})
-                        .property('URL', `${id}.jdbc.url`, 'jdbc:sqlserver://[host]:[port][;databaseName=database]');
+        if (dsBean) {
+            dsBean.property('user', `${id}.jdbc.username`, 'YOUR_USER_NAME')
+                .property('password', `${id}.jdbc.password`, 'YOUR_PASSWORD');
+        }
 
-                    break;
-                case 'MySQL':
-                    dsBean = new Bean('com.mysql.jdbc.jdbc2.optional.MysqlDataSource', id, {})
-                        .property('URL', `${id}.jdbc.url`, 'jdbc:mysql://[host]:[port]/[database]');
+        return dsBean;
+    }
 
-                    break;
-                case 'PostgreSQL':
-                    dsBean = new Bean('org.postgresql.ds.PGPoolingDataSource', id, {})
-                        .property('url', `${id}.jdbc.url`, 'jdbc:postgresql://[host]:[port]/[database]');
+    // Generate general section.
+    static clusterGeneral(cluster, cfg = this.igniteConfigurationBean(cluster), client = false) {
+        if (client)
+            cfg.prop('boolean', 'clientMode', true);
 
-                    break;
-                case 'H2':
-                    dsBean = new Bean('org.h2.jdbcx.JdbcDataSource', id, {})
-                        .property('URL', `${id}.jdbc.url`, 'jdbc:h2:tcp://[host]/[database]');
+        cfg.stringProperty('name', 'gridName')
+            .stringProperty('localHost');
 
-                    break;
-                default:
-            }
-
-            if (dsBean) {
-                dsBean.property('user', `${id}.jdbc.username`, 'YOUR_USER_NAME')
-                    .property('password', `${id}.jdbc.password`, 'YOUR_PASSWORD');
-            }
+        if (_.isNil(cluster.discovery))
+            return cfg;
 
-            return dsBean;
-        }
+        const discovery = new Bean('org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi', 'discovery',
+            cluster.discovery, clusterDflts.discovery);
 
-        // Generate general section.
-        static clusterGeneral(cluster, cfg = this.igniteConfigurationBean(cluster), client = false) {
-            if (client)
-                cfg.prop('boolean', 'clientMode', true);
+        let ipFinder;
 
-            cfg.stringProperty('name', 'gridName')
-                .stringProperty('localHost');
+        switch (discovery.valueOf('kind')) {
+            case 'Vm':
+                ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder',
+                    'ipFinder', cluster.discovery.Vm, clusterDflts.discovery.Vm);
 
-            if (_.isNil(cluster.discovery))
-                return cfg;
+                ipFinder.collectionProperty('addrs', 'addresses', cluster.discovery.Vm.addresses);
 
-            const discovery = new Bean('org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi', 'discovery',
-                cluster.discovery, clusterDflts.discovery);
+                break;
+            case 'Multicast':
+                ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder',
+                    'ipFinder', cluster.discovery.Multicast, clusterDflts.discovery.Multicast);
 
-            let ipFinder;
+                ipFinder.stringProperty('multicastGroup')
+                    .intProperty('multicastPort')
+                    .intProperty('responseWaitTime')
+                    .intProperty('addressRequestAttempts')
+                    .stringProperty('localAddress')
+                    .collectionProperty('addrs', 'addresses', cluster.discovery.Multicast.addresses);
 
-            switch (discovery.valueOf('kind')) {
-                case 'Vm':
-                    ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder',
-                        'ipFinder', cluster.discovery.Vm, clusterDflts.discovery.Vm);
+                break;
+            case 'S3':
+                ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.s3.TcpDiscoveryS3IpFinder',
+                    'ipFinder', cluster.discovery.S3, clusterDflts.discovery.S3);
 
-                    ipFinder.collectionProperty('addrs', 'addresses', cluster.discovery.Vm.addresses);
+                ipFinder.stringProperty('bucketName');
 
-                    break;
-                case 'Multicast':
-                    ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder',
-                        'ipFinder', cluster.discovery.Multicast, clusterDflts.discovery.Multicast);
+                break;
+            case 'Cloud':
+                ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.cloud.TcpDiscoveryCloudIpFinder',
+                    'ipFinder', cluster.discovery.Cloud, clusterDflts.discovery.Cloud);
 
-                    ipFinder.stringProperty('multicastGroup')
-                        .intProperty('multicastPort')
-                        .intProperty('responseWaitTime')
-                        .intProperty('addressRequestAttempts')
-                        .stringProperty('localAddress')
-                        .collectionProperty('addrs', 'addresses', cluster.discovery.Multicast.addresses);
+                ipFinder.stringProperty('credential')
+                    .pathProperty('credentialPath')
+                    .stringProperty('identity')
+                    .stringProperty('provider')
+                    .collectionProperty('regions', 'regions', cluster.discovery.Cloud.regions)
+                    .collectionProperty('zones', 'zones', cluster.discovery.Cloud.zones);
 
-                    break;
-                case 'S3':
-                    ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.s3.TcpDiscoveryS3IpFinder',
-                        'ipFinder', cluster.discovery.S3, clusterDflts.discovery.S3);
+                break;
+            case 'GoogleStorage':
+                ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.gce.TcpDiscoveryGoogleStorageIpFinder',
+                    'ipFinder', cluster.discovery.GoogleStorage, clusterDflts.discovery.GoogleStorage);
 
-                    ipFinder.stringProperty('bucketName');
+                ipFinder.stringProperty('projectName')
+                    .stringProperty('bucketName')
+                    .pathProperty('serviceAccountP12FilePath')
+                    .stringProperty('serviceAccountId');
 
-                    break;
-                case 'Cloud':
-                    ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.cloud.TcpDiscoveryCloudIpFinder',
-                        'ipFinder', cluster.discovery.Cloud, clusterDflts.discovery.Cloud);
+                break;
+            case 'Jdbc':
+                ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.jdbc.TcpDiscoveryJdbcIpFinder',
+                    'ipFinder', cluster.discovery.Jdbc, clusterDflts.discovery.Jdbc);
 
-                    ipFinder.stringProperty('credential')
-                        .pathProperty('credentialPath')
-                        .stringProperty('identity')
-                        .stringProperty('provider')
-                        .collectionProperty('regions', 'regions', cluster.discovery.Cloud.regions)
-                        .collectionProperty('zones', 'zones', cluster.discovery.Cloud.zones);
+                ipFinder.intProperty('initSchema');
 
-                    break;
-                case 'GoogleStorage':
-                    ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.gce.TcpDiscoveryGoogleStorageIpFinder',
-                        'ipFinder', cluster.discovery.GoogleStorage, clusterDflts.discovery.GoogleStorage);
+                if (ipFinder.includes('dataSourceBean', 'dialect')) {
+                    const id = ipFinder.valueOf('dataSourceBean');
 
-                    ipFinder.stringProperty('projectName')
-                        .stringProperty('bucketName')
-                        .pathProperty('serviceAccountP12FilePath')
-                        .stringProperty('serviceAccountId');
+                    ipFinder.dataSource(id, 'dataSource', this.dataSourceBean(id, ipFinder.valueOf('dialect')));
+                }
 
-                    break;
-                case 'Jdbc':
-                    ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.jdbc.TcpDiscoveryJdbcIpFinder',
-                        'ipFinder', cluster.discovery.Jdbc, clusterDflts.discovery.Jdbc);
+                break;
+            case 'SharedFs':
+                ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.sharedfs.TcpDiscoverySharedFsIpFinder',
+                    'ipFinder', cluster.discovery.SharedFs, clusterDflts.discovery.SharedFs);
+
+                ipFinder.pathProperty('path');
+
+                break;
+            case 'ZooKeeper':
+                const src = cluster.discovery.ZooKeeper;
+                const dflt = clusterDflts.discovery.ZooKeeper;
+
+                ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.zk.TcpDiscoveryZookeeperIpFinder',
+                    'ipFinder', src, dflt);
+
+                ipFinder.emptyBeanProperty('curator')
+                    .stringProperty('zkConnectionString');
+
+                const kind = _.get(src, 'retryPolicy.kind');
+
+                if (kind) {
+                    const policy = src.retryPolicy;
+
+                    let retryPolicyBean;
+
+                    switch (kind) {
+                        case 'ExponentialBackoff':
+                            retryPolicyBean = new Bean('org.apache.curator.retry.ExponentialBackoffRetry', null,
+                                policy.ExponentialBackoff, dflt.ExponentialBackoff)
+                                .intConstructorArgument('baseSleepTimeMs')
+                                .intConstructorArgument('maxRetries')
+                                .intConstructorArgument('maxSleepMs');
+
+                            break;
+                        case 'BoundedExponentialBackoff':
+                            retryPolicyBean = new Bean('org.apache.curator.retry.BoundedExponentialBackoffRetry',
+                                null, policy.BoundedExponentialBackoff, dflt.BoundedExponentialBackoffRetry)
+                                .intConstructorArgument('baseSleepTimeMs')
+                                .intConstructorArgument('maxSleepTimeMs')
+                                .intConstructorArgument('maxRetries');
+
+                            break;
+                        case 'UntilElapsed':
+                            retryPolicyBean = new Bean('org.apache.curator.retry.RetryUntilElapsed', null,
+                                policy.UntilElapsed, dflt.UntilElapsed)
+                                .intConstructorArgument('maxElapsedTimeMs')
+                                .intConstructorArgument('sleepMsBetweenRetries');
+
+                            break;
+
+                        case 'NTimes':
+                            retryPolicyBean = new Bean('org.apache.curator.retry.RetryNTimes', null,
+                                policy.NTimes, dflt.NTimes)
+                                .intConstructorArgument('n')
+                                .intConstructorArgument('sleepMsBetweenRetries');
+
+                            break;
+                        case 'OneTime':
+                            retryPolicyBean = new Bean('org.apache.curator.retry.RetryOneTime', null,
+                                policy.OneTime, dflt.OneTime)
+                                .intConstructorArgument('sleepMsBetweenRetry');
+
+                            break;
+                        case 'Forever':
+                            retryPolicyBean = new Bean('org.apache.curator.retry.RetryForever', null,
+                                policy.Forever, dflt.Forever)
+                                .intConstructorArgument('retryIntervalMs');
+
+                            break;
+                        case 'Custom':
+                            const className = _.get(policy, 'Custom.className');
+
+                            if (_.nonEmpty(className))
+                                retryPolicyBean = new EmptyBean(className);
+
+                            break;
+                        default:
+                            // No-op.
+                    }
 
-                    ipFinder.intProperty('initSchema');
+                    if (retryPolicyBean)
+                        ipFinder.beanProperty('retryPolicy', retryPolicyBean);
+                }
 
-                    if (ipFinder.includes('dataSourceBean', 'dialect')) {
-                        const id = ipFinder.valueOf('dataSourceBean');
+                ipFinder.pathProperty('basePath', '/services')
+                    .stringProperty('serviceName')
+                    .boolProperty('allowDuplicateRegistrations');
 
-                        ipFinder.dataSource(id, 'dataSource', this.dataSourceBean(id, ipFinder.valueOf('dialect')));
-                    }
+                break;
+            default:
+                // No-op.
+        }
 
-                    break;
-                case 'SharedFs':
-                    ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.sharedfs.TcpDiscoverySharedFsIpFinder',
-                        'ipFinder', cluster.discovery.SharedFs, clusterDflts.discovery.SharedFs);
+        if (ipFinder)
+            discovery.beanProperty('ipFinder', ipFinder);
 
-                    ipFinder.pathProperty('path');
+        this.clusterDiscovery(cluster.discovery, cfg, discovery);
 
-                    break;
-                case 'ZooKeeper':
-                    const src = cluster.discovery.ZooKeeper;
-                    const dflt = clusterDflts.discovery.ZooKeeper;
+        return cfg;
+    }
 
-                    ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.zk.TcpDiscoveryZookeeperIpFinder',
-                        'ipFinder', src, dflt);
+    static igfsDataCache(igfs) {
+        return this.cacheConfiguration({
+            name: igfs.name + '-data',
+            cacheMode: 'PARTITIONED',
+            atomicityMode: 'TRANSACTIONAL',
+            writeSynchronizationMode: 'FULL_SYNC',
+            backups: 0,
+            igfsAffinnityGroupSize: igfs.affinnityGroupSize || 512
+        });
+    }
 
-                    ipFinder.emptyBeanProperty('curator')
-                        .stringProperty('zkConnectionString');
+    static igfsMetaCache(igfs) {
+        return this.cacheConfiguration({
+            name: igfs.name + '-meta',
+            cacheMode: 'REPLICATED',
+            atomicityMode: 'TRANSACTIONAL',
+            writeSynchronizationMode: 'FULL_SYNC'
+        });
+    }
 
-                    const kind = _.get(src, 'retryPolicy.kind');
+    static clusterCaches(cluster, caches, igfss, client, cfg = this.igniteConfigurationBean(cluster)) {
+        const ccfgs = _.map(caches, (cache) => this.cacheConfiguration(cache));
 
-                    if (kind) {
-                        const policy = src.retryPolicy;
+        if (!client) {
+            _.forEach(igfss, (igfs) => {
+                ccfgs.push(this.igfsDataCache(igfs));
+                ccfgs.push(this.igfsMetaCache(igfs));
+            });
+        }
 
-                        let retryPolicyBean;
+        cfg.varArgProperty('ccfgs', 'cacheConfiguration', ccfgs, 'org.apache.ignite.configuration.CacheConfiguration');
 
-                        switch (kind) {
-                            case 'ExponentialBackoff':
-                                retryPolicyBean = new Bean('org.apache.curator.retry.ExponentialBackoffRetry', null,
-                                    policy.ExponentialBackoff, dflt.ExponentialBackoff)
-                                    .intConstructorArgument('baseSleepTimeMs')
-                                    .intConstructorArgument('maxRetries')
-                                    .intConstructorArgument('maxSleepMs');
+        return cfg;
+    }
 
-                                break;
-                            case 'BoundedExponentialBackoff':
-                                retryPolicyBean = new Bean('org.apache.curator.retry.BoundedExponentialBackoffRetry',
-                                    null, policy.BoundedExponentialBackoffRetry, dflt.BoundedExponentialBackoffRetry)
-                                    .intConstructorArgument('baseSleepTimeMs')
-                                    .intConstructorArgument('maxSleepTimeMs')
-                                    .intConstructorArgument('maxRetries');
+    // Generate atomics group.
+    static clusterAtomics(atomics, cfg = this.igniteConfigurationBean()) {
+        const acfg = new Bean('org.apache.ignite.configuration.AtomicConfiguration', 'atomicCfg',
+            atomics, clusterDflts.atomics);
 
-                                break;
-                            case 'UntilElapsed':
-                                retryPolicyBean = new Bean('org.apache.curator.retry.RetryUntilElapsed', null,
-                                    policy.UntilElapsed, dflt.UntilElapsed)
-                                    .intConstructorArgument('maxElapsedTimeMs')
-                                    .intConstructorArgument('sleepMsBetweenRetries');
+        acfg.enumProperty('cacheMode')
+            .intProperty('atomicSequenceReserveSize');
 
-                                break;
+        if (acfg.valueOf('cacheMode') === 'PARTITIONED')
+            acfg.intProperty('backups');
 
-                            case 'NTimes':
-                                retryPolicyBean = new Bean('org.apache.curator.retry.RetryNTimes', null,
-                                    policy.NTimes, dflt.NTimes)
-                                    .intConstructorArgument('n')
-                                    .intConstructorArgument('sleepMsBetweenRetries');
+        if (acfg.isEmpty())
+            return cfg;
 
-                                break;
-                            case 'OneTime':
-                                retryPolicyBean = new Bean('org.apache.curator.retry.RetryOneTime', null,
-                                    policy.OneTime, dflt.OneTime)
-                                    .intConstructorArgument('sleepMsBetweenRetry');
+        cfg.beanProperty('atomicConfiguration', acfg);
 
-                                break;
-                            case 'Forever':
-                                retryPolicyBean = new Bean('org.apache.curator.retry.RetryForever', null,
-                                    policy.Forever, dflt.Forever)
-                                    .intConstructorArgument('retryIntervalMs');
+        return cfg;
+    }
 
-                                break;
-                            case 'Custom':
-                                if (_.nonEmpty(policy.Custom.className))
-                                    retryPolicyBean = new EmptyBean(policy.Custom.className);
+    // Generate binary group.
+    static clusterBinary(binary, cfg = this.igniteConfigurationBean()) {
+        const binaryCfg = new Bean('org.apache.ignite.configuration.BinaryConfiguration', 'binaryCfg',
+            binary, clusterDflts.binary);
 
-                                break;
-                            default:
-                                // No-op.
-                        }
+        binaryCfg.emptyBeanProperty('idMapper')
+            .emptyBeanProperty('nameMapper')
+            .emptyBeanProperty('serializer');
 
-                        if (retryPolicyBean)
-                            ipFinder.beanProperty('retryPolicy', retryPolicyBean);
-                    }
+        const typeCfgs = [];
 
-                    ipFinder.pathProperty('basePath', '/services')
-                        .stringProperty('serviceName')
-                        .boolProperty('allowDuplicateRegistrations');
+        _.forEach(binary.typeConfigurations, (type) => {
+            const typeCfg = new Bean('org.apache.ignite.binary.BinaryTypeConfiguration',
+                javaTypes.toJavaName('binaryType', type.typeName), type, clusterDflts.binary.typeConfigurations);
 
-                    break;
-                default:
-                    // No-op.
-            }
+            typeCfg.stringProperty('typeName')
+                .emptyBeanProperty('idMapper')
+                .emptyBeanProperty('nameMapper')
+                .emptyBeanProperty('serializer')
+                .intProperty('enum');
 
-            if (ipFinder)
-                discovery.beanProperty('ipFinder', ipFinder);
+            if (typeCfg.nonEmpty())
+                typeCfgs.push(typeCfg);
+        });
 
-            this.clusterDiscovery(cluster.discovery, cfg, discovery);
+        binaryCfg.collectionProperty('types', 'typeConfigurations', typeCfgs, 'org.apache.ignite.binary.BinaryTypeConfiguration')
+            .boolProperty('compactFooter');
 
+        if (binaryCfg.isEmpty())
             return cfg;
-        }
-
-        static igfsDataCache(igfs) {
-            return this.cacheConfiguration({
-                name: igfs.name + '-data',
-                cacheMode: 'PARTITIONED',
-                atomicityMode: 'TRANSACTIONAL',
-                writeSynchronizationMode: 'FULL_SYNC',
-                backups: 0,
-                igfsAffinnityGroupSize: igfs.affinnityGroupSize || 512
-            });
-        }
 
-        static igfsMetaCache(igfs) {
-            return this.cacheConfiguration({
-                name: igfs.name + '-meta',
-                cacheMode: 'REPLICATED',
-                atomicityMode: 'TRANSACTIONAL',
-                writeSynchronizationMode: 'FULL_SYNC'
-            });
-        }
+        cfg.beanProperty('binaryConfiguration', binaryCfg);
 
-        static clusterCaches(cluster, caches, igfss, client, cfg = this.igniteConfigurationBean(cluster)) {
-            const ccfgs = _.map(caches, (cache) => this.cacheConfiguration(cache));
+        return cfg;
+    }
 
-            if (!client) {
-                _.forEach(igfss, (igfs) => {
-                    ccfgs.push(this.igfsDataCache(igfs));
-                    ccfgs.push(this.igfsMetaCache(igfs));
-                });
+    // Generate cache key configurations.
+    static clusterCacheKeyConfiguration(keyCfgs, cfg = this.igniteConfigurationBean()) {
+        const items = _.reduce(keyCfgs, (acc, keyCfg) => {
+            if (keyCfg.typeName && keyCfg.affinityKeyFieldName) {
+                acc.push(new Bean('org.apache.ignite.cache.CacheKeyConfiguration', null, keyCfg)
+                    .stringConstructorArgument('typeName')
+                    .stringConstructorArgument('affinityKeyFieldName'));
             }
 
-            cfg.varArgProperty('ccfgs', 'cacheConfiguration', ccfgs, 'org.apache.ignite.configuration.CacheConfiguration');
+            return acc;
+        }, []);
 
+        if (_.isEmpty(items))
             return cfg;
-        }
 
-        // Generate atomics group.
-        static clusterAtomics(atomics, cfg = this.igniteConfigurationBean()) {
-            const acfg = new Bean('org.apache.ignite.configuration.AtomicConfiguration', 'atomicCfg',
-                atomics, clusterDflts.atomics);
+        cfg.arrayProperty('cacheKeyConfiguration', 'cacheKeyConfiguration', items,
+            'org.apache.ignite.cache.CacheKeyConfiguration');
 
-            acfg.enumProperty('cacheMode')
-                .intProperty('atomicSequenceReserveSize');
+        return cfg;
+    }
 
-            if (acfg.valueOf('cacheMode') === 'PARTITIONED')
-                acfg.intProperty('backups');
+    // Generate checkpoint configurations.
+    static clusterCheckpoint(cluster, caches, cfg = this.igniteConfigurationBean()) {
+        const cfgs = _.filter(_.map(cluster.checkpointSpi, (spi) => {
+            switch (_.get(spi, 'kind')) {
+                case 'FS':
+                    const fsBean = new Bean('org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointSpi',
+                        'checkpointSpiFs', spi.FS);
 
-            if (acfg.isEmpty())
-                return cfg;
+                    fsBean.collectionProperty('directoryPaths', 'directoryPaths', _.get(spi, 'FS.directoryPaths'))
+                        .emptyBeanProperty('checkpointListener');
 
-            cfg.beanProperty('atomicConfiguration', acfg);
+                    return fsBean;
 
-            return cfg;
-        }
+                case 'Cache':
+                    const cacheBean = new Bean('org.apache.ignite.spi.checkpoint.cache.CacheCheckpointSpi',
+                        'checkpointSpiCache', spi.Cache);
 
-        // Generate binary group.
-        static clusterBinary(binary, cfg = this.igniteConfigurationBean()) {
-            const binaryCfg = new Bean('org.apache.ignite.configuration.BinaryConfiguration', 'binaryCfg',
-                binary, clusterDflts.binary);
+                    const curCache = _.get(spi, 'Cache.cache');
 
-            binaryCfg.emptyBeanProperty('idMapper')
-                .emptyBeanProperty('nameMapper')
-                .emptyBeanProperty('serializer');
+                    const cache = _.find(caches, (c) => curCache && (c._id === curCache || _.get(c, 'cache._id') === curCache));
 
-            const typeCfgs = [];
+                    if (cache)
+                        cacheBean.prop('java.lang.String', 'cacheName', cache.name || cache.cache.name);
 
-            _.forEach(binary.typeConfigurations, (type) => {
-                const typeCfg = new Bean('org.apache.ignite.binary.BinaryTypeConfiguration',
-                    JavaTypes.toJavaName('binaryType', type.typeName), type, clusterDflts.binary.typeConfigurations);
+                    cacheBean.stringProperty('cacheName')
+                        .emptyBeanProperty('checkpointListener');
 
-                typeCfg.stringProperty('typeName')
-                    .emptyBeanProperty('idMapper')
-                    .emptyBeanProperty('nameMapper')
-                    .emptyBeanProperty('serializer')
-                    .intProperty('enum');
+                    return cacheBean;
 
-                if (typeCfg.nonEmpty())
-                    typeCfgs.push(typeCfg);
-            });
+                case 'S3':
+                    const s3Bean = new Bean('org.apache.ignite.spi.checkpoint.s3.S3CheckpointSpi',
+                        'checkpointSpiS3', spi.S3, clusterDflts.checkpointSpi.S3);
 
-            binaryCfg.collectionProperty('types', 'typeConfigurations', typeCfgs, 'org.apache.ignite.binary.BinaryTypeConfiguration')
-                .boolProperty('compactFooter');
+                    let credentialsBean = null;
 
-            if (binaryCfg.isEmpty())
-                return cfg;
+                    switch (_.get(spi.S3, 'awsCredentials.kind')) {
+                        case 'Basic':
+                            credentialsBean = new Bean('com.amazonaws.auth.BasicAWSCredentials', 'awsCredentials', {});
 
-            cfg.beanProperty('binaryConfiguration', binaryCfg);
+                            credentialsBean.propertyConstructorArgument('checkpoint.s3.credentials.accessKey', 'YOUR_S3_ACCESS_KEY')
+                                .propertyConstructorArgument('checkpoint.s3.credentials.secretKey', 'YOUR_S3_SECRET_KEY');
 
-            return cfg;
-        }
+                            break;
 
-        // Generate cache key configurations.
-        static clusterCacheKeyConfiguration(keyCfgs, cfg = this.igniteConfigurationBean()) {
-            const items = _.reduce(keyCfgs, (acc, keyCfg) => {
-                if (keyCfg.typeName && keyCfg.affinityKeyFieldName) {
-                    acc.push(new Bean('org.apache.ignite.cache.CacheKeyConfiguration', null, keyCfg)
-                        .stringConstructorArgument('typeName')
-                        .stringConstructorArgument('affinityKeyFieldName'));
-                }
+                        case 'Properties':
+                            credentialsBean = new Bean('com.amazonaws.auth.PropertiesCredentials', 'awsCredentials', {});
 
-                return acc;
-            }, []);
+                            const fileBean = new Bean('java.io.File', '', spi.S3.awsCredentials.Properties)
+                                .pathConstructorArgument('path');
 
-            if (_.isEmpty(items))
-                return cfg;
+                            if (fileBean.nonEmpty())
+                                credentialsBean.beanConstructorArgument('file', fileBean);
 
-            cfg.arrayProperty('cacheKeyConfiguration', 'cacheKeyConfiguration', items,
-                'org.apache.ignite.cache.CacheKeyConfiguration');
+                            break;
 
-            return cfg;
-        }
+                        case 'Anonymous':
+                            credentialsBean = new Bean('com.amazonaws.auth.AnonymousAWSCredentials', 'awsCredentials', {});
 
-        // Generate checkpoint configurations.
-        static clusterCheckpoint(cluster, caches, cfg = this.igniteConfigurationBean()) {
-            const cfgs = _.filter(_.map(cluster.checkpointSpi, (spi) => {
-                switch (_.get(spi, 'kind')) {
-                    case 'FS':
-                        const fsBean = new Bean('org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointSpi',
-                            'checkpointSpiFs', spi.FS);
+                            break;
 
-                        fsBean.collectionProperty('directoryPaths', 'directoryPaths', _.get(spi, 'FS.directoryPaths'))
-                            .emptyBeanProperty('checkpointListener');
+                        case 'BasicSession':
+                            credentialsBean = new Bean('com.amazonaws.auth.BasicSessionCredentials', 'awsCredentials', {});
 
-                        return fsBean;
+                            // TODO 2054 Arguments in one line is very long string.
+                            credentialsBean.propertyConstructorArgument('checkpoint.s3.credentials.accessKey')
+                                .propertyConstructorArgument('checkpoint.s3.credentials.secretKey')
+                                .propertyConstructorArgument('checkpoint.s3.credentials.sessionToken');
 
-                    case 'Cache':
-                        const cacheBean = new Bean('org.apache.ignite.spi.checkpoint.cache.CacheCheckpointSpi',
-                            'checkpointSpiCache', spi.Cache);
+                            break;
 
-                        const curCache = _.get(spi, 'Cache.cache');
+                        case 'Custom':
+                            const className = _.get(spi.S3.awsCredentials, 'Custom.className');
 
-                        const cache = _.find(caches, (c) => curCache && (c._id === curCache || _.get(c, 'cache._id') === curCache));
+                            if (className)
+                                credentialsBean = new Bean(className, 'awsCredentials', {});
 
-                        if (cache)
-                            cacheBean.prop('java.lang.String', 'cacheName', cache.name || cache.cache.name);
+                            break;
 
-                        cacheBean.stringProperty('cacheName')
-                            .emptyBeanProperty('checkpointListener');
+                        default:
+                            break;
+                    }
 
-                        return cacheBean;
+                    if (credentialsBean)
+                        s3Bean.beanProperty('awsCredentials', credentialsBean);
 
-                    case 'S3':
-                        const s3Bean = new Bean('org.apache.ignite.spi.checkpoint.s3.S3CheckpointSpi',
-                            'checkpointSpiS3', spi.S3, clusterDflts.checkpointSpi.S3);
+                    s3Bean.stringProperty('bucketNameSuffix');
 
-                        let credentialsBean = null;
+                    const clientBean = new Bean('com.amazonaws.ClientConfiguration', 'clientCfg', spi.S3.clientConfiguration,
+                        clusterDflts.checkpointSpi.S3.clientConfiguration);
 
-                        switch (_.get(spi.S3, 'awsCredentials.kind')) {
-                            case 'Basic':
-                                credentialsBean = new Bean('com.amazonaws.auth.BasicAWSCredentials', 'awsCredentials', {});
+                    clientBean.enumProperty('protocol')
+                        .intProperty('maxConnections')
+                        .stringProperty('userAgent');
 
-                                credentialsBean.propertyConstructorArgument('checkpoint.s3.credentials.accessKey', 'YOUR_S3_ACCESS_KEY')
-                                    .propertyConstructorArgument('checkpoint.s3.credentials.secretKey', 'YOUR_S3_SECRET_KEY');
+                    const locAddr = new Bean('java.net.InetAddress', '', spi.S3.clientConfiguration)
+                        .factoryMethod('getByName')
+                        .stringConstructorArgument('localAddress');
 
-                                break;
+                    if (locAddr.nonEmpty())
+                        clientBean.beanProperty('localAddress', locAddr);
 
-                            case 'Properties':
-                                credentialsBean = new Bean('com.amazonaws.auth.PropertiesCredentials', 'awsCredentials', {});
+                    clientBean.stringProperty('proxyHost')
+                        .intProperty('proxyPort')
+                        .stringProperty('proxyUsername');
 
-                                const fileBean = new Bean('java.io.File', '', spi.S3.awsCredentials.Properties)
-                                    .pathConstructorArgument('path');
+                    const userName = clientBean.valueOf('proxyUsername');
 
-                                if (fileBean.nonEmpty())
-                                    credentialsBean.beanConstructorArgument('file', fileBean);
+                    if (userName)
+                        clientBean.property('proxyPassword', `checkpoint.s3.proxy.${userName}.password`);
 
-                                break;
+                    clientBean.stringProperty('proxyDomain')
+                        .stringProperty('proxyWorkstation');
 
-                            case 'Anonymous':
-                                credentialsBean = new Bean('com.amazonaws.auth.AnonymousAWSCredentials', 'awsCredentials', {});
+                    const retryPolicy = spi.S3.clientConfiguration.retryPolicy;
 
-                                break;
+                    if (retryPolicy) {
+                        const kind = retryPolicy.kind;
 
-                            case 'BasicSession':
-                                credentialsBean = new Bean('com.amazonaws.auth.BasicSessionCredentials', 'awsCredentials', {});
+                        const policy = retryPolicy[kind];
 
-                                // TODO 2054 Arguments in one line is very long string.
-                                credentialsBean.propertyConstructorArgument('checkpoint.s3.credentials.accessKey')
-                                    .propertyConstructorArgument('checkpoint.s3.credentials.secretKey')
-                                    .propertyConstructorArgument('checkpoint.s3.credentials.sessionToken');
+                        let retryBean;
+
+                        switch (kind) {
+                            case 'Default':
+                                retryBean = new Bean('com.amazonaws.retry.RetryPolicy', 'retryPolicy', {
+                                    retryCondition: 'DEFAULT_RETRY_CONDITION',
+                                    backoffStrategy: 'DEFAULT_BACKOFF_STRATEGY',
+                                    maxErrorRetry: 'DEFAULT_MAX_ERROR_RETRY',
+                                    honorMaxErrorRetryInClientConfig: true
+                                }, clusterDflts.checkpointSpi.S3.clientConfiguration.retryPolicy);
+
+                                retryBean.constantConstructorArgument('retryCondition')
+                                    .constantConstructorArgument('backoffStrategy')
+                                    .constantConstructorArgument('maxErrorRetry')
+                                    .constructorArgument('java.lang.Boolean', retryBean.valueOf('honorMaxErrorRetryInClientConfig'));
 
                                 break;
 
-                            case 'Custom':
-                                const className = _.get(spi.S3.awsCredentials, 'Custom.className');
+                            case 'DefaultMaxRetries':
+                                retryBean = new Bean('com.amazonaws.retry.RetryPolicy', 'retryPolicy', {
+                                    retryCondition: 'DEFAULT_RETRY_CONDITION',
+                                    backoffStrategy: 'DEFAULT_BACKOFF_STRATEGY',
+                                    maxErrorRetry: _.get(policy, 'maxErrorRetry') || -1,
+                                    honorMaxErrorRetryInClientConfig: false
+                                }, clusterDflts.checkpointSpi.S3.clientConfiguration.retryPolicy);
 
-                                credentialsBean = new Bean(className, 'awsCredentials', {});
+                                retryBean.constantConstructorArgument('retryCondition')
+                                    .constantConstructorArgument('backoffStrategy')
+                                    .constructorArgument('java.lang.Integer', retryBean.valueOf('maxErrorRetry'))
+                                    .constructorArgument('java.lang.Boolean', retryBean.valueOf('honorMaxErrorRetryInClientConfig'));
 
                                 break;
 
-                            default:
-                                break;
-                        }
+                            case 'DynamoDB':
+                                retryBean = new Bean('com.amazonaws.retry.RetryPolicy', 'retryPolicy', {
+                                    retryCondition: 'DEFAULT_RETRY_CONDITION',
+                                    backoffStrategy: 'DYNAMODB_DEFAULT_BACKOFF_STRATEGY',
+                                    maxErrorRetry: 'DYNAMODB_DEFAULT_MAX_ERROR_RETRY',
+                                    honorMaxErrorRetryInClientConfig: true
+                                }, clusterDflts.checkpointSpi.S3.clientConfiguration.retryPolicy);
 
-                        if (credentialsBean)
-                            s3Bean.beanProperty('awsCredentials', credentialsBean);
+                                retryBean.constantConstructorArgument('retryCondition')
+                                    .constantConstructorArgument('backoffStrategy')
+                                    .constantConstructorArgument('maxErrorRetry')
+                                    .constructorArgument('java.lang.Boolean', retryBean.valueOf('honorMaxErrorRetryInClientConfig'));
 
-                        s3Bean.stringProperty('bucketNameSuffix');
+                                break;
 
-                        const clientBean = new Bean('com.amazonaws.ClientConfiguration', 'clientCfg', spi.S3.clientConfiguration,
-                            clusterDflts.checkpointSpi.S3.clientConfiguration);
+                            case 'DynamoDBMaxRetries':
+                                retryBean = new Bean('com.amazonaws.retry.RetryPolicy', 'retryPolicy', {
+                                    retryCondition: 'DEFAULT_RETRY_CONDITION',
+                                    backoffStrategy: 'DYNAMODB_DEFAULT_BACKOFF_STRATEGY',
+                                    maxErrorRetry: _.get(policy, 'maxErrorRetry') || -1,
+                                    honorMaxErrorRetryInClientConfig: false
+                                }, clusterDflts.checkpointSpi.S3.clientConfiguration.retryPolicy);
 
-                        clientBean.enumProperty('protocol')
-                            .intProperty('maxConnections')
-                            .stringProperty('userAgent');
+                                retryBean.constantConstructorArgument('retryCondition')
+                                    .constantConstructorArgument('backoffStrategy')
+                                    .constructorArgument('java.lang.Integer', retryBean.valueOf('maxErrorRetry'))
+                                    .constructorArgument('java.lang.Boolean', retryBean.valueOf('honorMaxErrorRetryInClientConfig'));
 
-                        const locAddr = new Bean('java.net.InetAddress', '', spi.S3.clientConfiguration)
-                            .factoryMethod('getByName')
-                            .stringConstructorArgument('localAddress');
+                                break;
 
-                        if (locAddr.nonEmpty())
-                            clientBean.beanProperty('localAddress', locAddr);
+                            case 'Custom':
+                                retryBean = new Bean('com.amazonaws.retry.RetryPolicy', 'retryPolicy', policy);
 
-                        clientBean.stringProperty('proxyHost')
-                            .intProperty('proxyPort')
-                            .stringProperty('proxyUsername');
+                                retryBean.beanConstructorArgument('retryCondition', retryBean.valueOf('retryCondition') ? new EmptyBean(retryBean.valueOf('retryCondition')) : null)
+                                    .beanConstructorArgument('backoffStrategy', retryBean.valueOf('backoffStrategy') ? new EmptyBean(retryBean.valueOf('backoffStrategy')) : null)
+                                    .constructorArgument('java.lang.Integer', retryBean.valueOf('maxErrorRetry'))
+                                    .constructorArgument('java.lang.Boolean', retryBean.valueOf('honorMaxErrorRetryInClientConfig'));
 
-                        const userName = clientBean.valueOf('proxyUsername');
+                                break;
 
-                        if (userName)
-                            clientBean.property('proxyPassword', `checkpoint.s3.proxy.${userName}.password`);
+                            default:
+                                break;
+                        }
 
-                        clientBean.stringProperty('proxyDomain')
-                            .stringProperty('proxyWorkstation');
+                        if (retryBean)
+                            clientBean.beanProperty('retryPolicy', retryBean);
+                    }
 
-                        const retryPolicy = spi.S3.clientConfiguration.retryPolicy;
+                    clientBean.intProperty('maxErrorRetry')
+                        .intProperty('socketTimeout')
+                        .intProperty('connectionTimeout')
+                        .intProperty('requestTimeout')
+                        .intProperty('socketSendBufferSizeHints')
+                        .stringProperty('signerOverride')
+                        .intProperty('connectionTTL')
+                        .intProperty('connectionMaxIdleMillis')
+                        .emptyBeanProperty('dnsResolver')
+                        .intProperty('responseMetadataCacheSize')
+                        .emptyBeanProperty('secureRandom')
+                        .boolProperty('useReaper')
+                        .boolProperty('useGzip')
+                        .boolProperty('preemptiveBasicProxyAuth')
+                        .boolProperty('useTcpKeepAlive');
+
+                    if (clientBean.nonEmpty())
+                        s3Bean.beanProperty('clientConfiguration', clientBean);
+
+                    s3Bean.emptyBeanProperty('checkpointListener');
+
+                    return s3Bean;
+
+                case 'JDBC':
+                    const jdbcBean = new Bean('org.apache.ignite.spi.checkpoint.jdbc.JdbcCheckpointSpi',
+                        'checkpointSpiJdbc', spi.JDBC, clusterDflts.checkpointSpi.JDBC);
+
+                    const id = jdbcBean.valueOf('dataSourceBean');
+                    const dialect = _.get(spi.JDBC, 'dialect');
+
+                    jdbcBean.dataSource(id, 'dataSource', this.dataSourceBean(id, dialect));
+
+                    if (!_.isEmpty(jdbcBean.valueOf('user'))) {
+                        jdbcBean.stringProperty('user')
+                            .property('pwd', `checkpoint.${jdbcBean.valueOf('dataSourceBean')}.${jdbcBean.valueOf('user')}.jdbc.password`, 'YOUR_PASSWORD');
+                    }
 
-                        if (retryPolicy) {
-                            const kind = retryPolicy.kind;
+                    jdbcBean.stringProperty('checkpointTableName')
+                        .stringProperty('keyFieldName')
+                        .stringProperty('keyFieldType')
+                        .stringProperty('valueFieldName')
+                        .stringProperty('valueFieldType')
+                        .stringProperty('expireDateFieldName')
+                        .stringProperty('expireDateFieldType')
+                        .intProperty('numberOfRetries')
+                        .emptyBeanProperty('checkpointListener');
 
-                            const policy = retryPolicy[kind];
+                    return jdbcBean;
 
-                            let retryBean;
+                case 'Custom':
+                    const clsName = _.get(spi, 'Custom.className');
 
-                            switch (kind) {
-                                case 'Default':
-                                    retryBean = new Bean('com.amazonaws.retry.RetryPolicy', 'retryPolicy', {
-                                        retryCondition: 'DEFAULT_RETRY_CONDITION',
-                                        backoffStrategy: 'DEFAULT_BACKOFF_STRATEGY',
-                                        maxErrorRetry: 'DEFAULT_MAX_ERROR_RETRY',
-                                        honorMaxErrorRetryInClientConfig: true
-                                    }, clusterDflts.checkpointSpi.S3.clientConfiguration.retryPolicy);
+                    if (clsName)
+                        return new Bean(clsName, 'checkpointSpiCustom', spi.Cache);
 
-                                    retryBean.constantConstructorArgument('retryCondition')
-                                        .constantConstructorArgument('backoffStrategy')
-                                        .constantConstructorArgument('maxErrorRetry')
-                                        .constructorArgument('java.lang.Boolean', retryBean.valueOf('honorMaxErrorRetryInClientConfig'));
+                    return null;
 
-                                    break;
+                default:
+                    return null;
+            }
+        }), (checkpointBean) => _.nonNil(checkpointBean));
 
-                                case 'DefaultMaxRetries':
-                                    retryBean = new Bean('com.amazonaws.retry.RetryPolicy', 'retryPolicy', {
-                                        retryCondition: 'DEFAULT_RETRY_CONDITION',
-                                        backoffStrategy: 'DEFAULT_BACKOFF_STRATEGY',
-                                        maxErrorRetry: _.get(policy, 'maxErrorRetry') || -1,
-                                        honorMaxErrorRetryInClientConfig: false
-                                    }, clusterDflts.checkpointSpi.S3.clientConfiguration.retryPolicy);
+        cfg.arrayProperty('checkpointSpi', 'checkpointSpi', cfgs, 'org.apache.ignite.spi.checkpoint.CheckpointSpi');
 
-                                    retryBean.constantConstructorArgument('retryCondition')
-                                        .constantConstructorArgument('backoffStrategy')
-                                        .constructorArgument('java.lang.Integer', retryBean.valueOf('maxErrorRetry'))
-                                        .constructorArgument('java.lang.Boolean', retryBean.valueOf('honorMaxErrorRetryInClientConfig'));
+        return cfg;
+    }
 
-                                    break;
+    // Generate collision group.
+    static clusterCollision(collision, cfg = this.igniteConfigurationBean()) {
+        let colSpi;
+
+        switch (_.get(collision, 'kind')) {
+            case 'JobStealing':
+                colSpi = new Bean('org.apache.ignite.spi.collision.jobstealing.JobStealingCollisionSpi',
+                    'colSpi', collision.JobStealing, clusterDflts.collision.JobStealing);
+
+                colSpi.intProperty('activeJobsThreshold')
+                    .intProperty('waitJobsThreshold')
+                    .intProperty('messageExpireTime')
+                    .intProperty('maximumStealingAttempts')
+                    .boolProperty('stealingEnabled')
+                    .emptyBeanProperty('externalCollisionListener')
+                    .mapProperty('stealingAttrs', 'stealingAttributes');
+
+                break;
+            case 'FifoQueue':
+                colSpi = new Bean('org.apache.ignite.spi.collision.fifoqueue.FifoQueueCollisionSpi',
+                    'colSpi', collision.FifoQueue, clusterDflts.collision.FifoQueue);
+
+                colSpi.intProperty('parallelJobsNumber')
+                    .intProperty('waitingJobsNumber');
+
+                break;
+            case 'PriorityQueue':
+                colSpi = new Bean('org.apache.ignite.spi.collision.priorityqueue.PriorityQueueCollisionSpi',
+                    'colSpi', collision.PriorityQueue, clusterDflts.collision.PriorityQueue);
+
+                colSpi.intProperty('parallelJobsNumber')
+                    .intProperty('waitingJobsNumber')
+                    .intProperty('priorityAttributeKey')
+                    .intProperty('jobPriorityAttributeKey')
+                    .intProperty('defaultPriority')
+                    .intProperty('starvationIncrement')
+                    .boolProperty('starvationPreventionEnabled');
+
+                break;
+            case 'Custom':
+                if (_.nonNil(_.get(collision, 'Custom.class')))
+                    colSpi = new EmptyBean(collision.Custom.class);
+
+                break;
+            default:
+                return cfg;
+        }
 
-                                case 'DynamoDB':
-                                    retryBean = new Bean('com.amazonaws.retry.RetryPolicy', 'retryPolicy', {
-                                        retryCondition: 'DEFAULT_RETRY_CONDITION',
-                                        backoffStrategy: 'DYNAMODB_DEFAULT_BACKOFF_STRATEGY',
-                                        maxErrorRetry: 'DYNAMODB_DEFAULT_MAX_ERROR_RETRY',
-                                        honorMaxErrorRetryInClientConfig: true
-                                    }, clusterDflts.checkpointSpi.S3.clientConfiguration.retryPolicy);
+        if (_.nonNil(colSpi))
+            cfg.beanProperty('collisionSpi', colSpi);
 
-                                    retryBean.constantConstructorArgument('retryCondition')
-                                        .constantConstructorArgument('backoffStrategy')
-                                        .constantConstructorArgument('maxErrorRetry')
-                                        .constructorArgument('java.lang.Boolean', retryBean.valueOf('honorMaxErrorRetryInClientConfig'));
+        return cfg;
+    }
 
-                                    break;
+    // Generate communication group.
+    static clusterCommunication(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        const commSpi = new Bean('org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi', 'communicationSpi',
+            cluster.communication, clusterDflts.communication);
+
+        commSpi.emptyBeanProperty('listener')
+            .stringProperty('localAddress')
+            .intProperty('localPort')
+            .intProperty('localPortRange')
+            .intProperty('sharedMemoryPort')
+            .intProperty('directBuffer')
+            .intProperty('directSendBuffer')
+            .intProperty('idleConnectionTimeout')
+            .intProperty('connectTimeout')
+            .intProperty('maxConnectTimeout')
+            .intProperty('reconnectCount')
+            .intProperty('socketSendBuffer')
+            .intProperty('socketReceiveBuffer')
+            .intProperty('messageQueueLimit')
+            .intProperty('slowClientQueueLimit')
+            .intProperty('tcpNoDelay')
+            .intProperty('ackSendThreshold')
+            .intProperty('unacknowledgedMessagesBufferSize')
+            .intProperty('socketWriteTimeout')
+            .intProperty('selectorsCount')
+            .emptyBeanProperty('addressResolver');
+
+        if (commSpi.nonEmpty())
+            cfg.beanProperty('communicationSpi', commSpi);
+
+        cfg.intProperty('networkTimeout')
+            .intProperty('networkSendRetryDelay')
+            .intProperty('networkSendRetryCount')
+            .intProperty('discoveryStartupDelay');
+
+        return cfg;
+    }
 
-                                case 'DynamoDBMaxRetries':
-                                    retryBean = new Bean('com.amazonaws.retry.RetryPolicy', 'retryPolicy', {
-                                        retryCondition: 'DEFAULT_RETRY_CONDITION',
-                                        backoffStrategy: 'DYNAMODB_DEFAULT_BACKOFF_STRATEGY',
-                                        maxErrorRetry: _.get(policy, 'maxErrorRetry') || -1,
-                                        honorMaxErrorRetryInClientConfig: false
-                                    }, clusterDflts.checkpointSpi.S3.clientConfiguration.retryPolicy);
+    // Generate REST access configuration.
+    static clusterConnector(connector, cfg = this.igniteConfigurationBean()) {
+        const connCfg = new Bean('org.apache.ignite.configuration.ConnectorConfiguration',
+            'connectorConfiguration', connector, clusterDflts.connector);
+
+        if (connCfg.valueOf('enabled')) {
+            connCfg.pathProperty('jettyPath')
+                .stringProperty('host')
+                .intProperty('port')
+                .intProperty('portRange')
+                .intProperty('idleTimeout')
+                .intProperty('idleQueryCursorTimeout')
+                .intProperty('idleQueryCursorCheckFrequency')
+                .intProperty('receiveBufferSize')
+                .intProperty('sendBufferSize')
+                .intProperty('sendQueueLimit')
+                .intProperty('directBuffer')
+                .intProperty('noDelay')
+                .intProperty('selectorCount')
+                .intProperty('threadPoolSize')
+                .emptyBeanProperty('messageInterceptor')
+                .stringProperty('secretKey');
+
+            if (connCfg.valueOf('sslEnabled')) {
+                connCfg.intProperty('sslClientAuth')
+                    .emptyBeanProperty('sslFactory');
+            }
 
-                                    retryBean.constantConstructorArgument('retryCondition')
-                                        .constantConstructorArgument('backoffStrategy')
-                                        .constructorArgument('java.lang.Integer', retryBean.valueOf('maxErrorRetry'))
-                                        .constructorArgument('java.lang.Boolean', retryBean.valueOf('honorMaxErrorRetryInClientConfig'));
+            if (connCfg.nonEmpty())
+                cfg.beanProperty('connectorConfiguration', connCfg);
+        }
 
-                                    break;
+        return cfg;
+    }
 
-                                case 'Custom':
-                                    retryBean = new Bean('com.amazonaws.retry.RetryPolicy', 'retryPolicy', policy);
+    // Generate deployment group.
+    static clusterDeployment(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        cfg.enumProperty('deploymentMode')
+            .boolProperty('peerClassLoadingEnabled');
 
-                                    retryBean.beanConstructorArgument('retryCondition', retryBean.valueOf('retryCondition') ? new EmptyBean(retryBean.valueOf('retryCondition')) : null)
-                                        .beanConstructorArgument('backoffStrategy', retryBean.valueOf('backoffStrategy') ? new EmptyBean(retryBean.valueOf('backoffStrategy')) : null)
-                                        .constructorArgument('java.lang.Integer', retryBean.valueOf('maxErrorRetry'))
-                                        .constructorArgument('java.lang.Boolean', retryBean.valueOf('honorMaxErrorRetryInClientConfig'));
+        if (cfg.valueOf('peerClassLoadingEnabled')) {
+            cfg.intProperty('peerClassLoadingMissedResourcesCacheSize')
+                .intProperty('peerClassLoadingThreadPoolSize')
+                .varArgProperty('p2pLocClsPathExcl', 'peerClassLoadingLocalClassPathExclude',
+                   cluster.peerClassLoadingLocalClassPathExclude);
+        }
 
-                                    break;
+        let deploymentBean = null;
 
-                                default:
-                                    break;
-                            }
+        switch (_.get(cluster, 'deploymentSpi.kind')) {
+            case 'URI':
+                const uriDeployment = cluster.deploymentSpi.URI;
 
-                            if (retryBean)
-                                clientBean.beanProperty('retryPolicy', retryBean);
-                        }
+                deploymentBean = new Bean('org.apache.ignite.spi.deployment.uri.UriDeploymentSpi', 'deploymentSpi', uriDeployment);
 
-                        clientBean.intProperty('maxErrorRetry')
-                            .intProperty('socketTimeout')
-                            .intProperty('connectionTimeout')
-                            .intProperty('requestTimeout')
-                            .intProperty('socketSendBufferSizeHints')
-                            .stringProperty('signerOverride')
-                            .intProperty('connectionTTL')
-                            .intProperty('connectionMaxIdleMillis')
-                            .emptyBeanProperty('dnsResolver')
-                            .intProperty('responseMetadataCacheSize')
-                            .emptyBeanProperty('secureRandom')
-                            .boolProperty('useReaper')
-                            .boolProperty('useGzip')
-                            .boolProperty('preemptiveBasicProxyAuth')
-                            .boolProperty('useTcpKeepAlive');
-
-                        if (clientBean.nonEmpty())
-                            s3Bean.beanProperty('clientConfiguration', clientBean);
-
-                        s3Bean.emptyBeanProperty('checkpointListener');
-
-                        return s3Bean;
-
-                    case 'JDBC':
-                        const jdbcBean = new Bean('org.apache.ignite.spi.checkpoint.jdbc.JdbcCheckpointSpi',
-                            'checkpointSpiJdbc', spi.JDBC, clusterDflts.checkpointSpi.JDBC);
-
-                        const id = jdbcBean.valueOf('dataSourceBean');
-                        const dialect = _.get(spi.JDBC, 'dialect');
-
-                        jdbcBean.dataSource(id, 'dataSource', this.dataSourceBean(id, dialect));
-
-                        if (!_.isEmpty(jdbcBean.valueOf('user'))) {
-                            jdbcBean.stringProperty('user')
-                                .property('pwd', `checkpoint.${jdbcBean.valueOf('dataSourceBean')}.${jdbcBean.valueOf('user')}.jdbc.password`, 'YOUR_PASSWORD');
-                        }
+                const scanners = _.map(uriDeployment.scanners, (scanner) => new EmptyBean(scanner));
 
-                        jdbcBean.stringProperty('checkpointTableName')
-                            .stringProperty('keyFieldName')
-                            .stringProperty('keyFieldType')
-                            .stringProperty('valueFieldName')
-                            .stringProperty('valueFieldType')
-                            .stringProperty('expireDateFieldName')
-                            .stringProperty('expireDateFieldType')
-                            .intProperty('numberOfRetries')
-                            .emptyBeanProperty('checkpointListener');
+                deploymentBean.collectionProperty('uriList', 'uriList', uriDeployment.uriList)
+                    .stringProperty('temporaryDirectoryPath')
+                    .varArgProperty('scanners', 'scanners', scanners,
+                        'org.apache.ignite.spi.deployment.uri.scanners.UriDeploymentScanner')
+                    .emptyBeanProperty('listener')
+                    .boolProperty('checkMd5')
+                    .boolProperty('encodeUri');
 
-                        return jdbcBean;
+                cfg.beanProperty('deploymentSpi', deploymentBean);
 
-                    case 'Custom':
-                        const clsName = _.get(spi, 'Custom.className');
+                break;
 
-                        if (clsName)
-                            return new Bean(clsName, 'checkpointSpiCustom', spi.Cache);
+            case 'Local':
+                deploymentBean = new Bean('org.apache.ignite.spi.deployment.local.LocalDeploymentSpi', 'deploymentSpi', cluster.deploymentSpi.Local);
 
-                        return null;
+                deploymentBean.emptyBeanProperty('listener');
 
-                    default:
-                        return null;
-                }
-            }), (checkpointBean) => _.nonNil(checkpointBean));
+                cfg.beanProperty('deploymentSpi', deploymentBean);
 
-            cfg.arrayProperty('checkpointSpi', 'checkpointSpi', cfgs, 'org.apache.ignite.spi.checkpoint.CheckpointSpi');
+                break;
 
-            return cfg;
-        }
+            case 'Custom':
+                cfg.emptyBeanProperty('deploymentSpi.Custom.className');
 
-        // Generate collision group.
-        static clusterCollision(collision, cfg = this.igniteConfigurationBean()) {
-            let colSpi;
+                break;
 
-            switch (_.get(collision, 'kind')) {
-                case 'JobStealing':
-                    colSpi = new Bean('org.apache.ignite.spi.collision.jobstealing.JobStealingCollisionSpi',
-                        'colSpi', collision.JobStealing, clusterDflts.collision.JobStealing);
+            default:
+                // No-op.
+        }
 
-                    colSpi.intProperty('activeJobsThreshold')
-                        .intProperty('waitJobsThreshold')
-                        .intProperty('messageExpireTime')
-                        .intProperty('maximumStealingAttempts')
-                        .boolProperty('stealingEnabled')
-                        .emptyBeanProperty('externalCollisionListener')
-                        .mapProperty('stealingAttrs', 'stealingAttributes');
+        return cfg;
+    }
 
-                    break;
-                case 'FifoQueue':
-                    colSpi = new Bean('org.apache.ignite.spi.collision.fifoqueue.FifoQueueCollisionSpi',
-                        'colSpi', collision.FifoQueue, clusterDflts.collision.FifoQueue);
+    // Generate discovery group.
+    static clusterDiscovery(discovery, cfg = this.igniteConfigurationBean(), discoSpi = this.discoveryConfigurationBean(discovery)) {
+        discoSpi.stringProperty('localAddress')
+            .intProperty('localPort')
+            .intProperty('localPortRange')
+            .emptyBeanProperty('addressResolver')
+            .intProperty('socketTimeout')
+            .intProperty('ackTimeout')
+            .intProperty('maxAckTimeout')
+            .intProperty('networkTimeout')
+            .intProperty('joinTimeout')
+            .intProperty('threadPriority')
+            .intProperty('heartbeatFrequency')
+            .intProperty('maxMissedHeartbeats')
+            .intProperty('maxMissedClientHeartbeats')
+            .intProperty('topHistorySize')
+            .emptyBeanProperty('listener')
+            .emptyBeanProperty('dataExchange')
+            .emptyBeanProperty('metricsProvider')
+            .intProperty('reconnectCount')
+            .intProperty('statisticsPrintFrequency')
+            .intProperty('ipFinderCleanFrequency')
+            .emptyBeanProperty('authenticator')
+            .intProperty('forceServerMode')
+            .intProperty('clientReconnectDisabled');
+
+        if (discoSpi.nonEmpty())
+            cfg.beanProperty('discoverySpi', discoSpi);
+
+        return discoSpi;
+    }
 
-                    colSpi.intProperty('parallelJobsNumber')
-                        .intProperty('waitingJobsNumber');
+    // Generate events group.
+    static clusterEvents(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        const eventStorage = cluster.eventStorage;
 
-                    break;
-                case 'PriorityQueue':
-                    colSpi = new Bean('org.apache.ignite.spi.collision.priorityqueue.PriorityQueueCollisionSpi',
-                        'colSpi', collision.PriorityQueue, clusterDflts.collision.PriorityQueue);
-
-                    colSpi.intProperty('parallelJobsNumber')
-                        .intProperty('waitingJobsNumber')
-                        .intProperty('priorityAttributeKey')
-                        .intProperty('jobPriorityAttributeKey')
-                        .intProperty('defaultPriority')
-                        .intProperty('starvationIncrement')
-                        .boolProperty('starvationPreventionEnabled');
+        let eventStorageBean = null;
 
-                    break;
-                case 'Custom':
-                    if (_.nonNil(_.get(collision, 'Custom.class')))
-                        colSpi = new EmptyBean(collision.Custom.class);
+        switch (_.get(eventStorage, 'kind')) {
+            case 'Memory':
+                eventStorageBean = new Bean('org.apache.ignite.spi.eventstorage.memory.MemoryEventStorageSpi', 'eventStorage', eventStorage.Memory, clusterDflts.eventStorage.Memory);
 
-                    break;
-                default:
-                    return cfg;
-            }
+                eventStorageBean.intProperty('expireAgeMs')
+                    .intProperty('expireCount')
+                    .emptyBeanProperty('filter');
 
-            if (_.nonNil(colSpi))
-                cfg.beanProperty('collisionSpi', colSpi);
+                break;
 
-            return cfg;
-        }
+            case 'Custom':
+                const className = _.get(eventStorage, 'Custom.className');
 
-        // Generate communication group.
-        static clusterCommunication(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            const commSpi = new Bean('org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi', 'communicationSpi',
-                cluster.communication, clusterDflts.communication);
+                if (className)
+                    eventStorageBean = new EmptyBean(className);
 
-            commSpi.emptyBeanProperty('listener')
-                .stringProperty('localAddress')
-                .intProperty('localPort')
-                .intProperty('localPortRange')
-                .intProperty('sharedMemoryPort')
-                .intProperty('directBuffer')
-                .intProperty('directSendBuffer')
-                .intProperty('idleConnectionTimeout')
-                .intProperty('connectTimeout')
-                .intProperty('maxConnectTimeout')
-                .intProperty('reconnectCount')
-                .intProperty('socketSendBuffer')
-                .intProperty('socketReceiveBuffer')
-                .intProperty('messageQueueLimit')
-                .intProperty('slowClientQueueLimit')
-                .intProperty('tcpNoDelay')
-                .intProperty('ackSendThreshold')
-                .intProperty('unacknowledgedMessagesBufferSize')
-                .intProperty('socketWriteTimeout')
-                .intProperty('selectorsCount')
-                .emptyBeanProperty('addressResolver');
-
-            if (commSpi.nonEmpty())
-                cfg.beanProperty('communicationSpi', commSpi);
-
-            cfg.intProperty('networkTimeout')
-                .intProperty('networkSendRetryDelay')
-                .intProperty('networkSendRetryCount')
-                .intProperty('discoveryStartupDelay');
+                break;
 
-            return cfg;
+            default:
+                // No-op.
         }
 
-        // Generate REST access configuration.
-        static clusterConnector(connector, cfg = this.igniteConfigurationBean()) {
-            const connCfg = new Bean('org.apache.ignite.configuration.ConnectorConfiguration',
-                'connectorConfiguration', connector, clusterDflts.connector);
-
-            if (connCfg.valueOf('enabled')) {
-                connCfg.pathProperty('jettyPath')
-                    .stringProperty('host')
-                    .intProperty('port')
-                    .intProperty('portRange')
-                    .intProperty('idleTimeout')
-                    .intProperty('idleQueryCursorTimeout')
-                    .intProperty('idleQueryCursorCheckFrequency')
-                    .intProperty('receiveBufferSize')
-                    .intProperty('sendBufferSize')
-                    .intProperty('sendQueueLimit')
-                    .intProperty('directBuffer')
-                    .intProperty('noDelay')
-                    .intProperty('selectorCount')
-                    .intProperty('threadPoolSize')
-                    .emptyBeanProperty('messageInterceptor')
-                    .stringProperty('secretKey');
-
-                if (connCfg.valueOf('sslEnabled')) {
-                    connCfg.intProperty('sslClientAuth')
-                        .emptyBeanProperty('sslFactory');
-                }
-
-                if (connCfg.nonEmpty())
-                    cfg.beanProperty('connectorConfiguration', connCfg);
-            }
+        if (eventStorageBean && eventStorageBean.nonEmpty())
+            cfg.beanProperty('eventStorageSpi', eventStorageBean);
 
-            return cfg;
-        }
+        if (_.nonEmpty(cluster.includeEventTypes))
+            cfg.eventTypes('evts', 'includeEventTypes', cluster.includeEventTypes);
 
-        // Generate deployment group.
-        static clusterDeployment(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            cfg.enumProperty('deploymentMode')
-                .boolProperty('peerClassLoadingEnabled');
+        return cfg;
+    }
 
-            if (cfg.valueOf('peerClassLoadingEnabled')) {
-                cfg.intProperty('peerClassLoadingMissedResourcesCacheSize')
-                    .intProperty('peerClassLoadingThreadPoolSize')
-                    .varArgProperty('p2pLocClsPathExcl', 'peerClassLoadingLocalClassPathExclude',
-                       cluster.peerClassLoadingLocalClassPathExclude);
-            }
+    // Generate failover group.
+    static clusterFailover(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        const spis = [];
 
-            return cfg;
-        }
+        _.forEach(cluster.failoverSpi, (spi) => {
+            let failoverSpi;
 
-        // Generate discovery group.
-        static clusterDiscovery(discovery, cfg = this.igniteConfigurationBean(), discoSpi = this.discoveryConfigurationBean(discovery)) {
-            discoSpi.stringProperty('localAddress')
-                .intProperty('localPort')
-                .intProperty('localPortRange')
-                .emptyBeanProperty('addressResolver')
-                .intProperty('socketTimeout')
-                .intProperty('ackTimeout')
-                .intProperty('maxAckTimeout')
-                .intProperty('networkTimeout')
-                .intProperty('joinTimeout')
-                .intProperty('threadPriority')
-                .intProperty('heartbeatFrequency')
-                .intProperty('maxMissedHeartbeats')
-                .intProperty('maxMissedClientHeartbeats')
-                .intProperty('topHistorySize')
-                .emptyBeanProperty('listener')
-                .emptyBeanProperty('dataExchange')
-                .emptyBeanProperty('metricsProvider')
-                .intProperty('reconnectCount')
-                .intProperty('statisticsPrintFrequency')
-                .intProperty('ipFinderCleanFrequency')
-                .emptyBeanProperty('authenticator')
-                .intProperty('forceServerMode')
-                .intProperty('clientReconnectDisabled');
-
-            if (discoSpi.nonEmpty())
-                cfg.beanProperty('discoverySpi', discoSpi);
-
-            return discoSpi;
-        }
+            switch (_.get(spi, 'kind')) {
+                case 'JobStealing':
+                    failoverSpi = new Bean('org.apache.ignite.spi.failover.jobstealing.JobStealingFailoverSpi',
+                        'failoverSpi', spi.JobStealing, clusterDflts.failoverSpi.JobStealing);
 
-        // Generate events group.
-        static clusterEvents(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            const eventStorage = cluster.eventStorage;
+                    failoverSpi.intProperty('maximumFailoverAttempts');
 
-            let eventStorageBean = null;
+                    break;
+                case 'Never':
+                    failoverSpi = new Bean('org.apache.ignite.spi.failover.never.NeverFailoverSpi',
+                        'failoverSpi', spi.Never);
 
-            switch (_.get(eventStorage, 'kind')) {
-                case 'Memory':
-                    eventStorageBean = new Bean('org.apache.ignite.spi.eventstorage.memory.MemoryEventStorageSpi', 'eventStorage', eventStorage.Memory, clusterDflts.eventStorage.Memory);
+                    break;
+                case 'Always':
+                    failoverSpi = new Bean('org.apache.ignite.spi.failover.always.AlwaysFailoverSpi',
+                        'failoverSpi', spi.Always, clusterDflts.failoverSpi.Always);
 
-                    eventStorageBean.intProperty('expireAgeMs')
-                        .intProperty('expireCount')
-                        .emptyBeanProperty('filter');
+                    failoverSpi.intProperty('maximumFailoverAttempts');
 
                     break;
-
                 case 'Custom':
-                    const className = _.get(eventStorage, 'Custom.className');
+                    const className = _.get(spi, 'Custom.class');
 
                     if (className)
-                        eventStorageBean = new EmptyBean(className);
+                        failoverSpi = new EmptyBean(className);
 
                     break;
-
                 default:
                     // No-op.
             }
 
-            if (eventStorageBean && eventStorageBean.nonEmpty())
-                cfg.beanProperty('eventStorageSpi', eventStorageBean);
-
-            if (_.nonEmpty(cluster.includeEventTypes))
-                cfg.eventTypes('evts', 'includeEventTypes', cluster.includeEventTypes);
-
-            return cfg;
-        }
-
-        // Generate failover group.
-        static clusterFailover(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            const spis = [];
-
-            _.forEach(cluster.failoverSpi, (spi) => {
-                let failoverSpi;
+            if (failoverSpi)
+                spis.push(failoverSpi);
+        });
 
-                switch (_.get(spi, 'kind')) {
-                    case 'JobStealing':
-                        failoverSpi = new Bean('org.apache.ignite.spi.failover.jobstealing.JobStealingFailoverSpi',
-                            'failoverSpi', spi.JobStealing, clusterDflts.failoverSpi.JobStealing);
+        if (spis.length)
+            cfg.arrayProperty('failoverSpi', 'failoverSpi', spis, 'org.apache.ignite.spi.failover.FailoverSpi');
 
-                        failoverSpi.intProperty('maximumFailoverAttempts');
-
-                        break;
-                    case 'Never':
-                        failoverSpi = new Bean('org.apache.ignite.spi.failover.never.NeverFailoverSpi',
-                            'failoverSpi', spi.Never);
-
-                        break;
-                    case 'Always':
-                        failoverSpi = new Bean('org.apache.ignite.spi.failover.always.AlwaysFailoverSpi',
-                            'failoverSpi', spi.Always, clusterDflts.failoverSpi.Always);
+        return cfg;
+    }
 
-                        failoverSpi.intProperty('maximumFailoverAttempts');
+    // Generate load balancing configuration group.
+    static clusterLoadBalancing(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        const spis = [];
 
-                        break;
-                    case 'Custom':
-                        const className = _.get(spi, 'Custom.class');
+        _.forEach(cluster.loadBalancingSpi, (spi) => {
+            let loadBalancingSpi;
 
-                        if (className)
-                            failoverSpi = new EmptyBean(className);
+            switch (_.get(spi, 'kind')) {
+                case 'RoundRobin':
+                    loadBalancingSpi = new Bean('org.apache.ignite.spi.loadbalancing.roundrobin.RoundRobinLoadBalancingSpi', 'loadBalancingSpiRR', spi.RoundRobin, clusterDflts.loadBalancingSpi.RoundRobin);
 
-                        break;
-                    default:
-                        // No-op.
-                }
+                    loadBalancingSpi.boolProperty('perTask');
 
-                if (failoverSpi)
-                    spis.push(failoverSpi);
-            });
+                    break;
+                case 'Adaptive':
+                    loadBalancingSpi = new Bean('org.apache.ignite.spi.loadbalancing.adaptive.AdaptiveLoadBalancingSpi', 'loadBalancingSpiAdaptive', spi.Adaptive);
 
-            if (spis.length)
-                cfg.arrayProperty('failoverSpi', 'failoverSpi', spis, 'org.apache.ignite.spi.failover.FailoverSpi');
+                    let probeBean;
 
-            return cfg;
-        }
+                    switch (_.get(spi, 'Adaptive.loadProbe.kind')) {
+                        case 'Job':
+                            probeBean = new Bean('org.apache.ignite.spi.loadbalancing.adaptive.AdaptiveJobCountLoadProbe', 'jobProbe', spi.Adaptive.loadProbe.Job, clusterDflts.loadBalancingSpi.Adaptive.loadProbe.Job);
 
-        // Generate load balancing configuration group.
-        static clusterLoadBalancing(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            const spis = [];
+                            probeBean.boolProperty('useAverage');
 
-            _.forEach(cluster.loadBalancingSpi, (spi) => {
-                let loadBalancingSpi;
+                            break;
+                        case 'CPU':
+                            probeBean = new Bean('org.apache.ignite.spi.loadbalancing.adaptive.AdaptiveCpuLoadProbe', 'cpuProbe', spi.Adaptive.loadProbe.CPU, clusterDflts.loadBalancingSpi.Adaptive.loadProbe.CPU);
 
-                switch (_.get(spi, 'kind')) {
-                    case 'RoundRobin':
-                        loadBalancingSpi = new Bean('org.apache.ignite.spi.loadbalancing.roundrobin.RoundRobinLoadBalancingSpi', 'loadBalancingSpiRR', spi.RoundRobin, clusterDflts.loadBalancingSpi.RoundRobin);
+                            probeBean.boolProperty('useAverage')
+                                .boolProperty('useProcessors')
+                                .intProperty('processorCoefficient');
 
-                        loadBalancingSpi.boolProperty('perTask');
+                            break;
+                        case 'ProcessingTime':
+                            probeBean = new Bean('org.apache.ignite.spi.loadbalancing.adaptive.AdaptiveProcessingTimeLoadProbe', 'timeProbe', spi.Adaptive.loadProbe.ProcessingTime, clusterDflts.loadBalancingSpi.Adaptive.loadProbe.ProcessingTime);
 
-                        break;
-                    case 'Adaptive':
-                        loadBalancingSpi = new Bean('org.apache.ignite.spi.loadbalancing.adaptive.AdaptiveLoadBalancingSpi', 'loadBalancingSpiAdaptive', spi.Adaptive);
+                            probeBean.boolProperty('useAverage');
 
-                        let probeBean;
+                            break;
+                        case 'Custom':
+                            const className = _.get(spi, 'Adaptive.loadProbe.Custom.className');
 
-                        switch (_.get(spi, 'Adaptive.loadProbe.kind')) {
-                            case 'Job':
-                                probeBean = new Bean('org.apache.ignite.spi.loadbalancing.adaptive.AdaptiveJobCountLoadProbe', 'jobProbe', spi.Adaptive.loadProbe.Job, clusterDflts.loadBalancingSpi.Adaptive.loadProbe.Job);
+                            if (className)
+                                probeBean = new Bean(className, 'probe', spi.Adaptive.loadProbe.Job.Custom);
 
-                                probeBean.boolProperty('useAverage');
+                            break;
+                        default:
+                            // No-op.
+                    }
 
-                                break;
-                            case 'CPU':
-                                probeBean = new Bean('org.apache.ignite.spi.loadbalancing.adaptive.AdaptiveCpuLoadProbe', 'cpuProbe', spi.Adaptive.loadProbe.CPU, clusterDflts.loadBalancingSpi.Adaptive.loadProbe.CPU);
+                    if (probeBean)
+                        loadBalancingSpi.beanProperty('loadProbe', probeBean);
 
-                                probeBean.boolProperty('useAverage')
-                                    .boolProperty('useProcessors')
-                                    .intProperty('processorCoefficient');
+                    break;
+                case 'WeightedRandom':
+                    loadBalancingSpi = new Bean('org.apache.ignite.spi.loadbalancing.weightedrandom.WeightedRandomLoadBalancingSpi', 'loadBalancingSpiRandom', spi.WeightedRandom, clusterDflts.loadBalancingSpi.WeightedRandom);
 
-                                break;
-                            case 'ProcessingTime':
-                                probeBean = new Bean('org.apache.ignite.spi.loadbalancing.adaptive.AdaptiveProcessingTimeLoadProbe', 'timeProbe', spi.Adaptive.loadProbe.ProcessingTime, clusterDflts.loadBalancingSpi.Adaptive.loadProbe.ProcessingTime);
+                    loadBalancingSpi.intProperty('nodeWeight')
+                        .boolProperty('useWeights');
 
-                                probeBean.boolProperty('useAverage');
+                    break;
+                case 'Custom':
+                    const className = _.get(spi, 'Custom.className');
 
-                                break;
-                            case 'Custom':
-                                const className = _.get(spi, 'Adaptive.loadProbe.Custom.className');
+                    if (className)
+                        loadBalancingSpi = new Bean(className, 'loadBalancingSpiCustom', spi.Custom);
 
-                                if (className)
-                                    probeBean = new Bean(className, 'probe', spi.Adaptive.loadProbe.Job.Custom);
+                    break;
+                default:
+                    // No-op.
+            }
 
-                                break;
-                            default:
-                                // No-op.
-                        }
+            if (loadBalancingSpi)
+                spis.push(loadBalancingSpi);
+        });
 
-                        if (probeBean)
-                            loadBalancingSpi.beanProperty('loadProbe', probeBean);
+        if (spis.length)
+            cfg.varArgProperty('loadBalancingSpi', 'loadBalancingSpi', spis, 'org.apache.ignite.spi.loadbalancing.LoadBalancingSpi');
 
-                        break;
-                    case 'WeightedRandom':
-                        loadBalancingSpi = new Bean('org.apache.ignite.spi.loadbalancing.weightedrandom.WeightedRandomLoadBalancingSpi', 'loadBalancingSpiRandom', spi.WeightedRandom, clusterDflts.loadBalancingSpi.WeightedRandom);
+        return cfg;
+    }
 
-                        loadBalancingSpi.intProperty('nodeWeight')
-                            .boolProperty('useWeights');
+    // Generate logger group.
+    static clusterLogger(logger, cfg = this.igniteConfigurationBean()) {
+        let loggerBean;
 
-                        break;
-                    case 'Custom':
-                        const className = _.get(spi, 'Custom.className');
+        switch (_.get(logger, 'kind')) {
+            case 'Log4j':
+                if (logger.Log4j && (logger.Log4j.mode === 'Default' || logger.Log4j.mode === 'Path' && _.nonEmpty(logger.Log4j.path))) {
+                    loggerBean = new Bean('org.apache.ignite.logger.log4j.Log4JLogger',
+                        'logger', logger.Log4j, clusterDflts.logger.Log4j);
 
-                        if (className)
-                            loadBalancingSpi = new Bean(className, 'loadBalancingSpiCustom', spi.Custom);
+                    if (loggerBean.valueOf('mode') === 'Path')
+                        loggerBean.pathConstructorArgument('path');
 
-                        break;
-                    default:
-                        // No-op.
+                    loggerBean.enumProperty('level');
                 }
 
-                if (loadBalancingSpi)
-                    spis.push(loadBalancingSpi);
-            });
-
-            if (spis.length)
-                cfg.varArgProperty('loadBalancingSpi', 'loadBalancingSpi', spis, 'org.apache.ignite.spi.loadbalancing.LoadBalancingSpi');
+                break;
+            case 'Log4j2':
+                if (logger.Log4j2 && _.nonEmpty(logger.Log4j2.path)) {
+                    loggerBean = new Bean('org.apache.ignite.logger.log4j2.Log4J2Logger',
+                        'logger', logger.Log4j2, clusterDflts.logger.Log4j2);
 
-            return cfg;
-        }
-
-        // Generate logger group.
-        static clusterLogger(logger, cfg = this.igniteConfigurationBean()) {
-            let loggerBean;
-
-            switch (_.get(logger, 'kind')) {
-                case 'Log4j':
-                    if (logger.Log4j && (logger.Log4j.mode === 'Default' || logger.Log4j.mode === 'Path' && _.nonEmpty(logger.Log4j.path))) {
-                        loggerBean = new Bean('org.apache.ignite.logger.log4j.Log4JLogger',
-                            'logger', logger.Log4j, clusterDflts.logger.Log4j);
-
-                        if (loggerBean.valueOf('mode') === 'Path')
-                            loggerBean.pathConstructorArgument('path');
-
-                        loggerBean.enumProperty('level');
-                    }
-
-                    break;
-                case 'Log4j2':
-                    if (logger.Log4j2 && _.nonEmpty(logger.Log4j2.path)) {
-                        loggerBean = new Bean('org.apache.ignite.logger.log4j2.Log4J2Logger',
-                            'logger', logger.Log4j2, clusterDflts.logger.Log4j2);
-
-                        loggerBean.pathConstructorArgument('path')
-                            .enumProperty('level');
-                    }
-
-                    break;
-                case 'Null':
-                    loggerBean = new EmptyBean('org.apache.ignite.logger.NullLogger');
-
-                    break;
-                case 'Java':
-                    loggerBean = new EmptyBean('org.apache.ignite.logger.java.JavaLogger');
+                    loggerBean.pathConstructorArgument('path')
+                        .enumProperty('level');
+                }
 
-                    break;
-                case 'JCL':
-                    loggerBean = new EmptyBean('org.apache.ignite.logger.jcl.JclLogger');
+                break;
+            case 'Null':
+                loggerBean = new EmptyBean('org.apache.ignite.logger.NullLogger');
 
-                    break;
-                case 'SLF4J':
-                    loggerBean = new EmptyBean('org.apache.ignite.logger.slf4j.Slf4jLogger');
+                break;
+            case 'Java':
+                loggerBean = new EmptyBean('org.apache.ignite.logger.java.JavaLogger');
 
-                    break;
-                case 'Custom':
-                    if (logger.Custom && _.nonEmpty(logger.Custom.class))
-                        loggerBean = new EmptyBean(logger.Custom.class);
+                break;
+            case 'JCL':
+                loggerBean = new EmptyBean('org.apache.ignite.logger.jcl.JclLogger');
 
-                    break;
-                default:
-                    return cfg;
-            }
+                break;
+            case 'SLF4J':
+                loggerBean = new EmptyBean('org.apache.ignite.logger.slf4j.Slf4jLogger');
 
-            if (loggerBean)
-                cfg.beanProperty('gridLogger', loggerBean);
+                break;
+            case 'Custom':
+                if (logger.Custom && _.nonEmpty(logger.Custom.class))
+                    loggerBean = new EmptyBean(logger.Custom.class);
 
-            return cfg;
+                break;
+            default:
+                return cfg;
         }
 
-        // Generate IGFSs configs.
-        static clusterIgfss(igfss, cfg = this.igniteConfigurationBean()) {
-            const igfsCfgs = _.map(igfss, (igfs) => {
-                const igfsCfg = this.igfsGeneral(igfs);
+        if (loggerBean)
+            cfg.beanProperty('gridLogger', loggerBean);
 
-                this.igfsIPC(igfs, igfsCfg);
-                this.igfsFragmentizer(igfs, igfsCfg);
-                this.igfsDualMode(igfs, igfsCfg);
-                this.igfsSecondFS(igfs, igfsCfg);
-                this.igfsMisc(igfs, igfsCfg);
-
-                return igfsCfg;
-            });
-
-            cfg.varArgProperty('igfsCfgs', 'fileSystemConfiguration', igfsCfgs, 'org.apache.ignite.configuration.FileSystemConfiguration');
+        return cfg;
+    }
 
-            return cfg;
-        }
+    // Generate IGFSs configs.
+    static clusterIgfss(igfss, cfg = this.igniteConfigurationBean()) {
+        const igfsCfgs = _.map(igfss, (igfs) => {
+            const igfsCfg = this.igfsGeneral(igfs);
 
-        // Generate marshaller group.
-        static clusterMarshaller(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            const kind = _.get(cluster.marshaller, 'kind');
-            const settings = _.get(cluster.marshaller, kind);
+            this.igfsIPC(igfs, igfsCfg);
+            this.igfsFragmentizer(igfs, igfsCfg);
+            this.igfsDualMode(igfs, igfsCfg);
+            this.igfsSecondFS(igfs, igfsCfg);
+            this.igfsMisc(igfs, igfsCfg);
 
-            if (_.isNil(settings))
-                return cfg;
+            return igfsCfg;
+        });
 
-            let bean;
+        cfg.varArgProperty('igfsCfgs', 'fileSystemConfiguration', igfsCfgs, 'org.apache.ignite.configuration.FileSystemConfiguration');
 
-            switch (kind) {
-                case 'OptimizedMarshaller':
-                    bean = new Bean('org.apache.ignite.marshaller.optimized.OptimizedMarshaller', 'marshaller', settings)
-                        .intProperty('poolSize')
-                        .intProperty('requireSerializable');
+        return cfg;
+    }
 
-                    break;
+    // Generate marshaller group.
+    static clusterMarshaller(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        const kind = _.get(cluster.marshaller, 'kind');
+        const settings = _.get(cluster.marshaller, kind);
 
-                case 'JdkMarshaller':
-                    bean = new Bean('org.apache.ignite.marshaller.jdk.JdkMarshaller', 'marshaller', settings);
+        let bean;
 
-                    break;
+        switch (kind) {
+            case 'OptimizedMarshaller':
+                bean = new Bean('org.apache.ignite.marshaller.optimized.OptimizedMarshaller', 'marshaller', settings)
+                    .intProperty('poolSize')
+                    .intProperty('requireSerializable');
 
-                default:
-                    // No-op.
-            }
+                break;
 
-            if (bean)
-                cfg.beanProperty('marshaller', bean);
+            case 'JdkMarshaller':
+                bean = new Bean('org.apache.ignite.marshaller.jdk.JdkMarshaller', 'marshaller', settings);
 
-            cfg.intProperty('marshalLocalJobs')
-                .intProperty('marshallerCacheKeepAliveTime')
-                .intProperty('marshallerCacheThreadPoolSize', 'marshallerCachePoolSize');
+                break;
 
-            return cfg;
+            default:
+                // No-op.
         }
 
-        // Generate metrics group.
-        static clusterMetrics(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            cfg.intProperty('metricsExpireTime')
-                .intProperty('metricsHistorySize')
-                .intProperty('metricsLogFrequency')
-                .intProperty('metricsUpdateFrequency');
-
-            return cfg;
-        }
+        if (bean)
+            cfg.beanProperty('marshaller', bean);
 
-        // Generate ODBC group.
-        static clusterODBC(odbc, cfg = this.igniteConfigurationBean()) {
-            if (_.get(odbc, 'odbcEnabled') !== true)
-                return cfg;
+        cfg.intProperty('marshalLocalJobs')
+            .intProperty('marshallerCacheKeepAliveTime')
+            .intProperty('marshallerCacheThreadPoolSize', 'marshallerCachePoolSize');
 
-            const bean = new Bean('org.apache.ignite.configuration.OdbcConfiguration', 'odbcConfiguration',
-                odbc, clusterDflts.odbcConfiguration);
+        return cfg;
+    }
 
-            bean.stringProperty('endpointAddress')
-                .intProperty('maxOpenCursors');
+    // Generate metrics group.
+    static clusterMetrics(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        cfg.intProperty('metricsExpireTime')
+            .intProperty('metricsHistorySize')
+            .intProperty('metricsLogFrequency')
+            .intProperty('metricsUpdateFrequency');
 
-            cfg.beanProperty('odbcConfiguration', bean);
+        return cfg;
+    }
 
+    // Generate ODBC group.
+    static clusterODBC(odbc, cfg = this.igniteConfigurationBean()) {
+        if (_.get(odbc, 'odbcEnabled') !== true)
             return cfg;
-        }
-
-        // Java code generator for cluster's SSL configuration.
-        static clusterSsl(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            if (cluster.sslEnabled && _.nonNil(cluster.sslContextFactory)) {
-                const bean = new Bean('org.apache.ignite.ssl.SslContextFactory', 'sslCtxFactory',
-                    cluster.sslContextFactory);
 
-                bean.intProperty('keyAlgorithm')
-                    .pathProperty('keyStoreFilePath');
+        const bean = new Bean('org.apache.ignite.configuration.OdbcConfiguration', 'odbcConfiguration',
+            odbc, clusterDflts.odbcConfiguration);
 
-                if (_.nonEmpty(bean.valueOf('keyStoreFilePath')))
-                    bean.propertyChar('keyStorePassword', 'ssl.key.storage.password', 'YOUR_SSL_KEY_STORAGE_PASSWORD');
+        bean.stringProperty('endpointAddress')
+            .intProperty('maxOpenCursors');
 
-                bean.intProperty('keyStoreType')
-                    .intProperty('protocol');
+        cfg.beanProperty('odbcConfiguration', bean);
 
-                if (_.nonEmpty(cluster.sslContextFactory.trustManagers)) {
-                    bean.arrayProperty('trustManagers', 'trustManagers',
-                        _.map(cluster.sslContextFactory.trustManagers, (clsName) => new EmptyBean(clsName)),
-                        'javax.net.ssl.TrustManager');
-                }
-                else {
-                    bean.pathProperty('trustStoreFilePath');
+        return cfg;
+    }
 
-                    if (_.nonEmpty(bean.valueOf('trustStoreFilePath')))
-                        bean.propertyChar('trustStorePassword', 'ssl.trust.storage.password', 'YOUR_SSL_TRUST_STORAGE_PASSWORD');
+    // Java code generator for cluster's SSL configuration.
+    static clusterSsl(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        if (cluster.sslEnabled && _.nonNil(cluster.sslContextFactory)) {
+            const bean = new Bean('org.apache.ignite.ssl.SslContextFactory', 'sslCtxFactory',
+                cluster.sslContextFactory);
 
-                    bean.intProperty('trustStoreType');
-                }
+            bean.intProperty('keyAlgorithm')
+                .pathProperty('keyStoreFilePath');
 
-                cfg.beanProperty('sslContextFactory', bean);
-            }
+            if (_.nonEmpty(bean.valueOf('keyStoreFilePath')))
+                bean.propertyChar('keyStorePassword', 'ssl.key.storage.password', 'YOUR_SSL_KEY_STORAGE_PASSWORD');
 
-            return cfg;
-        }
+            bean.intProperty('keyStoreType')
+                .intProperty('protocol');
 
-        // Generate swap group.
-        static clusterSwap(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            if (_.get(cluster.swapSpaceSpi, 'kind') === 'FileSwapSpaceSpi') {
-                const bean = new Bean('org.apache.ignite.spi.swapspace.file.FileSwapSpaceSpi', 'swapSpaceSpi',
-                    cluster.swapSpaceSpi.FileSwapSpaceSpi);
+            if (_.nonEmpty(cluster.sslContextFactory.trustManagers)) {
+                bean.arrayProperty('trustManagers', 'trustManagers',
+                    _.map(cluster.sslContextFactory.trustManagers, (clsName) => new EmptyBean(clsName)),
+                    'javax.net.ssl.TrustManager');
+            }
+            else {
+                bean.pathProperty('trustStoreFilePath');
 
-                bean.pathProperty('baseDirectory')
-                    .intProperty('readStripesNumber')
-                    .floatProperty('maximumSparsity')
-                    .intProperty('maxWriteQueueSize')
-                    .intProperty('writeBufferSize');
+                if (_.nonEmpty(bean.valueOf('trustStoreFilePath')))
+                    bean.propertyChar('trustStorePassword', 'ssl.trust.storage.password', 'YOUR_SSL_TRUST_STORAGE_PASSWORD');
 
-                cfg.beanProperty('swapSpaceSpi', bean);
+                bean.intProperty('trustStoreType');
             }
 
-            return cfg;
+            cfg.beanProperty('sslContextFactory', bean);
         }
 
-        // Generate time group.
-        static clusterTime(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            cfg.intProperty('clockSyncSamples')
-                .intProperty('clockSyncFrequency')
-                .intProperty('timeServerPortBase')
-                .intProperty('timeServerPortRange');
+        return cfg;
+    }
 
-            return cfg;
-        }
+    // Generate swap group.
+    static clusterSwap(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        if (_.get(cluster.swapSpaceSpi, 'kind') === 'FileSwapSpaceSpi') {
+            const bean = new Bean('org.apache.ignite.spi.swapspace.file.FileSwapSpaceSpi', 'swapSpaceSpi',
+                cluster.swapSpaceSpi.FileSwapSpaceSpi);
 
-        // Generate thread pools group.
-        static clusterPools(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            cfg.intProperty('publicThreadPoolSize')
-                .intProperty('systemThreadPoolSize')
-                .intProperty('managementThreadPoolSize')
-                .intProperty('igfsThreadPoolSize')
-                .intProperty('rebalanceThreadPoolSize');
+            bean.pathProperty('baseDirectory')
+                .intProperty('readStripesNumber')
+                .floatProperty('maximumSparsity')
+                .intProperty('maxWriteQueueSize')
+                .intProperty('writeBufferSize');
 
-            return cfg;
+            cfg.beanProperty('swapSpaceSpi', bean);
         }
 
-        // Generate transactions group.
-        static clusterTransactions(transactionConfiguration, cfg = this.igniteConfigurationBean()) {
-            const bean = new Bean('org.apache.ignite.configuration.TransactionConfiguration', 'transactionConfiguration',
-                transactionConfiguration, clusterDflts.transactionConfiguration);
-
-            bean.enumProperty('defaultTxConcurrency')
-                .enumProperty('defaultTxIsolation')
-                .intProperty('defaultTxTimeout')
-                .intProperty('pessimisticTxLogLinger')
-                .intProperty('pessimisticTxLogSize')
-                .boolProperty('txSerializableEnabled')
-                .emptyBeanProperty('txManagerFactory');
+        return cfg;
+    }
 
-            if (bean.nonEmpty())
-                cfg.beanProperty('transactionConfiguration', bean);
+    // Generate time group.
+    static clusterTime(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        cfg.intProperty('clockSyncSamples')
+            .intProperty('clockSyncFrequency')
+            .intProperty('timeServerPortBase')
+            .intProperty('timeServerPortRange');
 
-            return cfg;
-        }
+        return cfg;
+    }
 
-        // Generate user attributes group.
-        static clusterUserAttributes(cluster, cfg = this.igniteConfigurationBean(cluster)) {
-            cfg.mapProperty('attrs', 'attributes', 'userAttributes');
+    // Generate thread pools group.
+    static clusterPools(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        cfg.intProperty('publicThreadPoolSize')
+            .intProperty('systemThreadPoolSize')
+            .intProperty('managementThreadPoolSize')
+            .intProperty('igfsThreadPoolSize')
+            .intProperty('rebalanceThreadPoolSize');
 
-            return cfg;
-        }
+        return cfg;
+    }
 
-        // Generate domain model for general group.
-        static domainModelGeneral(domain, cfg = this.domainConfigurationBean(domain)) {
-            switch (cfg.valueOf('queryMetadata')) {
-                case 'Annotations':
-                    if (_.nonNil(domain.keyType) && _.nonNil(domain.valueType))
-                        cfg.varArgProperty('indexedTypes', 'indexedTypes', [domain.keyType, domain.valueType], 'java.lang.Class');
+    // Generate transactions group.
+    static clusterTransactions(transactionConfiguration, cfg = this.igniteConfigurationBean()) {
+        const bean = new Bean('org.apache.ignite.configuration.TransactionConfiguration', 'transactionConfiguration',
+            transactionConfiguration, clusterDflts.transactionConfiguration);
 
-                    break;
-                case 'Configuration':
-                    cfg.stringProperty('keyType', 'keyType', (val) => JavaTypes.fullClassName(val))
-                        .stringProperty('valueType', 'valueType', (val) => JavaTypes.fullClassName(val));
+        bean.enumProperty('defaultTxConcurrency')
+            .enumProperty('defaultTxIsolation')
+            .intProperty('defaultTxTimeout')
+            .intProperty('pessimisticTxLogLinger')
+            .intProperty('pessimisticTxLogSize')
+            .boolProperty('txSerializableEnabled')
+            .emptyBeanProperty('txManagerFactory');
 
-                    break;
-                default:
-            }
+        if (bean.nonEmpty())
+            cfg.beanProperty('transactionConfiguration', bean);
 
-            return cfg;
-        }
+        return cfg;
+    }
 
-        // Generate domain model for query group.
-        static domainModelQuery(domain, cfg = this.domainConfigurationBean(domain)) {
-            if (cfg.valueOf('queryMetadata') === 'Configuration') {
-                const fields = _.map(domain.fields,
-                    (e) => ({name: e.name, className: JavaTypes.fullClassName(e.className)}));
+    // Generate user attributes group.
+    static clusterUserAttributes(cluster, cfg = this.igniteConfigurationBean(cluster)) {
+        cfg.mapProperty('attrs', 'attributes', 'userAttributes');
 
-                cfg.mapProperty('fields', fields, 'fields', true)
-                    .mapProperty('aliases', 'aliases');
+        return cfg;
+    }
 
-                const indexes = _.map(domain.indexes, (index) =>
-                    new Bean('org.apache.ignite.cache.QueryIndex', 'index', index, cacheDflts.indexes)
-                        .stringProperty('name')
-                        .enumProperty('indexType')
-                        .mapProperty('indFlds', 'fields', 'fields', true)
-                );
+    // Generate domain model for general group.
+    static domainModelGeneral(domain, cfg = this.domainConfigurationBean(domain)) {
+        switch (cfg.valueOf('queryMetadata')) {
+            case 'Annotations':
+                if (_.nonNil(domain.keyType) && _.nonNil(domain.valueType))
+                    cfg.varArgProperty('indexedTypes', 'indexedTypes', [domain.keyType, domain.valueType], 'java.lang.Class');
 
-                cfg.collectionProperty('indexes', 'indexes', indexes, 'org.apache.ignite.cache.QueryIndex');
-            }
+                break;
+            case 'Configuration':
+                cfg.stringProperty('keyType', 'keyType', (val) => javaTypes.fullClassName(val))
+                    .stringProperty('valueType', 'valueType', (val) => javaTypes.fullClassName(val));
 
-            return cfg;
+                break;
+            default:
         }
 
-        // Generate domain model db fields.
-        static _domainModelDatabaseFields(cfg, propName, domain) {
-            const fields = _.map(domain[propName], (field) => {
-                return new Bean('org.apache.ignite.cache.store.jdbc.JdbcTypeField', 'typeField', field, cacheDflts.typeField)
-                    .constantConstructorArgument('databaseFieldType')
-                    .stringConstructorArgument('databaseFieldName')
-                    .classConstructorArgument('javaFieldType')
-                    .stringConstructorArgument('javaFieldName');
-            });
-
-            cfg.varArgProperty(propName, propName, fields, 'org.apache.ignite.cache.store.jdbc.JdbcTypeField');
+        return cfg;
+    }
 
-            return cfg;
-        }
+    // Generate domain model for query group.
+    static domainModelQuery(domain, cfg = this.domainConfigurationBean(domain)) {
+        if (cfg.valueOf('queryMetadata') === 'Configuration') {
+            const fields = _.map(domain.fields,
+                (e) => ({name: e.name, className: javaTypes.fullClassName(e.className)}));
 
-        // Generate domain model for store group.
-        static domainStore(domain, cfg = this.domainConfigurationBean(domain)) {
-            cfg.stringProperty('databaseSchema')
-                .stringProperty('databaseTable');
+            cfg.mapProperty('fields', fields, 'fields', true)
+                .mapProperty('aliases', 'aliases');
 
-            this._domainModelDatabaseFields(cfg, 'keyFields', domain);
-            this._domainModelDatabaseFields(cfg, 'valueFields', domain);
+            const indexes = _.map(domain.indexes, (index) =>
+                new Bean('org.apache.ignite.cache.QueryIndex', 'index', index, cacheDflts.indexes)
+                    .stringProperty('name')
+                    .enumProperty('indexType')
+                    .mapProperty('indFlds', 'fields', 'fields', true)
+            );
 
-            return cfg;
+            cfg.collectionProperty('indexes', 'indexes', indexes, 'org.apache.ignite.cache.QueryIndex');
         }
 
-        /**
-         * Generate eviction policy object.
-         * @param {Object} ccfg Parent configuration.
-         * @param {String} name Property name.
-         * @param {Object} src Source.
-         * @param {Object} dflt Default.
-         * @returns {Object} Parent configuration.
-         * @private
-         */
-        static _evictionPolicy(ccfg, name, src, dflt) {
-            let bean;
-
-            switch (_.get(src, 'kind')) {
-                case 'LRU':
-                    bean = new Bean('org.apache.ignite.cache.eviction.lru.LruEvictionPolicy', 'evictionPlc',
-                        src.LRU, dflt.LRU);
-
-                    break;
-                case 'FIFO':
-                    bean = new Bean('org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy', 'evictionPlc',
-                        src.FIFO, dflt.FIFO);
-
-                    break;
-                case 'SORTED':
-                    bean = new Bean('org.apache.ignite.cache.eviction.sorted.SortedEvictionPolicy', 'evictionPlc',
-                        src.SORTED, dflt.SORTED);
-
-                    break;
-                default:
-                    return ccfg;
-            }
+        return cfg;
+    }
 
-            bean.intProperty('batchSize')
-                .intProperty('maxMemorySize')
-                .intProperty('maxSize');
+    // Generate domain model db fields.
+    static _domainModelDatabaseFields(cfg, propName, domain) {
+        const fields = _.map(domain[propName], (field) => {
+            return new Bean('org.apache.ignite.cache.store.jdbc.JdbcTypeField', 'typeField', field, cacheDflts.typeField)
+                .constantConstructorArgument('databaseFieldType')
+                .stringConstructorArgument('databaseFieldName')
+                .classConstructorArgument('javaFieldType')
+                .stringConstructorArgument('javaFieldName');
+        });
 
-            ccfg.beanProperty(name, bean);
+        cfg.varArgProperty(propName, propName, fields, 'org.apache.ignite.cache.store.jdbc.JdbcTypeField');
 
-            return ccfg;
-        }
-
-        // Generate cache general group.
-        static cacheGeneral(cache, ccfg = this.cacheConfigurationBean(cache)) {
-            ccfg.stringProperty('name')
-                .enumProperty('cacheMode')
-                .enumProperty('atomicityMode');
+        return cfg;
+    }
 
-            if (ccfg.valueOf('cacheMode') === 'PARTITIONED' && ccfg.valueOf('backups')) {
-                ccfg.intProperty('backups')
-                    .intProperty('readFromBackup');
-            }
+    // Generate domain model for store group.
+    static domainStore(domain, cfg = this.domainConfigurationBean(domain)) {
+        cfg.stringProperty('databaseSchema')
+            .stringProperty('databaseTable');
 
-            ccfg.intProperty('copyOnRead');
+        this._domainModelDatabaseFields(cfg, 'keyFields', domain);
+        this._domainModelDatabaseFields(cfg, 'valueFields', domain);
 
-            if (ccfg.valueOf('cacheMode') === 'PARTITIONED' && ccfg.valueOf('atomicityMode') === 'TRANSACTIONAL')
-                ccfg.intProperty('invalidate');
+        return cfg;
+    }
 
-            return ccfg;
+    /**
+     * Generate eviction policy object.
+     * @param {Object} ccfg Parent configuration.
+     * @param {String} name Property name.
+     * @param {Object} src Source.
+     * @param {Object} dflt Default.
+     * @returns {Object} Parent configuration.
+     * @private
+     */
+    static _evictionPolicy(ccfg, name, src, dflt) {
+        let bean;
+
+        switch (_.get(src, 'kind')) {
+            case 'LRU':
+                bean = new Bean('org.apache.ignite.cache.eviction.lru.LruEvictionPolicy', 'evictionPlc',
+                    src.LRU, dflt.LRU);
+
+                break;
+            case 'FIFO':
+                bean = new Bean('org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy', 'evictionPlc',
+                    src.FIFO, dflt.FIFO);
+
+                break;
+            case 'SORTED':
+                bean = new Bean('org.apache.ignite.cache.eviction.sorted.SortedEvictionPolicy', 'evictionPlc',
+                    src.SORTED, dflt.SORTED);
+
+                break;
+            default:
+                return ccfg;
         }
 
-        // Generate cache memory group.
-        static cacheMemory(cache, ccfg = this.cacheConfigurationBean(cache)) {
-            ccfg.enumProperty('memoryMode');
+        bean.intProperty('batchSize')
+            .intProperty('maxMemorySize')
+            .intProperty('maxSize');
 
-            if (ccfg.valueOf('memoryMode') !== 'OFFHEAP_VALUES')
-                ccfg.intProperty('offHeapMaxMemory');
+        ccfg.beanProperty(name, bean);
 
-            this._evictionPolicy(ccfg, 'evictionPolicy', cache.evictionPolicy, cacheDflts.evictionPolicy);
+        return ccfg;
+    }
 
-            ccfg.intProperty('startSize')
-                .boolProperty('swapEnabled');
+    // Generate cache general group.
+    static cacheGeneral(cache, ccfg = this.cacheConfigurationBean(cache)) {
+        ccfg.stringProperty('name')
+            .enumProperty('cacheMode')
+            .enumProperty('atomicityMode');
 
-            return ccfg;
+        if (ccfg.valueOf('cacheMode') === 'PARTITIONED' && ccfg.valueOf('backups')) {
+            ccfg.intProperty('backups')
+                .intProperty('readFromBackup');
         }
 
-        // Generate cache queries & Indexing group.
-        static cacheQuery(cache, domains, ccfg = this.cacheConfigurationBean(cache)) {
-            const indexedTypes = _.reduce(domains, (acc, domain) => {
-                if (domain.queryMetadata === 'Annotations')
-                    acc.push(domain.keyType, domain.valueType);
-
-                return acc;
-            }, []);
+        ccfg.intProperty('copyOnRead');
 
-            ccfg.stringProperty('sqlSchema')
-                .intProperty('sqlOnheapRowCacheSize')
-                .intProperty('longQueryWarningTimeout')
-                .arrayProperty('indexedTypes', 'indexedTypes', indexedTypes, 'java.lang.Class')
-                .arrayProperty('sqlFunctionClasses', 'sqlFunctionClasses', cache.sqlFunctionClasses, 'java.lang.Class')
-                .intProperty('snapshotableIndex')
-                .intProperty('sqlEscapeAll');
-
-            return ccfg;
-        }
+        if (ccfg.valueOf('cacheMode') === 'PARTITIONED' && ccfg.valueOf('atomicityMode') === 'TRANSACTIONAL')
+            ccfg.intProperty('invalidate');
 
-        // Generate cache store group.
-        static cacheStore(cache, domains, ccfg = this.cacheConfigurationBean(cache)) {
-            const kind = _.get(cache, 'cacheStoreFactory.kind');
+        return ccfg;
+    }
 
-            if (kind && cache.cacheStoreFactory[kind]) {
-                let bean = null;
+    // Generate cache memory group.
+    static cacheMemory(cache, ccfg = this.cacheConfigurationBean(cache)) {
+        ccfg.enumProperty('memoryMode');
 
-                const storeFactory = cache.cacheStoreFactory[kind];
+        if (ccfg.valueOf('memoryMode') !== 'OFFHEAP_VALUES')
+            ccfg.intProperty('offHeapMaxMemory');
 
-                switch (kind) {
-                    case 'CacheJdbcPojoStoreFactory':
-                        bean = new Bean('org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory', 'cacheStoreFactory',
-                            storeFactory, cacheDflts.cacheStoreFactory.CacheJdbcPojoStoreFactory);
+        this._evictionPolicy(ccfg, 'evictionPolicy', cache.evictionPolicy, cacheDflts.evictionPolicy);
 
-                        const jdbcId = bean.valueOf('dataSourceBean');
+        ccfg.intProperty('startSize')
+            .boolProperty('swapEnabled');
 
-                        bean.dataSource(jdbcId, 'dataSourceBean', this.dataSourceBean(jdbcId, storeFactory.dialect))
-                            .beanProperty('dialect', new EmptyBean(this.dialectClsName(storeFactory.dialect)));
+        return ccfg;
+    }
 
-                        bean.intProperty('batchSize')
-                            .intProperty('maximumPoolSize')
-                            .intProperty('maximumWriteAttempts')
-                            .intProperty('parallelLoadCacheMinimumThreshold')
-                            .emptyBeanProperty('hasher')
-                            .emptyBeanProperty('transformer')
-                            .boolProperty('sqlEscapeAll');
+    // Generate cache queries & Indexing group.
+    static cacheQuery(cache, domains, ccfg = this.cacheConfigurationBean(cache)) {
+        const indexedTypes = _.reduce(domains, (acc, domain) => {
+            if (domain.queryMetadata === 'Annotations')
+                acc.push(domain.keyType, domain.valueType);
+
+            return acc;
+        }, []);
+
+        ccfg.stringProperty('sqlSchema')
+            .intProperty('sqlOnheapRowCacheSize')
+            .intProperty('longQueryWarningTimeout')
+            .arrayProperty('indexedTypes', 'indexedTypes', indexedTypes, 'java.lang.Class')
+            .intProperty('queryDetailMetricsSize')
+            .arrayProperty('sqlFunctionClasses', 'sqlFunctionClasses', cache.sqlFunctionClasses, 'java.lang.Class')
+            .intProperty('snapshotableIndex')
+            .intProperty('sqlEscapeAll');
+
+        return ccfg;
+    }
 
-                        const setType = (typeBean, propName) => {
-                            if (JavaTypes.nonBuiltInClass(typeBean.valueOf(propName)))
-                                typeBean.stringProperty(propName);
-                            else
-                                typeBean.classProperty(propName);
-                        };
+    // Generate cache store group.
+    static cacheStore(cache, domains, ccfg = this.cacheConfigurationBean(cache)) {
+        const kind = _.get(cache, 'cacheStoreFactory.kind');
 
-                        const types = _.reduce(domains, (acc, domain) => {
-                            if (_.isNil(domain.databaseTable))
-                                return acc;
+        if (kind && cache.cacheStoreFactory[kind]) {
+            let bean = null;
 
-                            const typeBean = new Bean('org.apache.ignite.cache.store.jdbc.JdbcType', 'type',
-                                _.merge({}, domain, {cacheName: cache.name}))
-                                .stringProperty('cacheName');
+            const storeFactory = cache.cacheStoreFactory[kind];
 
-                            setType(typeBean, 'keyType');
-                            setType(typeBean, 'valueType');
+            switch (kind) {
+                case 'CacheJdbcPojoStoreFactory':
+                    bean = new Bean('org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory', 'cacheStoreFactory',
+                        storeFactory, cacheDflts.cacheStoreFactory.CacheJdbcPojoStoreFactory);
+
+                    const jdbcId = bean.valueOf('dataSourceBean');
+
+                    bean.dataSource(jdbcId, 'dataSourceBean', this.dataSourceBean(jdbcId, storeFactory.dialect))
+                        .beanProperty('dialect', new EmptyBean(this.dialectClsName(storeFactory.dialect)));
+
+                    bean.intProperty('batchSize')
+                        .intProperty('maximumPoolSize')
+                        .intProperty('maximumWriteAttempts')
+                        .intProperty('parallelLoadCacheMinimumThreshold')
+                        .emptyBeanProperty('hasher')
+                        .emptyBeanProperty('transformer')
+                        .boolProperty('sqlEscapeAll');
+
+                    const setType = (typeBean, propName) => {
+                        if (javaTypes.nonBuiltInClass(typeBean.valueOf(propName)))
+                            typeBean.stringProperty(propName);
+                        else
+                            typeBean.classProperty(propName);
+                    };
+
+                    const types = _.reduce(domains, (acc, domain) => {
+                        if (_.isNil(domain.databaseTable))
+                            return acc;
 
-                            this.domainStore(domain, typeBean);
+                        const typeBean = new Bean('org.apache.ignite.cache.store.jdbc.JdbcType', 'type',
+                            _.merge({}, domain, {cacheName: cache.name}))
+                            .stringProperty('cacheName');
 
-                            acc.push(typeBean);
+                        setType(typeBean, 'keyType');
+                        setType(typeBean, 'valueType');
 
-                            return acc;
-                        }, []);
+                        this.domainStore(domain, typeBean);
 
-                        bean.varArgProperty('types', 'types', types, 'org.apache.ignite.cache.store.jdbc.JdbcType');
+                        acc.push(typeBean);
 
-                        break;
-                    case 'CacheJdbcBlobStoreFactory':
-                        bean = new Bean('org.apache.ignite.cache.store.jdbc.CacheJdbcBlobStoreFactory', 'cacheStoreFactory',
-                            storeFactory);
+                        return acc;
+                    }, []);
 
-                        if (bean.valueOf('connectVia') === 'DataSource') {
-                            const blobId = bean.valueOf('dataSourceBean');
+                    bean.varArgProperty('types', 'types', types, 'org.apache.ignite.cache.store.jdbc.JdbcType');
 
-                            bean.dataSource(blobId, 'dataSourceBean', this.dataSourceBean(blobId, storeFactory.dialect));
-                        }
-                        else {
-                            ccfg.stringProperty('connectionUrl')
-                                .stringProperty('user')
-                                .property('password', `ds.${storeFactory.user}.password`, 'YOUR_PASSWORD');
-                        }
+                    break;
+                case 'CacheJdbcBlobStoreFactory':
+                    bean = new Bean('org.apache.ignite.cache.store.jdbc.CacheJdbcBlobStoreFactory', 'cacheStoreFactory',
+                        storeFactory);
 
-                        bean.boolProperty('initSchema')
-                            .stringProperty('createTableQuery')
-                            .stringProperty('loadQuery')
-                            .stringProperty('insertQuery')
-                            .stringProperty('updateQuery')
-                            .stringProperty('deleteQuery');
+                    if (bean.valueOf('connectVia') === 'DataSource') {
+                        const blobId = bean.valueOf('dataSourceBean');
 
-                        break;
-                    case 'CacheHibernateBlobStoreFactory':
-                        bean = new Bean('org.apache.ignite.cache.store.hibernate.CacheHibernateBlobStoreFactory',
-                            'cacheStoreFactory', storeFactory);
+                        bean.dataSource(blobId, 'dataSourceBean', this.dataSourceBean(blobId, storeFactory.dialect));
+                    }
+                    else {
+                        ccfg.stringProperty('connectionUrl')
+                            .stringProperty('user')
+                            .property('password', `ds.${storeFactory.user}.password`, 'YOUR_PASSWORD');
+                    }
 
-                        bean.propsProperty('props', 'hibernateProperties');
+                    bean.boolProperty('initSchema')
+                        .stringProperty('createTableQuery')
+                        .stringProperty('loadQuery')
+                        .stringProperty('insertQuery')
+                        .stringProperty('updateQuery')
+                        .stringProperty('deleteQuery');
 
-                        break;
-                    default:
-                }
+                    break;
+                case 'CacheHibernateBlobStoreFactory':
+                    bean = new Bean('org.apache.ignite.cache.store.hibernate.CacheHibernateBlobStoreFactory',
+                        'cacheStoreFactory', storeFactory);
 
-                if (bean)
-                    ccfg.beanProperty('cacheStoreFactory', bean);
-            }
+                    bean.propsProperty('props', 'hibernateProperties');
 
-            ccfg.boolProperty('storeKeepBinary')
-                .boolProperty('loadPreviousValue')
-                .boolProperty('readThrough')
-                .boolProperty('writeThrough');
-
-            if (ccfg.valueOf('writeBehindEnabled')) {
-                ccfg.boolProperty('writeBehindEnabled')
-                    .intProperty('writeBehindBatchSize')
-                    .intProperty('writeBehindFlushSize')
-                    .intProperty('writeBehindFlushFrequency')
-                    .intProperty('writeBehindFlushThreadCount');
+                    break;
+                default:
             }
 
-            return ccfg;
+            if (bean)
+                ccfg.beanProperty('cacheStoreFactory', bean);
         }
 
-        // Generate cache concurrency control group.
-        static cacheConcurrency(cache, ccfg = this.cacheConfigurationBean(cache)) {
-            ccfg.intProperty('maxConcurrentAsyncOperations')
-                .intProperty('defaultLockTimeout')
-                .enumProperty('atomicWriteOrderMode')
-                .enumProperty('writeSynchronizationMode');
-
-            return ccfg;
+        ccfg.boolProperty('storeKeepBinary')
+            .boolProperty('loadPreviousValue')
+            .boolProperty('readThrough')
+            .boolProperty('writeThrough');
+
+        if (ccfg.valueOf('writeBehindEnabled')) {
+            ccfg.boolProperty('writeBehindEnabled')
+                .intProperty('writeBehindBatchSize')
+                .intProperty('writeBehindFlushSize')
+                .intProperty('writeBehindFlushFrequency')
+                .intProperty('writeBehindFlushThreadCount');
         }
 
-        // Generate cache node filter group.
-        static cacheNodeFilter(cache, igfss, ccfg = this.cacheConfigurationBean(cache)) {
-            const kind = _.get(cache, 'nodeFilter.kind');
+        return ccfg;
+    }
 
-            const settings = _.get(cache.nodeFilter, kind);
+    // Generate cache concurrency control group.
+    static cacheConcurrency(cache, ccfg = this.cacheConfigurationBean(cache)) {
+        ccfg.intProperty('maxConcurrentAsyncOperations')
+            .intProperty('defaultLockTimeout')
+            .enumProperty('atomicWriteOrderMode')
+            .enumProperty('writeSynchronizationMode');
 
-            if (_.isNil(settings))
-                return ccfg;
+        return ccfg;
+    }
 
-            let bean = null;
+    // Generate cache node filter group.
+    static cacheNodeFilter(cache, igfss, ccfg = this.cacheConfigurationBean(cache)) {
+        const kind = _.get(cache, 'nodeFilter.kind');
 
-            switch (kind) {
-                case 'IGFS':
-                    const foundIgfs = _.find(igfss, {_id: settings.igfs});
+        const settings = _.get(cache.nodeFilter, kind);
 
-                    if (foundIgfs) {
-                        bean = new Bean('org.apache.ignite.internal.processors.igfs.IgfsNodePredicate', 'nodeFilter', foundIgfs)
-                            .stringConstructorArgument('name');
-                    }
+        if (_.isNil(settings))
+            return ccfg;
 
-                    break;
-                case 'Custom':
-                    if (_.nonEmpty(settings.className))
-                        bean = new EmptyBean(settings.className);
+        let bean = null;
 
-                    break;
-                default:
-                    // No-op.
-            }
+        switch (kind) {
+            case 'IGFS':
+                const foundIgfs = _.find(igfss, {_id: settings.igfs});
 
-            if (bean)
-                ccfg.beanProperty('nodeFilter', bean);
+                if (foundIgfs) {
+                    bean = new Bean('org.apache.ignite.internal.processors.igfs.IgfsNodePredicate', 'nodeFilter', foundIgfs)
+                        .stringConstructorArgument('name');
+                }
 
-            return ccfg;
-        }
+                break;
+            case 'Custom':
+                if (_.nonEmpty(settings.className))
+                    bean = new EmptyBean(settings.className);
 
-        // Generate cache rebalance group.
-        static cacheRebalance(cache, ccfg = this.cacheConfigurationBean(cache)) {
-            if (ccfg.valueOf('cacheMode') !== 'LOCAL') {
-                ccfg.enumProperty('rebalanceMode')
-                    .intProperty('rebalanceThreadPoolSize')
-                    .intProperty('rebalanceBatchSize')
-                    .intProperty('rebalanceBatchesPrefetchCount')
-                    .intProperty('rebalanceOrder')
-                    .intProperty('rebalanceDelay')
-                    .intProperty('rebalanceTimeout')
-                    .intProperty('rebalanceThrottle');
-            }
+                break;
+            default:
+                // No-op.
+        }
 
-            if (ccfg.includes('igfsAffinnityGroupSize')) {
-                const bean = new Bean('org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper', 'affinityMapper', cache)
-                    .intConstructorArgument('igfsAffinnityGroupSize');
+        if (bean)
+            ccfg.beanProperty('nodeFilter', bean);
 
-                ccfg.beanProperty('affinityMapper', bean);
-            }
+        return ccfg;
+    }
 
-            return ccfg;
+    // Generate cache rebalance group.
+    static cacheRebalance(cache, ccfg = this.cacheConfigurationBean(cache)) {
+        if (ccfg.valueOf('cacheMode') !== 'LOCAL') {
+            ccfg.enumProperty('rebalanceMode')
+                .intProperty('rebalanceThreadPoolSize')
+                .intProperty('rebalanceBatchSize')
+                .intProperty('rebalanceBatchesPrefetchCount')
+                .intProperty('rebalanceOrder')
+                .intProperty('rebalanceDelay')
+                .intProperty('rebalanceTimeout')
+                .intProperty('rebalanceThrottle');
         }
 
-        // Generate server near cache group.
-        static cacheNearServer(cache, ccfg = this.cacheConfigurationBean(cache)) {
-            if (ccfg.valueOf('cacheMode') === 'PARTITIONED' && _.get(cache, 'nearConfiguration.enabled')) {
-                const bean = new Bean('org.apache.ignite.configuration.NearCacheConfiguration', 'nearConfiguration',
-                    cache.nearConfiguration, cacheDflts.nearConfiguration);
+        if (ccfg.includes('igfsAffinnityGroupSize')) {
+            const bean = new Bean('org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper', 'affinityMapper', cache)
+                .intConstructorArgument('igfsAffinnityGroupSize');
 
-                bean.intProperty('nearStartSize');
+            ccfg.beanProperty('affinityMapper', bean);
+        }
 
-                this._evictionPolicy(bean, 'nearEvictionPolicy',
-                    bean.valueOf('nearEvictionPolicy'), cacheDflts.evictionPolicy);
+        return ccfg;
+    }
 
-                ccfg.beanProperty('nearConfiguration', bean);
-            }
+    // Generate server near cache group.
+    static cacheNearServer(cache, ccfg = this.cacheConfigurationBean(cache)) {
+        if (ccfg.valueOf('cacheMode') === 'PARTITIONED' && _.get(cache, 'nearConfiguration.enabled')) {
+            const bean = new Bean('org.apache.ignite.configuration.NearCacheConfiguration', 'nearConfiguration',
+                cache.nearConfiguration, cacheDflts.nearConfiguration);
 
-            return ccfg;
-        }
+            bean.intProperty('nearStartSize');
 
-        // Generate client near cache group.
-        static cacheNearClient(cache, ccfg = this.cacheConfigurationBean(cache)) {
-            if (ccfg.valueOf('cacheMode') === 'PARTITIONED' && _.get(cache, 'clientNearConfiguration.enabled')) {
-                const bean = new Bean('org.apache.ignite.configuration.NearCacheConfiguration',
-                    JavaTypes.toJavaName('nearConfiguration', ccfg.valueOf('name')),
-                    cache.clientNearConfiguration, cacheDflts.clientNearConfiguration);
+            this._evictionPolicy(bean, 'nearEvictionPolicy',
+                bean.valueOf('nearEvictionPolicy'), cacheDflts.evictionPolicy);
 
-                bean.intProperty('nearStartSize');
+            ccfg.beanProperty('nearConfiguration', bean);
+        }
 
-                this._evictionPolicy(bean, 'nearEvictionPolicy',
-                    bean.valueOf('nearEvictionPolicy'), cacheDflts.evictionPolicy);
+        return ccfg;
+    }
 
-                return bean;
-            }
+    // Generate client near cache group.
+    static cacheNearClient(cache, ccfg = this.cacheConfigurationBean(cache)) {
+        if (ccfg.valueOf('cacheMode') === 'PARTITIONED' && _.get(cache, 'clientNearConfiguration.enabled')) {
+            const bean = new Bean('org.apache.ignite.configuration.NearCacheConfiguration',
+                javaTypes.toJavaName('nearConfiguration', ccfg.valueOf('name')),
+                cache.clientNearConfiguration, cacheDflts.clientNearConfiguration);
 
-            return ccfg;
-        }
+            bean.intProperty('nearStartSize');
 
-        // Generate cache statistics group.
-        static cacheStatistics(cache, ccfg = this.cacheConfigurationBean(cache)) {
-            ccfg.boolProperty('statisticsEnabled')
-                .boolProperty('managementEnabled');
+            this._evictionPolicy(bean, 'nearEvictionPolicy',
+                bean.valueOf('nearEvictionPolicy'), cacheDflts.evictionPolicy);
 
-            return ccfg;
+            return bean;
         }
 
-        // Generate domain models configs.
-        static cacheDomains(domains, ccfg) {
-            const qryEntities = _.reduce(domains, (acc, domain) => {
-                if (_.isNil(domain.queryMetadata) || domain.queryMetadata === 'Configuration') {
-                    const qryEntity = this.domainModelGeneral(domain);
-
-                    this.domainModelQuery(domain, qryEntity);
+        return ccfg;
+    }
 
-                    acc.push(qryEntity);
-                }
+    // Generate cache statistics group.
+    static cacheStatistics(cache, ccfg = this.cacheConfigurationBean(cache)) {
+        ccfg.boolProperty('statisticsEnabled')
+            .boolProperty('managementEnabled');
 
-                return acc;
-            }, []);
+        return ccfg;
+    }
 
-            ccfg.collectionProperty('qryEntities', 'queryEntities', qryEntities, 'org.apache.ignite.cache.QueryEntity');
-        }
+    // Generate domain models configs.
+    static cacheDomains(domains, ccfg) {
+        const qryEntities = _.reduce(domains, (acc, domain) => {
+            if (_.isNil(domain.queryMetadata) || domain.queryMetadata === 'Configuration') {
+                const qryEntity = this.domainModelGeneral(domain);
 
-        static cacheConfiguration(cache, ccfg = this.cacheConfigurationBean(cache)) {
-            this.cacheGeneral(cache, ccfg);
-            this.cacheMemory(cache, ccfg);
-            this.cacheQuery(cache, cache.domains, ccfg);
-            this.cacheStore(cache, cache.domains, ccfg);
+                this.domainModelQuery(domain, qryEntity);
 
-            const igfs = _.get(cache, 'nodeFilter.IGFS.instance');
-            this.cacheNodeFilter(cache, igfs ? [igfs] : [], ccfg);
-            this.cacheConcurrency(cache, ccfg);
-            this.cacheRebalance(cache, ccfg);
-            this.cacheNearServer(cache, ccfg);
-            this.cacheStatistics(cache, ccfg);
-            this.cacheDomains(cache.domains, ccfg);
+                acc.push(qryEntity);
+            }
 
-            return ccfg;
-        }
+            return acc;
+        }, []);
 
-        // Generate IGFS general group.
-        static igfsGeneral(igfs, cfg = this.igfsConfigurationBean(igfs)) {
-            if (_.isEmpty(igfs.name))
-                return cfg;
+        ccfg.collectionProperty('qryEntities', 'queryEntities', qryEntities, 'org.apache.ignite.cache.QueryEntity');
+    }
 
-            cfg.stringProperty('name')
-                .stringProperty('name', 'dataCacheName', (name) => name + '-data')
-                .stringProperty('name', 'metaCacheName', (name) => name + '-meta')
-                .enumProperty('defaultMode');
+    static cacheConfiguration(cache, ccfg = this.cacheConfigurationBean(cache)) {
+        this.cacheGeneral(cache, ccfg);
+        this.cacheMemory(cache, ccfg);
+        this.cacheQuery(cache, cache.domains, ccfg);
+        this.cacheStore(cache, cache.domains, ccfg);
+
+        const igfs = _.get(cache, 'nodeFilter.IGFS.instance');
+        this.cacheNodeFilter(cache, igfs ? [igfs] : [], ccfg);
+        this.cacheConcurrency(cache, ccfg);
+        this.cacheRebalance(cache, ccfg);
+        this.cacheNearServer(cache, ccfg);
+        this.cacheStatistics(cache, ccfg);
+        this.cacheDomains(cache.domains, ccfg);
+
+        return ccfg;
+    }
 
+    // Generate IGFS general group.
+    static igfsGeneral(igfs, cfg = this.igfsConfigurationBean(igfs)) {
+        if (_.isEmpty(igfs.name))
             return cfg;
-        }
 
-        // Generate IGFS secondary file system group.
-        static igfsSecondFS(igfs, cfg = this.igfsConfigurationBean(igfs)) {
-            if (igfs.secondaryFileSystemEnabled) {
-                const secondFs = igfs.secondaryFileSystem || {};
+        cfg.stringProperty('name')
+            .stringProperty('name', 'dataCacheName', (name) => name + '-data')
+            .stringProperty('name', 'metaCacheName', (name) => name + '-meta')
+            .enumProperty('defaultMode');
 
-                const bean = new Bean('org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem',
-                    'secondaryFileSystem', secondFs, igfsDflts.secondaryFileSystem);
+        return cfg;
+    }
 
-                bean.stringProperty('userName', 'defaultUserName');
+    // Generate IGFS secondary file system group.
+    static igfsSecondFS(igfs, cfg = this.igfsConfigurationBean(igfs)) {
+        if (igfs.secondaryFileSystemEnabled) {
+            const secondFs = igfs.secondaryFileSystem || {};
 
-                const factoryBean = new Bean('org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory',
-                    'fac', secondFs);
+            const bean = new Bean('org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem',
+                'secondaryFileSystem', secondFs, igfsDflts.secondaryFileSystem);
 
-                factoryBean.stringProperty('uri')
-                    .pathProperty('cfgPath', 'configPaths');
+            bean.stringProperty('userName', 'defaultUserName');
 
-                bean.beanProperty('fileSystemFactory', factoryBean);
+            const factoryBean = new Bean('org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory',
+                'fac', secondFs);
 
-                cfg.beanProperty('secondaryFileSystem', bean);
-            }
+            factoryBean.stringProperty('uri')
+                .pathProperty('cfgPath', 'configPaths');
 
-            return cfg;
+            bean.beanProperty('fileSystemFactory', factoryBean);
+
+            cfg.beanProperty('secondaryFileSystem', bean);
         }
 
-        // Generate IGFS IPC group.
-        static igfsIPC(igfs, cfg = this.igfsConfigurationBean(igfs)) {
-            if (igfs.ipcEndpointEnabled) {
-                const bean = new Bean('org.apache.ignite.igfs.IgfsIpcEndpointConfiguration', 'ipcEndpointConfiguration',
-                    igfs.ipcEndpointConfiguration, igfsDflts.ipcEndpointConfiguration);
-
-                bean.enumProperty('type')
-                    .stringProperty('host')
-                    .intProperty('port')
-                    .intProperty('memorySize')
-                    .pathProperty('tokenDirectoryPath')
-                    .intProperty('threadCount');
-
-                if (bean.nonEmpty())
-                    cfg.beanProperty('ipcEndpointConfiguration', bean);
-            }
+        return cfg;
+    }
 
-            return cfg;
-        }
+    // Generate IGFS IPC group.
+    static igfsIPC(igfs, cfg = this.igfsConfigurationBean(igfs)) {
+        if (igfs.ipcEndpointEnabled) {
+            const bean = new Bean('org.apache.ignite.igfs.IgfsIpcEndpointConfiguration', 'ipcEndpointConfiguration',
+                igfs.ipcEndpointConfiguration, igfsDflts.ipcEndpointConfiguration);
 
-        // Generate IGFS fragmentizer group.
-        static igfsFragmentizer(igfs, cfg = this.igfsConfigurationBean(igfs)) {
-            if (igfs.fragmentizerEnabled) {
-                cfg.intProperty('fragmentizerConcurrentFiles')
-                    .intProperty('fragmentizerThrottlingBlockLength')
-                    .intProperty('fragmentizerThrottlingDelay');
-            }
-            else
-                cfg.boolProperty('fragmentizerEnabled');
+            bean.enumProperty('type')
+                .stringProperty('host')
+                .intProperty('port')
+                .intProperty('memorySize')
+                .pathProperty('tokenDirectoryPath')
+                .intProperty('threadCount');
 
-            return cfg;
+            if (bean.nonEmpty())
+                cfg.beanProperty('ipcEndpointConfiguration', bean);
         }
 
-        // Generate IGFS Dual mode group.
-        static igfsDualMode(igfs, cfg = this.igfsConfigurationBean(igfs)) {
-            cfg.intProperty('dualModeMaxPendingPutsSize')
-                .emptyBeanProperty('dualModePutExecutorService')
-                .intProperty('dualModePutExecutorServiceShutdown');
+        return cfg;
+    }
 
-            return cfg;
+    // Generate IGFS fragmentizer group.
+    static igfsFragmentizer(igfs, cfg = this.igfsConfigurationBean(igfs)) {
+        if (igfs.fragmentizerEnabled) {
+            cfg.intProperty('fragmentizerConcurrentFiles')
+                .intProperty('fragmentizerThrottlingBlockLength')
+                .intProperty('fragmentizerThrottlingDelay');
         }
+        else
+            cfg.boolProperty('fragmentizerEnabled');
 
-        // Generate IGFS miscellaneous group.
-        static igfsMisc(igfs, cfg = this.igfsConfigurationBean(igfs)) {
-            cfg.intProperty('blockSize')
-                .intProperty('streamBufferSize')
-                .intProperty('maxSpaceSize')
-                .intProperty('maximumTaskRangeLength')
-                .intProperty('managementPort')
-                .intProperty('perNodeBatchSize')
-                .intProperty('perNodeParallelBatchCount')
-                .intProperty('prefetchBlocks')
-                .intProperty('sequentialReadsBeforePrefetch')
-                .intProperty('trashPurgeTimeout')
-                .intProperty('colocateMetadata')
-                .intProperty('relaxedConsistency')
-                .mapProperty('pathModes', 'pathModes');
+        return cfg;
+    }
 
-            return cfg;
-        }
+    // Generate IGFS Dual mode group.
+    static igfsDualMode(igfs, cfg = this.igfsConfigurationBean(igfs)) {
+        cfg.intProperty('dualModeMaxPendingPutsSize')
+            .emptyBeanProperty('dualModePutExecutorService')
+            .intProperty('dualModePutExecutorServiceShutdown');
+
+        return cfg;
     }
 
-    return ConfigurationGenerator;
-}];
+    // Generate IGFS miscellaneous group.
+    static igfsMisc(igfs, cfg = this.igfsConfigurationBean(igfs)) {
+        cfg.intProperty('blockSize')
+            .intProperty('streamBufferSize')
+            .intProperty('maxSpaceSize')
+            .intProperty('maximumTaskRangeLength')
+            .intProperty('managementPort')
+            .intProperty('perNodeBatchSize')
+            .intProperty('perNodeParallelBatchCount')
+            .intProperty('prefetchBlocks')
+            .intProperty('sequentialReadsBeforePrefetch')
+            .intProperty('trashPurgeTimeout')
+            .intProperty('colocateMetadata')
+            .intProperty('relaxedConsistency')
+            .mapProperty('pathModes', 'pathModes');
+
+        return cfg;
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/generator-optional.js b/modules/web-console/frontend/app/modules/configuration/generator/Custom.service.js
similarity index 82%
rename from modules/web-console/frontend/app/modules/configuration/generator/generator-optional.js
rename to modules/web-console/frontend/app/modules/configuration/generator/Custom.service.js
index 61de1a2d3e7f0..a185485c16fa9 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/generator-optional.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Custom.service.js
@@ -16,10 +16,8 @@
  */
 
 // Optional content generation entry point.
-const $generatorOptional = {};
-
-$generatorOptional.optionalContent = function(zip, cluster) { // eslint-disable-line no-unused-vars
-    // No-op.
-};
-
-export default $generatorOptional;
+export default class IgniteCustomGenerator {
+    optionalContent(zip, cluster) { // eslint-disable-line no-unused-vars
+        // No-op.
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.js
index f9776a29f792e..bcfa2e24a50eb 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.js
@@ -18,7 +18,7 @@
 /**
  * Docker file generation entry point.
  */
-class GeneratorDocker {
+export default class IgniteDockerGenerator {
     /**
      * Generate from section.
      *
@@ -74,5 +74,3 @@ class GeneratorDocker {
         ].join('\n');
     }
 }
-
-export default ['GeneratorDocker', GeneratorDocker];
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/JavaTransformer.service.js b/modules/web-console/frontend/app/modules/configuration/generator/JavaTransformer.service.js
index b123ab5529f39..959077906f055 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/JavaTransformer.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/JavaTransformer.service.js
@@ -158,7 +158,7 @@ const PREDEFINED_QUERIES = [
 ];
 
 // Var name generator function.
-const beenNameSeed = () => {
+const beanNameSeed = () => {
     let idx = '';
     const names = [];
 
@@ -174,1551 +174,1577 @@ const beenNameSeed = () => {
     };
 };
 
-export default ['JavaTypes', 'igniteEventGroups', 'IgniteConfigurationGenerator', (JavaTypes, eventGroups, generator) => {
-    class JavaTransformer extends AbstractTransformer {
-        static generator = generator;
-
-        // Mapping for objects to method call.
-        static METHOD_MAPPING = {
-            'org.apache.ignite.configuration.CacheConfiguration': {
-                id: (ccfg) => JavaTypes.toJavaName('cache', ccfg.findProperty('name').value),
-                args: '',
-                generator: (sb, id, ccfg) => {
-                    const cacheName = ccfg.findProperty('name').value;
-                    const dataSources = JavaTransformer.collectDataSources(ccfg);
-
-                    const javadoc = [
-                        `Create configuration for cache "${cacheName}".`,
-                        '',
-                        '@return Configured cache.'
-                    ];
+export default class IgniteJavaTransformer extends AbstractTransformer {
+    // Mapping for objects to method call.
+    static METHOD_MAPPING = {
+        'org.apache.ignite.configuration.CacheConfiguration': {
+            prefix: 'cache',
+            name: 'name',
+            args: '',
+            generator: (sb, id, ccfg) => {
+                const cacheName = ccfg.findProperty('name').value;
+                const dataSources = IgniteJavaTransformer.collectDataSources(ccfg);
+
+                const javadoc = [
+                    `Create configuration for cache "${cacheName}".`,
+                    '',
+                    '@return Configured cache.'
+                ];
 
-                    if (dataSources.length)
-                        javadoc.push('@throws Exception if failed to create cache configuration.');
+                if (dataSources.length)
+                    javadoc.push('@throws Exception if failed to create cache configuration.');
 
-                    JavaTransformer.commentBlock(sb, ...javadoc);
-                    sb.startBlock(`public static CacheConfiguration ${id}()${dataSources.length ? ' throws Exception' : ''} {`);
+                IgniteJavaTransformer.commentBlock(sb, ...javadoc);
+                sb.startBlock(`public static CacheConfiguration ${id}()${dataSources.length ? ' throws Exception' : ''} {`);
 
-                    JavaTransformer.constructBean(sb, ccfg, [], true);
+                IgniteJavaTransformer.constructBean(sb, ccfg, [], true);
 
-                    sb.emptyLine();
-                    sb.append(`return ${ccfg.id};`);
+                sb.emptyLine();
+                sb.append(`return ${ccfg.id};`);
 
-                    sb.endBlock('}');
+                sb.endBlock('}');
 
-                    return sb;
-                }
-            },
-            'org.apache.ignite.cache.store.jdbc.JdbcType': {
-                id: (type) => JavaTypes.toJavaName('jdbcType', JavaTypes.shortClassName(type.findProperty('valueType').value)),
-                args: 'ccfg.getName()',
-                generator: (sb, name, jdbcType) => {
-                    const javadoc = [
-                        `Create JDBC type for "${name}".`,
-                        '',
-                        '@param cacheName Cache name.',
-                        '@return Configured JDBC type.'
-                    ];
+                return sb;
+            }
+        },
+        'org.apache.ignite.cache.store.jdbc.JdbcType': {
+            prefix: 'jdbcType',
+            name: 'valueType',
+            args: 'ccfg.getName()',
+            generator: (sb, name, jdbcType) => {
+                const javadoc = [
+                    `Create JDBC type for "${name}".`,
+                    '',
+                    '@param cacheName Cache name.',
+                    '@return Configured JDBC type.'
+                ];
 
-                    JavaTransformer.commentBlock(sb, ...javadoc);
-                    sb.startBlock(`private static JdbcType ${name}(String cacheName) {`);
+                IgniteJavaTransformer.commentBlock(sb, ...javadoc);
+                sb.startBlock(`private static JdbcType ${name}(String cacheName) {`);
 
-                    const cacheName = jdbcType.findProperty('cacheName');
+                const cacheName = jdbcType.findProperty('cacheName');
 
-                    cacheName.clsName = 'var';
-                    cacheName.value = 'cacheName';
+                cacheName.clsName = 'var';
+                cacheName.value = 'cacheName';
 
-                    JavaTransformer.constructBean(sb, jdbcType);
+                IgniteJavaTransformer.constructBean(sb, jdbcType);
 
-                    sb.emptyLine();
-                    sb.append(`return ${jdbcType.id};`);
+                sb.emptyLine();
+                sb.append(`return ${jdbcType.id};`);
 
-                    sb.endBlock('}');
+                sb.endBlock('}');
 
-                    return sb;
-                }
+                return sb;
             }
-        };
-
-        // Append comment line.
-        static comment(sb, ...lines) {
-            _.forEach(lines, (line) => sb.append(`// ${line}`));
         }
+    };
 
-        // Append comment block.
-        static commentBlock(sb, ...lines) {
-            if (lines.length === 1)
-                sb.append(`/** ${_.head(lines)} **/`);
-            else {
-                sb.append('/**');
+    // Append comment line.
+    static comment(sb, ...lines) {
+        _.forEach(lines, (line) => sb.append(`// ${line}`));
+    }
 
-                _.forEach(lines, (line) => sb.append(` * ${line}`));
+    // Append comment block.
+    static commentBlock(sb, ...lines) {
+        if (lines.length === 1)
+            sb.append(`/** ${_.head(lines)} **/`);
+        else {
+            sb.append('/**');
 
-                sb.append(' **/');
-            }
+            _.forEach(lines, (line) => sb.append(` * ${line}`));
+
+            sb.append(' **/');
         }
+    }
 
-        /**
-         * @param {Bean} bean
-         */
-        static _newBean(bean) {
-            const shortClsName = JavaTypes.shortClassName(bean.clsName);
-
-            if (_.isEmpty(bean.arguments))
-                return `new ${shortClsName}()`;
-
-            const args = _.map(bean.arguments, (arg) => {
-                switch (arg.clsName) {
-                    case 'MAP':
-                        return arg.id;
-                    case 'BEAN':
-                        return this._newBean(arg.value);
-                    default:
-                        return this._toObject(arg.clsName, arg.value);
-                }
-            });
+    /**
+     * @param {Bean} bean
+     */
+    static _newBean(bean) {
+        const shortClsName = this.javaTypes.shortClassName(bean.clsName);
+
+        if (_.isEmpty(bean.arguments))
+            return `new ${shortClsName}()`;
+
+        const args = _.map(bean.arguments, (arg) => {
+            switch (arg.clsName) {
+                case 'MAP':
+                    return arg.id;
+                case 'BEAN':
+                    return this._newBean(arg.value);
+                default:
+                    return this._toObject(arg.clsName, arg.value);
+            }
+        });
 
-            if (bean.factoryMtd)
-                return `${shortClsName}.${bean.factoryMtd}(${args.join(', ')})`;
+        if (bean.factoryMtd)
+            return `${shortClsName}.${bean.factoryMtd}(${args.join(', ')})`;
 
-            return `new ${shortClsName}(${args.join(', ')})`;
-        }
+        return `new ${shortClsName}(${args.join(', ')})`;
+    }
 
-        /**
-         * @param {StringBuilder} sb
-         * @param {String} parentId
-         * @param {String} propertyName
-         * @param {String} value
-         * @private
-         */
-        static _setProperty(sb, parentId, propertyName, value) {
-            sb.append(`${parentId}.set${_.upperFirst(propertyName)}(${value});`);
-        }
+    /**
+     * @param {StringBuilder} sb
+     * @param {String} parentId
+     * @param {String} propertyName
+     * @param {String} value
+     * @private
+     */
+    static _setProperty(sb, parentId, propertyName, value) {
+        sb.append(`${parentId}.set${_.upperFirst(propertyName)}(${value});`);
+    }
 
-        /**
-         * @param {StringBuilder} sb
-         * @param {Array.<String>} vars
-         * @param {Boolean} limitLines
-         * @param {Bean} bean
-         * @param {String} id
-
-         * @private
-         */
-        static constructBean(sb, bean, vars = [], limitLines = false, id = bean.id) {
-            _.forEach(bean.arguments, (arg) => {
-                switch (arg.clsName) {
-                    case 'MAP':
-                        this._constructMap(sb, arg, vars);
+    /**
+     * @param {StringBuilder} sb
+     * @param {Array.<String>} vars
+     * @param {Boolean} limitLines
+     * @param {Bean} bean
+     * @param {String} id
+
+     * @private
+     */
+    static constructBean(sb, bean, vars = [], limitLines = false, id = bean.id) {
+        _.forEach(bean.arguments, (arg) => {
+            switch (arg.clsName) {
+                case 'MAP':
+                    this._constructMap(sb, arg, vars);
 
-                        sb.emptyLine();
+                    sb.emptyLine();
 
-                        break;
-                    default:
-                        if (this._isBean(arg.clsName) && arg.value.isComplex()) {
-                            this.constructBean(sb, arg.value, vars, limitLines);
+                    break;
+                default:
+                    if (this._isBean(arg.clsName) && arg.value.isComplex()) {
+                        this.constructBean(sb, arg.value, vars, limitLines);
 
-                            sb.emptyLine();
-                        }
-                }
-            });
+                        sb.emptyLine();
+                    }
+            }
+        });
 
-            const clsName = JavaTypes.shortClassName(bean.clsName);
+        const clsName = this.javaTypes.shortClassName(bean.clsName);
 
-            sb.append(`${this.varInit(clsName, id, vars)} = ${this._newBean(bean)};`);
+        sb.append(`${this.varInit(clsName, id, vars)} = ${this._newBean(bean)};`);
 
-            if (_.nonEmpty(bean.properties)) {
-                sb.emptyLine();
+        if (_.nonEmpty(bean.properties)) {
+            sb.emptyLine();
 
-                this._setProperties(sb, bean, vars, limitLines, id);
-            }
+            this._setProperties(sb, bean, vars, limitLines, id);
         }
+    }
 
-        /**
-         * @param {StringBuilder} sb
-         * @param {Bean} bean
-         * @param {Array.<String>} vars
-         * @param {Boolean} limitLines
-         * @private
-         */
-        static constructStoreFactory(sb, bean, vars, limitLines = false) {
-            const shortClsName = JavaTypes.shortClassName(bean.clsName);
-
-            if (_.includes(vars, bean.id))
-                sb.append(`${bean.id} = ${this._newBean(bean)};`);
-            else {
-                vars.push(bean.id);
-
-                sb.append(`${shortClsName} ${bean.id} = ${this._newBean(bean)};`);
-            }
-
-            sb.emptyLine();
+    /**
+     * @param {StringBuilder} sb
+     * @param {Bean} bean
+     * @param {Array.<String>} vars
+     * @param {Boolean} limitLines
+     * @private
+     */
+    static constructStoreFactory(sb, bean, vars, limitLines = false) {
+        const shortClsName = this.javaTypes.shortClassName(bean.clsName);
+
+        if (_.includes(vars, bean.id))
+            sb.append(`${bean.id} = ${this._newBean(bean)};`);
+        else {
+            vars.push(bean.id);
+
+            sb.append(`${shortClsName} ${bean.id} = ${this._newBean(bean)};`);
+        }
 
-            sb.startBlock(`${bean.id}.setDataSourceFactory(new Factory<DataSource>() {`);
-            this.commentBlock(sb, '{@inheritDoc}');
-            sb.startBlock('@Override public DataSource create() {');
+        sb.emptyLine();
 
-            sb.append(`return DataSources.INSTANCE_${bean.findProperty('dataSourceBean').id};`);
+        sb.startBlock(`${bean.id}.setDataSourceFactory(new Factory<DataSource>() {`);
+        this.commentBlock(sb, '{@inheritDoc}');
+        sb.startBlock('@Override public DataSource create() {');
 
-            sb.endBlock('};');
-            sb.endBlock('});');
+        sb.append(`return DataSources.INSTANCE_${bean.findProperty('dataSourceBean').id};`);
 
-            const storeFactory = _.cloneDeep(bean);
+        sb.endBlock('};');
+        sb.endBlock('});');
 
-            _.remove(storeFactory.properties, (p) => _.includes(['dataSourceBean'], p.name));
+        const storeFactory = _.cloneDeep(bean);
 
-            if (storeFactory.properties.length) {
-                sb.emptyLine();
+        _.remove(storeFactory.properties, (p) => _.includes(['dataSourceBean'], p.name));
 
-                this._setProperties(sb, storeFactory, vars, limitLines);
-            }
-        }
+        if (storeFactory.properties.length) {
+            sb.emptyLine();
 
-        static _isBean(clsName) {
-            return JavaTypes.nonBuiltInClass(clsName) && JavaTypes.nonEnum(clsName) && _.includes(clsName, '.');
+            this._setProperties(sb, storeFactory, vars, limitLines);
         }
+    }
 
-        static _toObject(clsName, val) {
-            const items = _.isArray(val) ? val : [val];
+    static _isBean(clsName) {
+        return this.javaTypes.nonBuiltInClass(clsName) && this.javaTypes.nonEnum(clsName) && _.includes(clsName, '.');
+    }
 
-            return _.map(items, (item) => {
-                if (_.isNil(item))
-                    return 'null';
+    static _toObject(clsName, val) {
+        const items = _.isArray(val) ? val : [val];
+
+        return _.map(items, (item) => {
+            if (_.isNil(item))
+                return 'null';
+
+            switch (clsName) {
+                case 'var':
+                    return item;
+                case 'byte':
+                    return `(byte) ${item}`;
+                case 'float':
+                    return `${item}f`;
+                case 'long':
+                    return `${item}L`;
+                case 'java.io.Serializable':
+                case 'java.lang.String':
+                    return `"${item.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
+                case 'PATH':
+                    return `"${item.replace(/\\/g, '\\\\')}"`;
+                case 'java.lang.Class':
+                    return `${this.javaTypes.shortClassName(item)}.class`;
+                case 'java.util.UUID':
+                    return `UUID.fromString("${item}")`;
+                case 'PROPERTY':
+                    return `props.getProperty("${item}")`;
+                case 'PROPERTY_CHAR':
+                    return `props.getProperty("${item}").toCharArray()`;
+                case 'PROPERTY_INT':
+                    return `Integer.parseInt(props.getProperty("${item}"))`;
+                default:
+                    if (this._isBean(clsName)) {
+                        if (item.isComplex())
+                            return item.id;
+
+                        return this._newBean(item);
+                    }
 
-                switch (clsName) {
-                    case 'var':
+                    if (this.javaTypes.nonEnum(clsName))
                         return item;
-                    case 'byte':
-                        return `(byte) ${item}`;
-                    case 'float':
-                        return `${item}f`;
-                    case 'long':
-                        return `${item}L`;
-                    case 'java.io.Serializable':
-                    case 'java.lang.String':
-                        return `"${item.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
-                    case 'PATH':
-                        return `"${item.replace(/\\/g, '\\\\')}"`;
-                    case 'java.lang.Class':
-                        return `${JavaTypes.shortClassName(item)}.class`;
-                    case 'java.util.UUID':
-                        return `UUID.fromString("${item}")`;
-                    case 'PROPERTY':
-                        return `props.getProperty("${item}")`;
-                    case 'PROPERTY_CHAR':
-                        return `props.getProperty("${item}").toCharArray()`;
-                    case 'PROPERTY_INT':
-                        return `Integer.parseInt(props.getProperty("${item}"))`;
-                    default:
-                        if (this._isBean(clsName)) {
-                            if (item.isComplex())
-                                return item.id;
-
-                            return this._newBean(item);
-                        }
 
-                        if (JavaTypes.nonEnum(clsName))
-                            return item;
-
-                        return `${JavaTypes.shortClassName(clsName)}.${item}`;
-                }
-            });
-        }
+                    return `${this.javaTypes.shortClassName(clsName)}.${item}`;
+            }
+        });
+    }
 
-        static _constructBeans(sb, type, items, vars, limitLines) {
-            if (this._isBean(type)) {
-                // Construct objects inline for preview or simple objects.
-                const mapper = this.METHOD_MAPPING[type];
+    static _mapperId(mapper) {
+        return (item) => this.javaTypes.toJavaName(mapper.prefix, item.findProperty(mapper.name).value);
+    }
 
-                const nextId = mapper ? mapper.id : beenNameSeed();
+    static _constructBeans(sb, type, items, vars, limitLines) {
+        if (this._isBean(type)) {
+            // Construct objects inline for preview or simple objects.
+            const mapper = this.METHOD_MAPPING[type];
 
-                // Prepare objects refs.
-                return _.map(items, (item) => {
-                    if (limitLines && mapper)
-                        return mapper.id(item) + (limitLines ? `(${mapper.args})` : '');
+            const nextId = mapper ? this._mapperId(mapper) : beanNameSeed();
 
-                    if (item.isComplex()) {
-                        const id = nextId(item);
+            // Prepare objects refs.
+            return _.map(items, (item) => {
+                if (limitLines && mapper)
+                    return nextId(item) + (limitLines ? `(${mapper.args})` : '');
 
-                        this.constructBean(sb, item, vars, limitLines, id);
+                if (item.isComplex()) {
+                    const id = nextId(item);
 
-                        sb.emptyLine();
+                    this.constructBean(sb, item, vars, limitLines, id);
 
-                        return id;
-                    }
+                    sb.emptyLine();
 
-                    return this._newBean(item);
-                });
-            }
+                    return id;
+                }
 
-            return this._toObject(type, items);
+                return this._newBean(item);
+            });
         }
 
-        /**
-         *
-         * @param sb
-         * @param parentId
-         * @param arrProp
-         * @param vars
-         * @param limitLines
-         * @private
-         */
-        static _setVarArg(sb, parentId, arrProp, vars, limitLines) {
-            const refs = this._constructBeans(sb, arrProp.typeClsName, arrProp.items, vars, limitLines);
-
-            // Set refs to property.
-            if (refs.length === 1)
-                this._setProperty(sb, parentId, arrProp.name, _.head(refs));
-            else {
-                sb.startBlock(`${parentId}.set${_.upperFirst(arrProp.name)}(`);
-
-                const lastIdx = refs.length - 1;
-
-                _.forEach(refs, (ref, idx) => {
-                    sb.append(ref + (lastIdx !== idx ? ',' : ''));
-                });
+        return this._toObject(type, items);
+    }
 
-                sb.endBlock(');');
-            }
+    /**
+     *
+     * @param sb
+     * @param parentId
+     * @param arrProp
+     * @param vars
+     * @param limitLines
+     * @private
+     */
+    static _setVarArg(sb, parentId, arrProp, vars, limitLines) {
+        const refs = this._constructBeans(sb, arrProp.typeClsName, arrProp.items, vars, limitLines);
+
+        // Set refs to property.
+        if (refs.length === 1)
+            this._setProperty(sb, parentId, arrProp.name, _.head(refs));
+        else {
+            sb.startBlock(`${parentId}.set${_.upperFirst(arrProp.name)}(`);
+
+            const lastIdx = refs.length - 1;
+
+            _.forEach(refs, (ref, idx) => {
+                sb.append(ref + (lastIdx !== idx ? ',' : ''));
+            });
+
+            sb.endBlock(');');
         }
+    }
 
-        /**
-         *
-         * @param sb
-         * @param parentId
-         * @param arrProp
-         * @param vars
-         * @param limitLines
-         * @private
-         */
-        static _setArray(sb, parentId, arrProp, vars, limitLines) {
-            const refs = this._constructBeans(sb, arrProp.typeClsName, arrProp.items, vars, limitLines);
+    /**
+     *
+     * @param sb
+     * @param parentId
+     * @param arrProp
+     * @param vars
+     * @param limitLines
+     * @private
+     */
+    static _setArray(sb, parentId, arrProp, vars, limitLines) {
+        const refs = this._constructBeans(sb, arrProp.typeClsName, arrProp.items, vars, limitLines);
 
-            const arrType = JavaTypes.shortClassName(arrProp.typeClsName);
+        const arrType = this.javaTypes.shortClassName(arrProp.typeClsName);
 
-            // Set refs to property.
-            sb.startBlock(`${parentId}.set${_.upperFirst(arrProp.name)}(new ${arrType}[] {`);
+        // Set refs to property.
+        sb.startBlock(`${parentId}.set${_.upperFirst(arrProp.name)}(new ${arrType}[] {`);
 
-            const lastIdx = refs.length - 1;
+        const lastIdx = refs.length - 1;
 
-            _.forEach(refs, (ref, idx) => sb.append(ref + (lastIdx !== idx ? ',' : '')));
+        _.forEach(refs, (ref, idx) => sb.append(ref + (lastIdx !== idx ? ',' : '')));
 
-            sb.endBlock('});');
-        }
+        sb.endBlock('});');
+    }
 
-        static _constructMap(sb, map, vars = []) {
-            const keyClsName = JavaTypes.shortClassName(map.keyClsName);
-            const valClsName = JavaTypes.shortClassName(map.valClsName);
+    static _constructMap(sb, map, vars = []) {
+        const keyClsName = this.javaTypes.shortClassName(map.keyClsName);
+        const valClsName = this.javaTypes.shortClassName(map.valClsName);
 
-            const mapClsName = map.ordered ? 'LinkedHashMap' : 'HashMap';
+        const mapClsName = map.ordered ? 'LinkedHashMap' : 'HashMap';
 
-            const type = `${mapClsName}<${keyClsName}, ${valClsName}>`;
+        const type = `${mapClsName}<${keyClsName}, ${valClsName}>`;
 
-            sb.append(`${this.varInit(type, map.id, vars)} = new ${mapClsName}<>();`);
+        sb.append(`${this.varInit(type, map.id, vars)} = new ${mapClsName}<>();`);
 
-            sb.emptyLine();
+        sb.emptyLine();
 
-            _.forEach(map.entries, (entry) => {
-                const key = this._toObject(map.keyClsName, entry[map.keyField]);
-                const val = entry[map.valField];
+        _.forEach(map.entries, (entry) => {
+            const key = this._toObject(map.keyClsName, entry[map.keyField]);
+            const val = entry[map.valField];
 
-                if (_.isArray(val) && map.valClsName === 'java.lang.String') {
-                    if (val.length > 1) {
-                        sb.startBlock(`${map.id}.put(${key},`);
+            if (_.isArray(val) && map.valClsName === 'java.lang.String') {
+                if (val.length > 1) {
+                    sb.startBlock(`${map.id}.put(${key},`);
 
-                        _.forEach(val, (line, idx) => {
-                            sb.append(`"${line}"${idx !== val.length - 1 ? ' +' : ''}`);
-                        });
+                    _.forEach(val, (line, idx) => {
+                        sb.append(`"${line}"${idx !== val.length - 1 ? ' +' : ''}`);
+                    });
 
-                        sb.endBlock(');');
-                    }
-                    else
-                        sb.append(`${map.id}.put(${key}, ${this._toObject(map.valClsName, _.head(val))});`);
+                    sb.endBlock(');');
                 }
                 else
-                    sb.append(`${map.id}.put(${key}, ${this._toObject(map.valClsName, val)});`);
-            });
-        }
+                    sb.append(`${map.id}.put(${key}, ${this._toObject(map.valClsName, _.head(val))});`);
+            }
+            else
+                sb.append(`${map.id}.put(${key}, ${this._toObject(map.valClsName, val)});`);
+        });
+    }
 
-        static varInit(type, id, vars) {
-            if (_.includes(vars, id))
-                return id;
+    static varInit(type, id, vars) {
+        if (_.includes(vars, id))
+            return id;
 
-            vars.push(id);
+        vars.push(id);
 
-            return `${type} ${id}`;
-        }
+        return `${type} ${id}`;
+    }
 
-        /**
-         *
-         * @param {StringBuilder} sb
-         * @param {Bean} bean
-         * @param {String} id
-         * @param {Array.<String>} vars
-         * @param {Boolean} limitLines
-         * @returns {StringBuilder}
-         */
-        static _setProperties(sb = new StringBuilder(), bean, vars = [], limitLines = false, id = bean.id) {
-            _.forEach(bean.properties, (prop, idx) => {
-                switch (prop.clsName) {
-                    case 'DATA_SOURCE':
-                        this._setProperty(sb, id, 'dataSource', `DataSources.INSTANCE_${prop.id}`);
-
-                        break;
-                    case 'EVENT_TYPES':
-                        if (prop.eventTypes.length === 1)
-                            this._setProperty(sb, id, prop.name, _.head(prop.eventTypes));
-                        else {
-                            sb.append(`int[] ${prop.id} = new int[${_.head(prop.eventTypes)}.length`);
+    /**
+     *
+     * @param {StringBuilder} sb
+     * @param {Bean} bean
+     * @param {String} id
+     * @param {Array.<String>} vars
+     * @param {Boolean} limitLines
+     * @returns {StringBuilder}
+     */
+    static _setProperties(sb = new StringBuilder(), bean, vars = [], limitLines = false, id = bean.id) {
+        _.forEach(bean.properties, (prop, idx) => {
+            switch (prop.clsName) {
+                case 'DATA_SOURCE':
+                    this._setProperty(sb, id, 'dataSource', `DataSources.INSTANCE_${prop.id}`);
+
+                    break;
+                case 'EVENT_TYPES':
+                    if (prop.eventTypes.length === 1)
+                        this._setProperty(sb, id, prop.name, _.head(prop.eventTypes));
+                    else {
+                        sb.append(`int[] ${prop.id} = new int[${_.head(prop.eventTypes)}.length`);
+
+                        _.forEach(_.tail(prop.eventTypes), (evtGrp) => {
+                            sb.append(`    + ${evtGrp}.length`);
+                        });
 
-                            _.forEach(_.tail(prop.eventTypes), (evtGrp) => {
-                                sb.append(`    + ${evtGrp}.length`);
-                            });
+                        sb.append('];');
 
-                            sb.append('];');
+                        sb.emptyLine();
 
+                        sb.append('int k = 0;');
+
+                        _.forEach(prop.eventTypes, (evtGrp, evtIdx) => {
                             sb.emptyLine();
 
-                            sb.append('int k = 0;');
+                            sb.append(`System.arraycopy(${evtGrp}, 0, ${prop.id}, k, ${evtGrp}.length);`);
 
-                            _.forEach(prop.eventTypes, (evtGrp, evtIdx) => {
-                                sb.emptyLine();
+                            if (evtIdx < prop.eventTypes.length - 1)
+                                sb.append(`k += ${evtGrp}.length;`);
+                        });
 
-                                sb.append(`System.arraycopy(${evtGrp}, 0, ${prop.id}, k, ${evtGrp}.length);`);
+                        sb.emptyLine();
 
-                                if (evtIdx < prop.eventTypes.length - 1)
-                                    sb.append(`k += ${evtGrp}.length;`);
-                            });
+                        sb.append(`cfg.setIncludeEventTypes(${prop.id});`);
+                    }
 
-                            sb.emptyLine();
+                    break;
+                case 'ARRAY':
+                    if (prop.varArg)
+                        this._setVarArg(sb, id, prop, vars, limitLines);
+                    else
+                        this._setArray(sb, id, prop, vars, limitLines);
 
-                            sb.append(`cfg.setIncludeEventTypes(${prop.id});`);
-                        }
+                    break;
+                case 'COLLECTION':
+                    const nonBean = !this._isBean(prop.typeClsName);
 
-                        break;
-                    case 'ARRAY':
-                        if (prop.varArg)
-                            this._setVarArg(sb, id, prop, vars, limitLines);
-                        else
-                            this._setArray(sb, id, prop, vars, limitLines);
+                    if (nonBean && prop.implClsName === 'java.util.ArrayList') {
+                        const items = _.map(prop.items, (item) => this._toObject(prop.typeClsName, item));
 
-                        break;
-                    case 'COLLECTION':
-                        const nonBean = !this._isBean(prop.typeClsName);
+                        if (items.length > 1) {
+                            sb.startBlock(`${id}.set${_.upperFirst(prop.name)}(Arrays.asList(`);
 
-                        if (nonBean && prop.implClsName === 'java.util.ArrayList') {
-                            const items = _.map(prop.items, (item) => this._toObject(prop.typeClsName, item));
+                            _.forEach(items, (item, i) => sb.append(item + (i !== items.length - 1 ? ',' : '')));
 
-                            if (items.length > 1) {
-                                sb.startBlock(`${id}.set${_.upperFirst(prop.name)}(Arrays.asList(`);
+                            sb.endBlock('));');
+                        }
+                        else
+                            this._setProperty(sb, id, prop.name, `Arrays.asList(${items})`);
+                    }
+                    else {
+                        const colTypeClsName = this.javaTypes.shortClassName(prop.typeClsName);
+                        const implClsName = this.javaTypes.shortClassName(prop.implClsName);
 
-                                _.forEach(items, (item, i) => sb.append(item + (i !== items.length - 1 ? ',' : '')));
+                        sb.append(`${this.varInit(`${implClsName}<${colTypeClsName}>`, prop.id, vars)} = new ${implClsName}<>();`);
 
-                                sb.endBlock('));');
-                            }
-                            else
-                                this._setProperty(sb, id, prop.name, `Arrays.asList(${items})`);
+                        sb.emptyLine();
+
+                        if (nonBean) {
+                            _.forEach(this._toObject(colTypeClsName, prop.items), (item) => {
+                                sb.append(`${prop.id}.add("${item}");`);
+
+                                sb.emptyLine();
+                            });
                         }
                         else {
-                            const colTypeClsName = JavaTypes.shortClassName(prop.typeClsName);
-                            const implClsName = JavaTypes.shortClassName(prop.implClsName);
+                            _.forEach(prop.items, (item) => {
+                                this.constructBean(sb, item, vars, limitLines);
 
-                            sb.append(`${this.varInit(`${implClsName}<${colTypeClsName}>`, prop.id, vars)} = new ${implClsName}<>();`);
+                                sb.append(`${prop.id}.add(${item.id});`);
 
-                            sb.emptyLine();
+                                sb.emptyLine();
+                            });
+                        }
 
-                            if (nonBean) {
-                                _.forEach(this._toObject(colTypeClsName, prop.items), (item) => {
-                                    sb.append(`${prop.id}.add("${item}");`);
+                        this._setProperty(sb, id, prop.name, prop.id);
+                    }
 
-                                    sb.emptyLine();
-                                });
-                            }
-                            else {
-                                _.forEach(prop.items, (item) => {
-                                    this.constructBean(sb, item, vars, limitLines);
+                    break;
+                case 'MAP':
+                    this._constructMap(sb, prop, vars);
 
-                                    sb.append(`${prop.id}.add(${item.id});`);
+                    if (_.nonEmpty(prop.entries))
+                        sb.emptyLine();
 
-                                    sb.emptyLine();
-                                });
-                            }
+                    this._setProperty(sb, id, prop.name, prop.id);
 
-                            this._setProperty(sb, id, prop.name, prop.id);
-                        }
+                    break;
+                case 'java.util.Properties':
+                    sb.append(`${this.varInit('Properties', prop.id, vars)} = new Properties();`);
 
-                        break;
-                    case 'MAP':
-                        this._constructMap(sb, prop, vars);
+                    if (_.nonEmpty(prop.entries))
+                        sb.emptyLine();
 
-                        if (_.nonEmpty(prop.entries))
-                            sb.emptyLine();
+                    _.forEach(prop.entries, (entry) => {
+                        const key = this._toObject('java.lang.String', entry.name);
+                        const val = this._toObject('java.lang.String', entry.value);
 
-                        this._setProperty(sb, id, prop.name, prop.id);
+                        sb.append(`${prop.id}.setProperty(${key}, ${val});`);
+                    });
 
-                        break;
-                    case 'java.util.Properties':
-                        sb.append(`${this.varInit('Properties', prop.id, vars)} = new Properties();`);
+                    sb.emptyLine();
 
-                        if (_.nonEmpty(prop.entries))
-                            sb.emptyLine();
+                    this._setProperty(sb, id, prop.name, prop.id);
 
-                        _.forEach(prop.entries, (entry) => {
-                            const key = this._toObject('java.lang.String', entry.name);
-                            const val = this._toObject('java.lang.String', entry.value);
+                    break;
+                case 'BEAN':
+                    const embedded = prop.value;
 
-                            sb.append(`${prop.id}.setProperty(${key}, ${val});`);
-                        });
+                    if (_.includes(STORE_FACTORY, embedded.clsName)) {
+                        this.constructStoreFactory(sb, embedded, vars, limitLines);
 
                         sb.emptyLine();
 
-                        this._setProperty(sb, id, prop.name, prop.id);
+                        this._setProperty(sb, id, prop.name, embedded.id);
+                    }
+                    else if (embedded.isComplex()) {
+                        this.constructBean(sb, embedded, vars, limitLines);
 
-                        break;
-                    case 'BEAN':
-                        const embedded = prop.value;
+                        sb.emptyLine();
 
-                        if (_.includes(STORE_FACTORY, embedded.clsName)) {
-                            this.constructStoreFactory(sb, embedded, vars, limitLines);
+                        this._setProperty(sb, id, prop.name, embedded.id);
+                    }
+                    else
+                        this._setProperty(sb, id, prop.name, this._newBean(embedded));
 
-                            sb.emptyLine();
+                    break;
+                default:
+                    this._setProperty(sb, id, prop.name, this._toObject(prop.clsName, prop.value));
+            }
 
-                            this._setProperty(sb, id, prop.name, embedded.id);
-                        }
-                        else if (embedded.isComplex()) {
-                            this.constructBean(sb, embedded, vars, limitLines);
+            this._emptyLineIfNeeded(sb, bean.properties, idx);
+        });
 
-                            sb.emptyLine();
+        return sb;
+    }
 
-                            this._setProperty(sb, id, prop.name, embedded.id);
-                        }
-                        else
-                            this._setProperty(sb, id, prop.name, this._newBean(embedded));
+    static _collectMapImports(prop) {
+        const imports = [];
 
-                        break;
-                    default:
-                        this._setProperty(sb, id, prop.name, this._toObject(prop.clsName, prop.value));
-                }
+        imports.push(prop.ordered ? 'java.util.LinkedHashMap' : 'java.util.HashMap');
+        imports.push(prop.keyClsName);
+        imports.push(prop.valClsName);
 
-                this._emptyLineIfNeeded(sb, bean.properties, idx);
-            });
+        return imports;
+    }
 
-            return sb;
-        }
+    static collectBeanImports(bean) {
+        const imports = [bean.clsName];
 
-        static collectBeanImports(bean) {
-            const imports = [bean.clsName];
+        _.forEach(bean.arguments, (arg) => {
+            switch (arg.clsName) {
+                case 'BEAN':
+                    imports.push(...this.collectPropertiesImports(arg.value.properties));
 
-            _.forEach(bean.arguments, (arg) => {
-                switch (arg.clsName) {
-                    case 'BEAN':
-                        imports.push(...this.collectPropertiesImports(arg.value.properties));
+                    break;
+                case 'java.lang.Class':
+                    imports.push(this.javaTypes.fullClassName(arg.value));
 
-                        break;
-                    case 'java.lang.Class':
-                        imports.push(JavaTypes.fullClassName(arg.value));
+                    break;
 
-                        break;
-                    default:
-                        imports.push(arg.clsName);
-                }
-            });
+                case 'MAP':
+                    imports.push(...this._collectMapImports(arg));
 
-            imports.push(...this.collectPropertiesImports(bean.properties));
+                    break;
+                default:
+                    imports.push(arg.clsName);
+            }
+        });
 
-            if (_.includes(STORE_FACTORY, bean.clsName))
-                imports.push('javax.sql.DataSource', 'javax.cache.configuration.Factory');
+        imports.push(...this.collectPropertiesImports(bean.properties));
 
-            return imports;
-        }
+        if (_.includes(STORE_FACTORY, bean.clsName))
+            imports.push('javax.sql.DataSource', 'javax.cache.configuration.Factory');
 
-        /**
-         * @param {Array.<Object>} props
-         * @returns {Array.<String>}
-         */
-        static collectPropertiesImports(props) {
-            const imports = [];
+        return imports;
+    }
 
-            _.forEach(props, (prop) => {
-                switch (prop.clsName) {
-                    case 'DATA_SOURCE':
-                        imports.push(prop.value.clsName);
+    /**
+     * @param {Array.<Object>} props
+     * @returns {Array.<String>}
+     */
+    static collectPropertiesImports(props) {
+        const imports = [];
 
-                        break;
-                    case 'PROPERTY':
-                    case 'PROPERTY_CHAR':
-                    case 'PROPERTY_INT':
-                        imports.push('java.io.InputStream', 'java.util.Properties');
+        _.forEach(props, (prop) => {
+            switch (prop.clsName) {
+                case 'DATA_SOURCE':
+                    imports.push(prop.value.clsName);
 
-                        break;
-                    case 'BEAN':
-                        imports.push(...this.collectBeanImports(prop.value));
+                    break;
+                case 'PROPERTY':
+                case 'PROPERTY_CHAR':
+                case 'PROPERTY_INT':
+                    imports.push('java.io.InputStream', 'java.util.Properties');
 
-                        break;
-                    case 'ARRAY':
-                        imports.push(prop.typeClsName);
+                    break;
+                case 'BEAN':
+                    imports.push(...this.collectBeanImports(prop.value));
 
-                        if (this._isBean(prop.typeClsName))
-                            _.forEach(prop.items, (item) => imports.push(...this.collectBeanImports(item)));
+                    break;
+                case 'ARRAY':
+                    imports.push(prop.typeClsName);
 
-                        break;
-                    case 'COLLECTION':
-                        imports.push(prop.typeClsName);
+                    if (this._isBean(prop.typeClsName))
+                        _.forEach(prop.items, (item) => imports.push(...this.collectBeanImports(item)));
 
-                        if (this._isBean(prop.typeClsName)) {
-                            _.forEach(prop.items, (item) => imports.push(...this.collectBeanImports(item)));
+                    break;
+                case 'COLLECTION':
+                    imports.push(prop.typeClsName);
 
-                            imports.push(prop.implClsName);
-                        }
-                        else if (prop.implClsName === 'java.util.ArrayList')
-                            imports.push('java.util.Arrays');
-                        else
-                            imports.push(prop.implClsName);
-
-                        break;
-                    case 'MAP':
-                        imports.push(prop.ordered ? 'java.util.LinkedHashMap' : 'java.util.HashMap');
-                        imports.push(prop.keyClsName);
-                        imports.push(prop.valClsName);
-
-                        break;
-                    default:
-                        if (!JavaTypes.nonEnum(prop.clsName))
-                            imports.push(prop.clsName);
-                }
-            });
+                    if (this._isBean(prop.typeClsName)) {
+                        _.forEach(prop.items, (item) => imports.push(...this.collectBeanImports(item)));
 
-            return imports;
-        }
+                        imports.push(prop.implClsName);
+                    }
+                    else if (prop.implClsName === 'java.util.ArrayList')
+                        imports.push('java.util.Arrays');
+                    else
+                        imports.push(prop.implClsName);
 
-        static _prepareImports(imports) {
-            return _.sortedUniq(_.sortBy(_.filter(imports, (cls) => !cls.startsWith('java.lang.') && _.includes(cls, '.'))));
-        }
+                    break;
+                case 'MAP':
+                    imports.push(...this._collectMapImports(prop));
 
-        /**
-         * @param {Bean} bean
-         * @returns {Array.<String>}
-         */
-        static collectStaticImports(bean) {
-            const imports = [];
+                    break;
+                default:
+                    if (!this.javaTypes.nonEnum(prop.clsName))
+                        imports.push(prop.clsName);
+            }
+        });
 
-            _.forEach(bean.properties, (prop) => {
-                switch (prop.clsName) {
-                    case 'EVENT_TYPES':
-                        _.forEach(prop.eventTypes, (value) => {
-                            const evtGrp = _.find(eventGroups, {value});
+        return imports;
+    }
 
-                            imports.push(`${evtGrp.class}.${evtGrp.value}`);
-                        });
+    static _prepareImports(imports) {
+        return _.sortedUniq(_.sortBy(_.filter(imports, (cls) => !cls.startsWith('java.lang.') && _.includes(cls, '.'))));
+    }
 
-                        break;
-                    default:
-                        // No-op.
-                }
-            });
+    /**
+     * @param {Bean} bean
+     * @returns {Array.<String>}
+     */
+    static collectStaticImports(bean) {
+        const imports = [];
 
-            return imports;
-        }
+        _.forEach(bean.properties, (prop) => {
+            switch (prop.clsName) {
+                case 'EVENT_TYPES':
+                    _.forEach(prop.eventTypes, (value) => {
+                        const evtGrp = _.find(this.eventGroups, {value});
 
-        /**
-         * @param {Bean} bean
-         * @returns {Object}
-         */
-        static collectBeansWithMapping(bean) {
-            const beans = {};
+                        imports.push(`${evtGrp.class}.${evtGrp.value}`);
+                    });
 
-            _.forEach(bean.properties, (prop) => {
-                switch (prop.clsName) {
-                    case 'BEAN':
-                        _.merge(beans, this.collectBeansWithMapping(prop.value));
+                    break;
+                default:
+                    // No-op.
+            }
+        });
 
-                        break;
-                    case 'ARRAY':
-                        if (this._isBean(prop.typeClsName)) {
-                            const mapping = this.METHOD_MAPPING[prop.typeClsName];
+        return imports;
+    }
 
-                            _.reduce(prop.items, (acc, item) => {
-                                if (mapping) {
-                                    acc[mapping.id(item)] = item;
+    /**
+     * @param {Bean} bean
+     * @returns {Object}
+     */
+    static collectBeansWithMapping(bean) {
+        const beans = {};
 
-                                    _.merge(acc, this.collectBeansWithMapping(item));
-                                }
-                                return acc;
-                            }, beans);
-                        }
+        _.forEach(bean.properties, (prop) => {
+            switch (prop.clsName) {
+                case 'BEAN':
+                    _.merge(beans, this.collectBeansWithMapping(prop.value));
 
-                        break;
-                    default:
-                        // No-op.
-                }
-            });
+                    break;
+                case 'ARRAY':
+                    if (this._isBean(prop.typeClsName)) {
+                        const mapper = this.METHOD_MAPPING[prop.typeClsName];
 
-            return beans;
-        }
+                        const mapperId = mapper ? this._mapperId(mapper) : null;
 
-        /**
-         * Build Java startup class with configuration.
-         *
-         * @param {Bean} cfg
-         * @param pkg Package name.
-         * @param {String} clsName Class name for generate factory class otherwise generate code snippet.
-         * @param {Array.<Object>} clientNearCaches Is client node.
-         * @returns {StringBuilder}
-         */
-        static igniteConfiguration(cfg, pkg, clsName, clientNearCaches) {
-            const sb = new StringBuilder();
-
-            sb.append(`package ${pkg};`);
-            sb.emptyLine();
+                        _.reduce(prop.items, (acc, item) => {
+                            if (mapperId)
+                                acc[mapperId(item)] = item;
 
-            const imports = this.collectBeanImports(cfg);
+                            _.merge(acc, this.collectBeansWithMapping(item));
 
-            if (_.nonEmpty(clientNearCaches))
-                imports.push('org.apache.ignite.configuration.NearCacheConfiguration');
+                            return acc;
+                        }, beans);
+                    }
 
-            if (_.includes(imports, 'oracle.jdbc.pool.OracleDataSource'))
-                imports.push('java.sql.SQLException');
+                    break;
+                default:
+                    // No-op.
+            }
+        });
 
-            const hasProps = this.hasProperties(cfg);
+        return beans;
+    }
 
-            if (hasProps)
-                imports.push('java.util.Properties', 'java.io.InputStream');
+    /**
+     * Build Java startup class with configuration.
+     *
+     * @param {Bean} cfg
+     * @param pkg Package name.
+     * @param {String} clsName Class name for generate factory class otherwise generate code snippet.
+     * @param {Array.<Object>} clientNearCaches Is client node.
+     * @returns {StringBuilder}
+     */
+    static igniteConfiguration(cfg, pkg, clsName, clientNearCaches) {
+        const sb = new StringBuilder();
 
-            _.forEach(this._prepareImports(imports), (cls) => sb.append(`import ${cls};`));
+        sb.append(`package ${pkg};`);
+        sb.emptyLine();
 
-            sb.emptyLine();
+        const imports = this.collectBeanImports(cfg);
 
-            const staticImports = this._prepareImports(this.collectStaticImports(cfg));
+        const nearCacheBeans = [];
 
-            if (staticImports.length) {
-                _.forEach(this._prepareImports(staticImports), (cls) => sb.append(`import static ${cls};`));
+        if (_.nonEmpty(clientNearCaches)) {
+            imports.push('org.apache.ignite.configuration.NearCacheConfiguration');
 
-                sb.emptyLine();
-            }
+            _.forEach(clientNearCaches, (cache) => {
+                const nearCacheBean = this.generator.cacheNearClient(cache);
 
-            this.mainComment(sb);
-            sb.startBlock(`public class ${clsName} {`);
+                nearCacheBean.cacheName = cache.name;
 
-            // 2. Add external property file
-            if (hasProps) {
-                this.commentBlock(sb, 'Secret properties loading.');
-                sb.append('private static final Properties props = new Properties();');
-                sb.emptyLine();
-                sb.startBlock('static {');
-                sb.startBlock('try (InputStream in = IgniteConfiguration.class.getClassLoader().getResourceAsStream("secret.properties")) {');
-                sb.append('props.load(in);');
-                sb.endBlock('}');
-                sb.startBlock('catch (Exception ignored) {');
-                sb.append('// No-op.');
-                sb.endBlock('}');
-                sb.endBlock('}');
-                sb.emptyLine();
-            }
+                imports.push(...this.collectBeanImports(nearCacheBean));
 
-            // 3. Add data sources.
-            const dataSources = this.collectDataSources(cfg);
+                nearCacheBeans.push(nearCacheBean);
+            });
+        }
 
-            if (dataSources.length) {
-                this.commentBlock(sb, 'Helper class for datasource creation.');
-                sb.startBlock('public static class DataSources {');
+        if (_.includes(imports, 'oracle.jdbc.pool.OracleDataSource'))
+            imports.push('java.sql.SQLException');
 
-                _.forEach(dataSources, (ds, idx) => {
-                    const dsClsName = JavaTypes.shortClassName(ds.clsName);
+        const hasProps = this.hasProperties(cfg);
 
-                    if (idx !== 0)
-                        sb.emptyLine();
+        if (hasProps)
+            imports.push('java.util.Properties', 'java.io.InputStream');
 
-                    sb.append(`public static final ${dsClsName} INSTANCE_${ds.id} = create${ds.id}();`);
-                    sb.emptyLine();
+        _.forEach(this._prepareImports(imports), (cls) => sb.append(`import ${cls};`));
 
-                    sb.startBlock(`private static ${dsClsName} create${ds.id}() {`);
+        sb.emptyLine();
 
-                    if (dsClsName === 'OracleDataSource')
-                        sb.startBlock('try {');
+        const staticImports = this._prepareImports(this.collectStaticImports(cfg));
 
-                    this.constructBean(sb, ds);
+        if (staticImports.length) {
+            _.forEach(this._prepareImports(staticImports), (cls) => sb.append(`import static ${cls};`));
 
-                    sb.emptyLine();
-                    sb.append(`return ${ds.id};`);
+            sb.emptyLine();
+        }
 
-                    if (dsClsName === 'OracleDataSource') {
-                        sb.endBlock('}');
-                        sb.startBlock('catch (SQLException ex) {');
-                        sb.append('throw new Error(ex);');
-                        sb.endBlock('}');
-                    }
+        this.mainComment(sb);
+        sb.startBlock(`public class ${clsName} {`);
 
-                    sb.endBlock('}');
-                });
+        // 2. Add external property file
+        if (hasProps) {
+            this.commentBlock(sb, 'Secret properties loading.');
+            sb.append('private static final Properties props = new Properties();');
+            sb.emptyLine();
+            sb.startBlock('static {');
+            sb.startBlock('try (InputStream in = IgniteConfiguration.class.getClassLoader().getResourceAsStream("secret.properties")) {');
+            sb.append('props.load(in);');
+            sb.endBlock('}');
+            sb.startBlock('catch (Exception ignored) {');
+            sb.append('// No-op.');
+            sb.endBlock('}');
+            sb.endBlock('}');
+            sb.emptyLine();
+        }
 
-                sb.endBlock('}');
+        // 3. Add data sources.
+        const dataSources = this.collectDataSources(cfg);
+
+        if (dataSources.length) {
+            this.commentBlock(sb, 'Helper class for datasource creation.');
+            sb.startBlock('public static class DataSources {');
+
+            _.forEach(dataSources, (ds, idx) => {
+                const dsClsName = this.javaTypes.shortClassName(ds.clsName);
 
+                if (idx !== 0)
+                    sb.emptyLine();
+
+                sb.append(`public static final ${dsClsName} INSTANCE_${ds.id} = create${ds.id}();`);
                 sb.emptyLine();
-            }
 
-            _.forEach(clientNearCaches, (cache) => {
-                this.commentBlock(sb, `Configuration of near cache for cache: ${cache.name}.`,
-                    '',
-                    '@return Near cache configuration.',
-                    '@throws Exception If failed to construct near cache configuration instance.'
-                );
+                sb.startBlock(`private static ${dsClsName} create${ds.id}() {`);
 
-                const nearCacheBean = generator.cacheNearClient(cache);
+                if (dsClsName === 'OracleDataSource')
+                    sb.startBlock('try {');
 
-                sb.startBlock(`public static NearCacheConfiguration ${nearCacheBean.id}() throws Exception {`);
+                this.constructBean(sb, ds);
 
-                this.constructBean(sb, nearCacheBean);
                 sb.emptyLine();
+                sb.append(`return ${ds.id};`);
 
-                sb.append(`return ${nearCacheBean.id};`);
-                sb.endBlock('}');
+                if (dsClsName === 'OracleDataSource') {
+                    sb.endBlock('}');
+                    sb.startBlock('catch (SQLException ex) {');
+                    sb.append('throw new Error(ex);');
+                    sb.endBlock('}');
+                }
 
-                sb.emptyLine();
+                sb.endBlock('}');
             });
 
-            this.commentBlock(sb, 'Configure grid.',
+            sb.endBlock('}');
+
+            sb.emptyLine();
+        }
+
+        _.forEach(nearCacheBeans, (nearCacheBean) => {
+            this.commentBlock(sb, `Configuration of near cache for cache: ${nearCacheBean.cacheName}.`,
                 '',
-                '@return Ignite configuration.',
-                '@throws Exception If failed to construct Ignite configuration instance.'
+                '@return Near cache configuration.',
+                '@throws Exception If failed to construct near cache configuration instance.'
             );
-            sb.startBlock('public static IgniteConfiguration createConfiguration() throws Exception {');
 
-            this.constructBean(sb, cfg, [], true);
+            sb.startBlock(`public static NearCacheConfiguration ${nearCacheBean.id}() throws Exception {`);
 
+            this.constructBean(sb, nearCacheBean);
             sb.emptyLine();
 
-            sb.append(`return ${cfg.id};`);
-
+            sb.append(`return ${nearCacheBean.id};`);
             sb.endBlock('}');
 
-            const beans = this.collectBeansWithMapping(cfg);
+            sb.emptyLine();
+        });
 
-            _.forEach(beans, (bean, id) => {
-                sb.emptyLine();
+        this.commentBlock(sb, 'Configure grid.',
+            '',
+            '@return Ignite configuration.',
+            '@throws Exception If failed to construct Ignite configuration instance.'
+        );
+        sb.startBlock('public static IgniteConfiguration createConfiguration() throws Exception {');
 
-                this.METHOD_MAPPING[bean.clsName].generator(sb, id, bean);
-            });
+        this.constructBean(sb, cfg, [], true);
 
-            sb.endBlock('}');
+        sb.emptyLine();
 
-            return sb;
-        }
+        sb.append(`return ${cfg.id};`);
 
-        static cluster(cluster, pkg, clsName, client) {
-            const cfg = this.generator.igniteConfiguration(cluster, client);
+        sb.endBlock('}');
 
-            const clientNearCaches = client ? _.filter(cluster.caches, (cache) => _.get(cache, 'clientNearConfiguration.enabled')) : [];
+        const beans = this.collectBeansWithMapping(cfg);
 
-            return this.igniteConfiguration(cfg, pkg, clsName, clientNearCaches);
-        }
-
-        /**
-         * Generate source code for type by its domain model.
-         *
-         * @param fullClsName Full class name.
-         * @param fields Fields.
-         * @param addConstructor If 'true' then empty and full constructors should be generated.
-         * @returns {StringBuilder}
-         */
-        static pojo(fullClsName, fields, addConstructor) {
-            const dotIdx = fullClsName.lastIndexOf('.');
+        _.forEach(beans, (bean, id) => {
+            sb.emptyLine();
 
-            const pkg = fullClsName.substring(0, dotIdx);
-            const clsName = fullClsName.substring(dotIdx + 1);
+            this.METHOD_MAPPING[bean.clsName].generator(sb, id, bean);
+        });
 
-            const sb = new StringBuilder();
+        sb.endBlock('}');
 
-            sb.append(`package ${pkg};`);
-            sb.emptyLine();
-
-            const imports = ['java.io.Serializable'];
+        return sb;
+    }
 
-            _.forEach(fields, (field) => imports.push(JavaTypes.fullClassName(field.javaFieldType)));
+    static cluster(cluster, pkg, clsName, client) {
+        const cfg = this.generator.igniteConfiguration(cluster, client);
 
-            _.forEach(this._prepareImports(imports), (cls) => sb.append(`import ${cls};`));
+        const clientNearCaches = client ? _.filter(cluster.caches, (cache) => _.get(cache, 'clientNearConfiguration.enabled')) : [];
 
-            sb.emptyLine();
+        return this.igniteConfiguration(cfg, pkg, clsName, clientNearCaches);
+    }
 
-            this.mainComment(sb,
-                `${clsName} definition.`,
-                ''
-            );
-            sb.startBlock(`public class ${clsName} implements Serializable {`);
-            sb.append('/** */');
-            sb.append('private static final long serialVersionUID = 0L;');
-            sb.emptyLine();
+    /**
+     * Generate source code for type by its domain model.
+     *
+     * @param fullClsName Full class name.
+     * @param fields Fields.
+     * @param addConstructor If 'true' then empty and full constructors should be generated.
+     * @returns {StringBuilder}
+     */
+    static pojo(fullClsName, fields, addConstructor) {
+        const dotIdx = fullClsName.lastIndexOf('.');
 
-            // Generate fields declaration.
-            _.forEach(fields, (field) => {
-                const fldName = field.javaFieldName;
-                const fldType = JavaTypes.shortClassName(field.javaFieldType);
+        const pkg = fullClsName.substring(0, dotIdx);
+        const clsName = fullClsName.substring(dotIdx + 1);
 
-                sb.append(`/** Value for ${fldName}. */`);
-                sb.append(`private ${fldType} ${fldName};`);
+        const sb = new StringBuilder();
 
-                sb.emptyLine();
-            });
+        sb.append(`package ${pkg};`);
+        sb.emptyLine();
 
-            // Generate constructors.
-            if (addConstructor) {
-                this.commentBlock(sb, 'Empty constructor.');
-                sb.startBlock(`public ${clsName}() {`);
-                this.comment(sb, 'No-op.');
-                sb.endBlock('}');
+        const imports = ['java.io.Serializable'];
 
-                sb.emptyLine();
+        _.forEach(fields, (field) => imports.push(this.javaTypes.fullClassName(field.javaFieldType)));
 
-                this.commentBlock(sb, 'Full constructor.');
+        _.forEach(this._prepareImports(imports), (cls) => sb.append(`import ${cls};`));
 
-                const arg = (field) => {
-                    const fldType = JavaTypes.shortClassName(field.javaFieldType);
+        sb.emptyLine();
 
-                    return `${fldType} ${field.javaFieldName}`;
-                };
+        this.mainComment(sb,
+            `${clsName} definition.`,
+            ''
+        );
+        sb.startBlock(`public class ${clsName} implements Serializable {`);
+        sb.append('/** */');
+        sb.append('private static final long serialVersionUID = 0L;');
+        sb.emptyLine();
 
-                sb.startBlock(`public ${clsName}(${arg(_.head(fields))}${fields.length === 1 ? ') {' : ','}`);
+        // Generate fields declaration.
+        _.forEach(fields, (field) => {
+            const fldName = field.javaFieldName;
+            const fldType = this.javaTypes.shortClassName(field.javaFieldType);
 
-                _.forEach(_.tail(fields), (field, idx) => {
-                    sb.append(`${arg(field)}${idx !== fields.length - 2 ? ',' : ') {'}`);
-                });
+            sb.append(`/** Value for ${fldName}. */`);
+            sb.append(`private ${fldType} ${fldName};`);
 
-                _.forEach(fields, (field) => sb.append(`this.${field.javaFieldName} = ${field.javaFieldName};`));
+            sb.emptyLine();
+        });
 
-                sb.endBlock('}');
+        // Generate constructors.
+        if (addConstructor) {
+            this.commentBlock(sb, 'Empty constructor.');
+            sb.startBlock(`public ${clsName}() {`);
+            this.comment(sb, 'No-op.');
+            sb.endBlock('}');
 
-                sb.emptyLine();
-            }
+            sb.emptyLine();
 
-            // Generate getters and setters methods.
-            _.forEach(fields, (field) => {
-                const fldType = JavaTypes.shortClassName(field.javaFieldType);
-                const fldName = field.javaFieldName;
+            this.commentBlock(sb, 'Full constructor.');
 
-                this.commentBlock(sb,
-                    `Gets ${fldName}`,
-                    '',
-                    `@return Value for ${fldName}.`
-                );
-                sb.startBlock(`public ${fldType} ${JavaTypes.toJavaName('get', fldName)}() {`);
-                sb.append('return ' + fldName + ';');
-                sb.endBlock('}');
+            const arg = (field) => {
+                const fldType = this.javaTypes.shortClassName(field.javaFieldType);
 
-                sb.emptyLine();
+                return `${fldType} ${field.javaFieldName}`;
+            };
 
-                this.commentBlock(sb,
-                    `Sets ${fldName}`,
-                    '',
-                    `@param ${fldName} New value for ${fldName}.`
-                );
-                sb.startBlock(`public void ${JavaTypes.toJavaName('set', fldName)}(${fldType} ${fldName}) {`);
-                sb.append(`this.${fldName} = ${fldName};`);
-                sb.endBlock('}');
+            sb.startBlock(`public ${clsName}(${arg(_.head(fields))}${fields.length === 1 ? ') {' : ','}`);
 
-                sb.emptyLine();
+            _.forEach(_.tail(fields), (field, idx) => {
+                sb.append(`${arg(field)}${idx !== fields.length - 2 ? ',' : ') {'}`);
             });
 
-            // Generate equals() method.
-            this.commentBlock(sb, '{@inheritDoc}');
-            sb.startBlock('@Override public boolean equals(Object o) {');
-            sb.startBlock('if (this == o)');
-            sb.append('return true;');
+            _.forEach(fields, (field) => sb.append(`this.${field.javaFieldName} = ${field.javaFieldName};`));
 
-            sb.endBlock('');
+            sb.endBlock('}');
 
-            sb.startBlock(`if (!(o instanceof ${clsName}))`);
-            sb.append('return false;');
+            sb.emptyLine();
+        }
 
-            sb.endBlock('');
+        // Generate getters and setters methods.
+        _.forEach(fields, (field) => {
+            const fldType = this.javaTypes.shortClassName(field.javaFieldType);
+            const fldName = field.javaFieldName;
 
-            sb.append(`${clsName} that = (${clsName})o;`);
+            this.commentBlock(sb,
+                `Gets ${fldName}`,
+                '',
+                `@return Value for ${fldName}.`
+            );
+            sb.startBlock(`public ${fldType} ${this.javaTypes.toJavaName('get', fldName)}() {`);
+            sb.append('return ' + fldName + ';');
+            sb.endBlock('}');
 
-            _.forEach(fields, (field) => {
-                sb.emptyLine();
+            sb.emptyLine();
 
-                const javaName = field.javaFieldName;
-                const javaType = field.javaFieldType;
+            this.commentBlock(sb,
+                `Sets ${fldName}`,
+                '',
+                `@param ${fldName} New value for ${fldName}.`
+            );
+            sb.startBlock(`public void ${this.javaTypes.toJavaName('set', fldName)}(${fldType} ${fldName}) {`);
+            sb.append(`this.${fldName} = ${fldName};`);
+            sb.endBlock('}');
 
-                switch (javaType) {
-                    case 'float':
-                        sb.startBlock(`if (Float.compare(${javaName}, that.${javaName}) != 0)`);
+            sb.emptyLine();
+        });
 
-                        break;
-                    case 'double':
-                        sb.startBlock(`if (Double.compare(${javaName}, that.${javaName}) != 0)`);
+        // Generate equals() method.
+        this.commentBlock(sb, '{@inheritDoc}');
+        sb.startBlock('@Override public boolean equals(Object o) {');
+        sb.startBlock('if (this == o)');
+        sb.append('return true;');
 
-                        break;
-                    default:
-                        if (JavaTypes.isJavaPrimitive(javaType))
-                            sb.startBlock('if (' + javaName + ' != that.' + javaName + ')');
-                        else
-                            sb.startBlock('if (' + javaName + ' != null ? !' + javaName + '.equals(that.' + javaName + ') : that.' + javaName + ' != null)');
-                }
+        sb.endBlock('');
 
-                sb.append('return false;');
+        sb.startBlock(`if (!(o instanceof ${clsName}))`);
+        sb.append('return false;');
 
-                sb.endBlock('');
-            });
+        sb.endBlock('');
 
-            sb.append('return true;');
-            sb.endBlock('}');
+        sb.append(`${clsName} that = (${clsName})o;`);
 
+        _.forEach(fields, (field) => {
             sb.emptyLine();
 
-            // Generate hashCode() method.
-            this.commentBlock(sb, '{@inheritDoc}');
-            sb.startBlock('@Override public int hashCode() {');
+            const javaName = field.javaFieldName;
+            const javaType = field.javaFieldType;
 
-            let first = true;
-            let tempVar = false;
+            switch (javaType) {
+                case 'float':
+                    sb.startBlock(`if (Float.compare(${javaName}, that.${javaName}) != 0)`);
 
-            _.forEach(fields, (field) => {
-                const javaName = field.javaFieldName;
-                const javaType = field.javaFieldType;
+                    break;
+                case 'double':
+                    sb.startBlock(`if (Double.compare(${javaName}, that.${javaName}) != 0)`);
 
-                let fldHashCode;
+                    break;
+                default:
+                    if (this.javaTypes.isPrimitive(javaType))
+                        sb.startBlock('if (' + javaName + ' != that.' + javaName + ')');
+                    else
+                        sb.startBlock('if (' + javaName + ' != null ? !' + javaName + '.equals(that.' + javaName + ') : that.' + javaName + ' != null)');
+            }
 
-                switch (javaType) {
-                    case 'boolean':
-                        fldHashCode = `${javaName} ? 1 : 0`;
+            sb.append('return false;');
 
-                        break;
-                    case 'byte':
-                    case 'short':
-                        fldHashCode = `(int)${javaName}`;
+            sb.endBlock('');
+        });
 
-                        break;
-                    case 'int':
-                        fldHashCode = `${javaName}`;
+        sb.append('return true;');
+        sb.endBlock('}');
 
-                        break;
-                    case 'long':
-                        fldHashCode = `(int)(${javaName} ^ (${javaName} >>> 32))`;
+        sb.emptyLine();
 
-                        break;
-                    case 'float':
-                        fldHashCode = `${javaName} != +0.0f ? Float.floatToIntBits(${javaName}) : 0`;
+        // Generate hashCode() method.
+        this.commentBlock(sb, '{@inheritDoc}');
+        sb.startBlock('@Override public int hashCode() {');
 
-                        break;
-                    case 'double':
-                        sb.append(`${tempVar ? 'ig_hash_temp' : 'long ig_hash_temp'} = Double.doubleToLongBits(${javaName});`);
+        let first = true;
+        let tempVar = false;
 
-                        tempVar = true;
+        _.forEach(fields, (field) => {
+            const javaName = field.javaFieldName;
+            const javaType = field.javaFieldType;
 
-                        fldHashCode = `${javaName} != +0.0f ? Float.floatToIntBits(${javaName}) : 0`;
+            let fldHashCode;
 
-                        break;
-                    default:
-                        fldHashCode = `${javaName} != null ? ${javaName}.hashCode() : 0`;
-                }
+            switch (javaType) {
+                case 'boolean':
+                    fldHashCode = `${javaName} ? 1 : 0`;
 
-                sb.append(first ? `int res = ${fldHashCode};` : `res = 31 * res + ${fldHashCode.startsWith('(') ? fldHashCode : `(${fldHashCode})`};`);
+                    break;
+                case 'byte':
+                case 'short':
+                    fldHashCode = `(int)${javaName}`;
 
-                first = false;
+                    break;
+                case 'int':
+                    fldHashCode = `${javaName}`;
 
-                sb.emptyLine();
-            });
+                    break;
+                case 'long':
+                    fldHashCode = `(int)(${javaName} ^ (${javaName} >>> 32))`;
 
-            sb.append('return res;');
-            sb.endBlock('}');
+                    break;
+                case 'float':
+                    fldHashCode = `${javaName} != +0.0f ? Float.floatToIntBits(${javaName}) : 0`;
 
-            sb.emptyLine();
+                    break;
+                case 'double':
+                    sb.append(`${tempVar ? 'ig_hash_temp' : 'long ig_hash_temp'} = Double.doubleToLongBits(${javaName});`);
 
-            this.commentBlock(sb, '{@inheritDoc}');
-            sb.startBlock('@Override public String toString() {');
-            sb.startBlock(`return "${clsName} [" + `);
+                    tempVar = true;
 
-            _.forEach(fields, (field, idx) => {
-                sb.append(`"${field.javaFieldName}=" + ${field.javaFieldName}${idx < fields.length - 1 ? ' + ", " + ' : ' +'}`);
-            });
+                    fldHashCode = `${javaName} != +0.0f ? Float.floatToIntBits(${javaName}) : 0`;
 
-            sb.endBlock('"]";');
-            sb.endBlock('}');
+                    break;
+                default:
+                    fldHashCode = `${javaName} != null ? ${javaName}.hashCode() : 0`;
+            }
 
-            sb.endBlock('}');
+            sb.append(first ? `int res = ${fldHashCode};` : `res = 31 * res + ${fldHashCode.startsWith('(') ? fldHashCode : `(${fldHashCode})`};`);
 
-            return sb.asString();
-        }
+            first = false;
 
-        /**
-         * Generate source code for type by its domain models.
-         *
-         * @param caches List of caches to generate POJOs for.
-         * @param addConstructor If 'true' then generate constructors.
-         * @param includeKeyFields If 'true' then include key fields into value POJO.
-         */
-        static pojos(caches, addConstructor, includeKeyFields) {
-            const pojos = [];
-
-            _.forEach(caches, (cache) => {
-                _.forEach(cache.domains, (domain) => {
-                    // Process only  domains with 'generatePojo' flag and skip already generated classes.
-                    if (domain.generatePojo && !_.find(pojos, {valueType: domain.valueType}) &&
-                        // Skip domain models without value fields.
-                        _.nonEmpty(domain.valueFields)) {
-                        const pojo = {};
-
-                        // Key class generation only if key is not build in java class.
-                        if (_.nonNil(domain.keyFields) && domain.keyFields.length > 0) {
-                            pojo.keyType = domain.keyType;
-                            pojo.keyClass = this.pojo(domain.keyType, domain.keyFields, addConstructor);
-                        }
+            sb.emptyLine();
+        });
 
-                        const valueFields = _.clone(domain.valueFields);
+        sb.append('return res;');
+        sb.endBlock('}');
 
-                        if (includeKeyFields) {
-                            _.forEach(domain.keyFields, (fld) => {
-                                if (!_.find(valueFields, {javaFieldName: fld.javaFieldName}))
-                                    valueFields.push(fld);
-                            });
-                        }
+        sb.emptyLine();
 
-                        pojo.valueType = domain.valueType;
-                        pojo.valueClass = this.pojo(domain.valueType, valueFields, addConstructor);
+        this.commentBlock(sb, '{@inheritDoc}');
+        sb.startBlock('@Override public String toString() {');
+        sb.startBlock(`return "${clsName} [" + `);
 
-                        pojos.push(pojo);
-                    }
-                });
-            });
+        _.forEach(fields, (field, idx) => {
+            sb.append(`"${field.javaFieldName}=" + ${field.javaFieldName}${idx < fields.length - 1 ? ' + ", " + ' : ' +'}`);
+        });
 
-            return pojos;
-        }
+        sb.endBlock('"]";');
+        sb.endBlock('}');
 
-        // Generate creation and execution of cache query.
-        static _multilineQuery(sb, query, prefix, postfix) {
-            if (_.isEmpty(query))
-                return;
+        sb.endBlock('}');
 
-            _.forEach(query, (line, ix) => {
-                if (ix === 0) {
-                    if (query.length === 1)
-                        sb.append(`${prefix}"${line}"${postfix}`);
-                    else
-                        sb.startBlock(`${prefix}"${line}" +`);
+        return sb.asString();
+    }
+
+    /**
+     * Generate source code for type by its domain models.
+     *
+     * @param caches List of caches to generate POJOs for.
+     * @param addConstructor If 'true' then generate constructors.
+     * @param includeKeyFields If 'true' then include key fields into value POJO.
+     */
+    static pojos(caches, addConstructor, includeKeyFields) {
+        const pojos = [];
+
+        _.forEach(caches, (cache) => {
+            _.forEach(cache.domains, (domain) => {
+                // Process only  domains with 'generatePojo' flag and skip already generated classes.
+                if (domain.generatePojo && !_.find(pojos, {valueType: domain.valueType}) &&
+                    // Skip domain models without value fields.
+                    _.nonEmpty(domain.valueFields)) {
+                    const pojo = {
+                        keyType: domain.keyType,
+                        valueType: domain.valueType
+                    };
+
+                    // Key class generation only if key is not build in java class.
+                    if (this.javaTypes.nonBuiltInClass(domain.keyType) && _.nonEmpty(domain.keyFields))
+                        pojo.keyClass = this.pojo(domain.keyType, domain.keyFields, addConstructor);
+
+                    const valueFields = _.clone(domain.valueFields);
+
+                    if (includeKeyFields) {
+                        _.forEach(domain.keyFields, (fld) => {
+                            if (!_.find(valueFields, {javaFieldName: fld.javaFieldName}))
+                                valueFields.push(fld);
+                        });
+                    }
+
+                    pojo.valueClass = this.pojo(domain.valueType, valueFields, addConstructor);
+
+                    pojos.push(pojo);
                 }
-                else
-                    sb.append(`"${line}"${ix === query.length - 1 ? postfix : ' +'}`);
             });
+        });
 
-            if (query.length > 1)
-                sb.endBlock('');
-            else
-                sb.emptyLine();
-        }
+        return pojos;
+    }
 
-        // Generate creation and execution of prepared statement.
-        static _prepareStatement(sb, conVar, query) {
-            this._multilineQuery(sb, query, `${conVar}.prepareStatement(`, ').executeUpdate();');
-        }
+    // Generate creation and execution of cache query.
+    static _multilineQuery(sb, query, prefix, postfix) {
+        if (_.isEmpty(query))
+            return;
 
-        static demoStartup(sb, cluster, shortFactoryCls) {
-            const cachesWithDataSource = _.filter(cluster.caches, (cache) => {
-                const kind = _.get(cache, 'cacheStoreFactory.kind');
+        _.forEach(query, (line, ix) => {
+            if (ix === 0) {
+                if (query.length === 1)
+                    sb.append(`${prefix}"${line}"${postfix}`);
+                else
+                    sb.startBlock(`${prefix}"${line}" +`);
+            }
+            else
+                sb.append(`"${line}"${ix === query.length - 1 ? postfix : ' +'}`);
+        });
 
-                if (kind) {
-                    const store = cache.cacheStoreFactory[kind];
+        if (query.length > 1)
+            sb.endBlock('');
+        else
+            sb.emptyLine();
+    }
 
-                    return (store.connectVia === 'DataSource' || _.isNil(store.connectVia)) && store.dialect;
-                }
+    // Generate creation and execution of prepared statement.
+    static _prepareStatement(sb, conVar, query) {
+        this._multilineQuery(sb, query, `${conVar}.prepareStatement(`, ').executeUpdate();');
+    }
 
-                return false;
-            });
+    static demoStartup(sb, cluster, shortFactoryCls) {
+        const cachesWithDataSource = _.filter(cluster.caches, (cache) => {
+            const kind = _.get(cache, 'cacheStoreFactory.kind');
 
-            const uniqDomains = [];
+            if (kind) {
+                const store = cache.cacheStoreFactory[kind];
 
-            // Prepare array of cache and his demo domain model list. Every domain is contained only in first cache.
-            const demoTypes = _.reduce(cachesWithDataSource, (acc, cache) => {
-                const domains = _.filter(cache.domains, (domain) => _.nonEmpty(domain.valueFields) &&
-                    !_.includes(uniqDomains, domain));
+                return (store.connectVia === 'DataSource' || _.isNil(store.connectVia)) && store.dialect;
+            }
 
-                if (_.nonEmpty(domains)) {
-                    uniqDomains.push(...domains);
+            return false;
+        });
 
-                    acc.push({
-                        cache,
-                        domains
-                    });
-                }
+        const uniqDomains = [];
 
-                return acc;
-            }, []);
+        // Prepare array of cache and his demo domain model list. Every domain is contained only in first cache.
+        const demoTypes = _.reduce(cachesWithDataSource, (acc, cache) => {
+            const domains = _.filter(cache.domains, (domain) => _.nonEmpty(domain.valueFields) &&
+                !_.includes(uniqDomains, domain));
 
-            if (_.nonEmpty(demoTypes)) {
-                // Group domain modes by data source
-                const typeByDs = _.groupBy(demoTypes, ({cache}) => cache.cacheStoreFactory[cache.cacheStoreFactory.kind].dataSourceBean);
+            if (_.nonEmpty(domains)) {
+                uniqDomains.push(...domains);
 
-                let rndNonDefined = true;
+                acc.push({
+                    cache,
+                    domains
+                });
+            }
 
-                const generatedConsts = [];
+            return acc;
+        }, []);
 
-                _.forEach(typeByDs, (types) => {
-                    _.forEach(types, (type) => {
-                        _.forEach(type.domains, (domain) => {
-                            const valType = domain.valueType.toUpperCase();
+        if (_.nonEmpty(demoTypes)) {
+            // Group domain modes by data source
+            const typeByDs = _.groupBy(demoTypes, ({cache}) => cache.cacheStoreFactory[cache.cacheStoreFactory.kind].dataSourceBean);
 
-                            const desc = _.find(PREDEFINED_QUERIES, (qry) => valType.endsWith(qry.type));
+            let rndNonDefined = true;
 
-                            if (desc) {
-                                if (rndNonDefined && desc.rndRequired) {
-                                    this.commentBlock(sb, 'Random generator for demo data.');
-                                    sb.append('private static final Random rnd = new Random();');
+            const generatedConsts = [];
 
-                                    sb.emptyLine();
+            _.forEach(typeByDs, (types) => {
+                _.forEach(types, (type) => {
+                    _.forEach(type.domains, (domain) => {
+                        const valType = domain.valueType.toUpperCase();
 
-                                    rndNonDefined = false;
-                                }
+                        const desc = _.find(PREDEFINED_QUERIES, (qry) => valType.endsWith(qry.type));
 
-                                _.forEach(desc.insertCntConsts, (cnt) => {
-                                    if (!_.includes(generatedConsts, cnt.name)) {
-                                        this.commentBlock(sb, cnt.comment);
-                                        sb.append(`private static final int ${cnt.name} = ${cnt.val};`);
+                        if (desc) {
+                            if (rndNonDefined && desc.rndRequired) {
+                                this.commentBlock(sb, 'Random generator for demo data.');
+                                sb.append('private static final Random rnd = new Random();');
 
-                                        sb.emptyLine();
+                                sb.emptyLine();
 
-                                        generatedConsts.push(cnt.name);
-                                    }
-                                });
+                                rndNonDefined = false;
                             }
-                        });
-                    });
-                });
 
-                // Generation of fill database method
-                this.commentBlock(sb, 'Fill data for Demo.');
-                sb.startBlock('private static void prepareDemoData() throws SQLException {');
+                            _.forEach(desc.insertCntConsts, (cnt) => {
+                                if (!_.includes(generatedConsts, cnt.name)) {
+                                    this.commentBlock(sb, cnt.comment);
+                                    sb.append(`private static final int ${cnt.name} = ${cnt.val};`);
 
-                let firstDs = true;
+                                    sb.emptyLine();
 
-                _.forEach(typeByDs, (types, ds) => {
-                    const conVar = ds + 'Con';
+                                    generatedConsts.push(cnt.name);
+                                }
+                            });
+                        }
+                    });
+                });
+            });
 
-                    if (firstDs)
-                        firstDs = false;
-                    else
-                        sb.emptyLine();
+            // Generation of fill database method
+            this.commentBlock(sb, 'Fill data for Demo.');
+            sb.startBlock('private static void prepareDemoData() throws SQLException {');
 
-                    sb.startBlock(`try (Connection ${conVar} = ${shortFactoryCls}.DataSources.INSTANCE_${ds}.getConnection()) {`);
+            let firstDs = true;
 
-                    let first = true;
-                    let stmtFirst = true;
+            _.forEach(typeByDs, (types, ds) => {
+                const conVar = ds + 'Con';
 
-                    _.forEach(types, (type) => {
-                        _.forEach(type.domains, (domain) => {
-                            const valType = domain.valueType.toUpperCase();
+                if (firstDs)
+                    firstDs = false;
+                else
+                    sb.emptyLine();
 
-                            const desc = _.find(PREDEFINED_QUERIES, (qry) => valType.endsWith(qry.type));
+                sb.startBlock(`try (Connection ${conVar} = ${shortFactoryCls}.DataSources.INSTANCE_${ds}.getConnection()) {`);
 
-                            if (desc) {
-                                if (first)
-                                    first = false;
-                                else
-                                    sb.emptyLine();
+                let first = true;
+                let stmtFirst = true;
 
-                                this.comment(sb, `Generate ${desc.type}.`);
+                _.forEach(types, (type) => {
+                    _.forEach(type.domains, (domain) => {
+                        const valType = domain.valueType.toUpperCase();
 
-                                if (desc.schema)
-                                    this._prepareStatement(sb, conVar, [`CREATE SCHEMA IF NOT EXISTS ${desc.schema}`]);
+                        const desc = _.find(PREDEFINED_QUERIES, (qry) => valType.endsWith(qry.type));
 
-                                this._prepareStatement(sb, conVar, desc.create);
+                        if (desc) {
+                            if (first)
+                                first = false;
+                            else
+                                sb.emptyLine();
 
-                                this._prepareStatement(sb, conVar, desc.clearQuery);
+                            this.comment(sb, `Generate ${desc.type}.`);
 
-                                let stmtVar = 'stmt';
+                            if (desc.schema)
+                                this._prepareStatement(sb, conVar, [`CREATE SCHEMA IF NOT EXISTS ${desc.schema}`]);
 
-                                if (stmtFirst) {
-                                    stmtFirst = false;
+                            this._prepareStatement(sb, conVar, desc.create);
 
-                                    stmtVar = 'PreparedStatement stmt';
-                                }
+                            this._prepareStatement(sb, conVar, desc.clearQuery);
 
-                                if (_.isFunction(desc.customGeneration))
-                                    desc.customGeneration(sb, conVar, stmtVar);
-                                else {
-                                    sb.append(`${stmtVar} = ${conVar}.prepareStatement("${desc.insertPattern}");`);
+                            let stmtVar = 'stmt';
 
-                                    sb.emptyLine();
+                            if (stmtFirst) {
+                                stmtFirst = false;
 
-                                    sb.startBlock(`for (int id = 0; id < ${desc.insertCntConsts[0].name}; id ++) {`);
+                                stmtVar = 'PreparedStatement stmt';
+                            }
 
-                                    desc.fillInsertParameters(sb);
+                            if (_.isFunction(desc.customGeneration))
+                                desc.customGeneration(sb, conVar, stmtVar);
+                            else {
+                                sb.append(`${stmtVar} = ${conVar}.prepareStatement("${desc.insertPattern}");`);
 
-                                    sb.emptyLine();
+                                sb.emptyLine();
 
-                                    sb.append('stmt.executeUpdate();');
+                                sb.startBlock(`for (int id = 0; id < ${desc.insertCntConsts[0].name}; id ++) {`);
 
-                                    sb.endBlock('}');
-                                }
+                                desc.fillInsertParameters(sb);
 
                                 sb.emptyLine();
 
-                                sb.append(`${conVar}.commit();`);
+                                sb.append('stmt.executeUpdate();');
+
+                                sb.endBlock('}');
                             }
-                        });
-                    });
 
-                    sb.endBlock('}');
+                            sb.emptyLine();
+
+                            sb.append(`${conVar}.commit();`);
+                        }
+                    });
                 });
 
                 sb.endBlock('}');
+            });
 
-                sb.emptyLine();
+            sb.endBlock('}');
 
-                this.commentBlock(sb, 'Print result table to console.');
-                sb.startBlock('private static void printResult(List<Cache.Entry<Object, Object>> rows) {');
-                sb.append('for (Cache.Entry<Object, Object> row: rows)');
-                sb.append('    System.out.println(row);');
-                sb.endBlock('}');
+            sb.emptyLine();
 
-                sb.emptyLine();
+            this.commentBlock(sb, 'Print result table to console.');
+            sb.startBlock('private static void printResult(List<Cache.Entry<Object, Object>> rows) {');
+            sb.append('for (Cache.Entry<Object, Object> row: rows)');
+            sb.append('    System.out.println(row);');
+            sb.endBlock('}');
 
-                // Generation of execute queries method.
-                this.commentBlock(sb, 'Run demo.');
-                sb.startBlock('private static void runDemo(Ignite ignite) throws SQLException {');
+            sb.emptyLine();
 
-                const getType = (fullType) => fullType.substr(fullType.lastIndexOf('.') + 1);
+            // Generation of execute queries method.
+            this.commentBlock(sb, 'Run demo.');
+            sb.startBlock('private static void runDemo(Ignite ignite) throws SQLException {');
 
-                const cacheLoaded = [];
-                let rowVariableDeclared = false;
-                firstDs = true;
+            const getType = (fullType) => fullType.substr(fullType.lastIndexOf('.') + 1);
 
-                _.forEach(typeByDs, (types, ds) => {
-                    const conVar = ds + 'Con';
+            const cacheLoaded = [];
+            let rowVariableDeclared = false;
+            firstDs = true;
 
-                    if (firstDs)
-                        firstDs = false;
-                    else
-                        sb.emptyLine();
+            _.forEach(typeByDs, (types, ds) => {
+                const conVar = ds + 'Con';
 
-                    sb.startBlock(`try (Connection ${conVar} = ${shortFactoryCls}.DataSources.INSTANCE_${ds}.getConnection()) {`);
+                if (firstDs)
+                    firstDs = false;
+                else
+                    sb.emptyLine();
 
-                    let first = true;
+                sb.startBlock(`try (Connection ${conVar} = ${shortFactoryCls}.DataSources.INSTANCE_${ds}.getConnection()) {`);
 
-                    _.forEach(types, (type) => {
-                        _.forEach(type.domains, (domain) => {
-                            const valType = domain.valueType.toUpperCase();
+                let first = true;
 
-                            const desc = _.find(PREDEFINED_QUERIES, (qry) => valType.endsWith(qry.type));
+                _.forEach(types, (type) => {
+                    _.forEach(type.domains, (domain) => {
+                        const valType = domain.valueType.toUpperCase();
 
-                            if (desc) {
-                                if (_.isEmpty(desc.selectQuery))
-                                    return;
+                        const desc = _.find(PREDEFINED_QUERIES, (qry) => valType.endsWith(qry.type));
 
-                                if (first)
-                                    first = false;
-                                else
-                                    sb.emptyLine();
+                        if (desc) {
+                            if (_.isEmpty(desc.selectQuery))
+                                return;
 
-                                const cacheName = type.cache.name;
+                            if (first)
+                                first = false;
+                            else
+                                sb.emptyLine();
 
-                                if (!_.includes(cacheLoaded, cacheName)) {
-                                    sb.append(`ignite.cache("${cacheName}").loadCache(null);`);
+                            const cacheName = type.cache.name;
 
-                                    sb.emptyLine();
+                            if (!_.includes(cacheLoaded, cacheName)) {
+                                sb.append(`ignite.cache("${cacheName}").loadCache(null);`);
 
-                                    cacheLoaded.push(cacheName);
-                                }
+                                sb.emptyLine();
 
-                                const varRows = rowVariableDeclared ? 'rows' : 'List<Cache.Entry<Object, Object>> rows';
+                                cacheLoaded.push(cacheName);
+                            }
 
-                                this._multilineQuery(sb, desc.selectQuery, `${varRows} = ignite.cache("${cacheName}").query(new SqlQuery<>("${getType(domain.valueType)}", `, ')).getAll();');
+                            const varRows = rowVariableDeclared ? 'rows' : 'List<Cache.Entry<Object, Object>> rows';
 
-                                sb.append('printResult(rows);');
+                            this._multilineQuery(sb, desc.selectQuery, `${varRows} = ignite.cache("${cacheName}").query(new SqlQuery<>("${getType(domain.valueType)}", `, ')).getAll();');
 
-                                rowVariableDeclared = true;
-                            }
-                        });
-                    });
+                            sb.append('printResult(rows);');
 
-                    sb.endBlock('}');
+                            rowVariableDeclared = true;
+                        }
+                    });
                 });
 
                 sb.endBlock('}');
-            }
-        }
-
-        /**
-         * Function to generate java class for node startup with cluster configuration.
-         *
-         * @param {Object} cluster Cluster to process.
-         * @param {String} fullClsName Full class name.
-         * @param {String} cfgRef Config.
-         * @param {String} [factoryCls] fully qualified class name of configuration factory.
-         * @param {Array.<Object>} [clientNearCaches] Is client node.
-         */
-        static nodeStartup(cluster, fullClsName, cfgRef, factoryCls, clientNearCaches) {
-            const dotIdx = fullClsName.lastIndexOf('.');
-
-            const pkg = fullClsName.substring(0, dotIdx);
-            const clsName = fullClsName.substring(dotIdx + 1);
+            });
 
-            const demo = clsName === 'DemoStartup';
+            sb.endBlock('}');
+        }
+    }
 
-            const sb = new StringBuilder();
+    /**
+     * Function to generate java class for node startup with cluster configuration.
+     *
+     * @param {Object} cluster Cluster to process.
+     * @param {String} fullClsName Full class name.
+     * @param {String} cfgRef Config.
+     * @param {String} [factoryCls] fully qualified class name of configuration factory.
+     * @param {Array.<Object>} [clientNearCaches] Is client node.
+     */
+    static nodeStartup(cluster, fullClsName, cfgRef, factoryCls, clientNearCaches) {
+        const dotIdx = fullClsName.lastIndexOf('.');
 
-            const imports = ['org.apache.ignite.Ignition', 'org.apache.ignite.Ignite'];
+        const pkg = fullClsName.substring(0, dotIdx);
+        const clsName = fullClsName.substring(dotIdx + 1);
 
-            if (demo) {
-                imports.push('org.h2.tools.Server', 'java.sql.Connection', 'java.sql.PreparedStatement',
-                    'java.sql.SQLException', 'java.util.Random', 'java.util.List', 'javax.cache.Cache',
-                    'org.apache.ignite.cache.query.SqlQuery');
-            }
+        const demo = clsName === 'DemoStartup';
 
-            let shortFactoryCls;
+        const sb = new StringBuilder();
 
-            if (factoryCls) {
-                imports.push(factoryCls);
+        const imports = ['org.apache.ignite.Ignition', 'org.apache.ignite.Ignite'];
 
-                shortFactoryCls = JavaTypes.shortClassName(factoryCls);
-            }
+        if (demo) {
+            imports.push('org.h2.tools.Server', 'java.sql.Connection', 'java.sql.PreparedStatement',
+                'java.sql.SQLException', 'java.util.Random', 'java.util.List', 'javax.cache.Cache',
+                'org.apache.ignite.cache.query.SqlQuery');
+        }
 
-            sb.append(`package ${pkg};`)
-                .emptyLine();
+        let shortFactoryCls;
 
-            _.forEach(this._prepareImports(imports), (cls) => sb.append(`import ${cls};`));
-            sb.emptyLine();
+        if (factoryCls) {
+            imports.push(factoryCls);
 
-            if (demo) {
-                this.mainComment(sb,
-                    'To start demo configure data sources in secret.properties file.',
-                    'For H2 database it should be like following:',
-                    'dsH2.jdbc.url=jdbc:h2:tcp://localhost/mem:DemoDB;DB_CLOSE_DELAY=-1',
-                    'dsH2.jdbc.username=sa',
-                    'dsH2.jdbc.password=',
-                    ''
-                );
-            }
-            else
-                this.mainComment(sb);
+            shortFactoryCls = this.javaTypes.shortClassName(factoryCls);
+        }
 
-            sb.startBlock(`public class ${clsName} {`);
+        sb.append(`package ${pkg};`)
+            .emptyLine();
 
-            if (demo && shortFactoryCls)
-                this.demoStartup(sb, cluster, shortFactoryCls);
+        _.forEach(this._prepareImports(imports), (cls) => sb.append(`import ${cls};`));
+        sb.emptyLine();
 
-            this.commentBlock(sb,
-                'Start up node with specified configuration.',
-                '',
-                '@param args Command line arguments, none required.',
-                '@throws Exception If failed.'
+        if (demo) {
+            this.mainComment(sb,
+                'To start demo configure data sources in secret.properties file.',
+                'For H2 database it should be like following:',
+                'dsH2.jdbc.url=jdbc:h2:tcp://localhost/mem:DemoDB;DB_CLOSE_DELAY=-1',
+                'dsH2.jdbc.username=sa',
+                'dsH2.jdbc.password=',
+                ''
             );
-            sb.startBlock('public static void main(String[] args) throws Exception {');
-
-            if (demo) {
-                sb.startBlock('try {');
-                sb.append('// Start H2 database server.');
-                sb.append('Server.createTcpServer("-tcpDaemon").start();');
-                sb.endBlock('}');
-                sb.startBlock('catch (SQLException ignore) {');
-                sb.append('// No-op.');
-                sb.endBlock('}');
-
-                sb.emptyLine();
-            }
-
-            if ((_.nonEmpty(clientNearCaches) || demo) && shortFactoryCls) {
-                sb.append(`Ignite ignite = Ignition.start(${cfgRef});`);
+        }
+        else
+            this.mainComment(sb);
 
-                _.forEach(clientNearCaches, (cache, idx) => {
-                    sb.emptyLine();
+        sb.startBlock(`public class ${clsName} {`);
 
-                    if (idx === 0)
-                        sb.append('// Demo of near cache creation on client node.');
+        if (demo && shortFactoryCls)
+            this.demoStartup(sb, cluster, shortFactoryCls);
 
-                    const nearCacheMtd = JavaTypes.toJavaName('nearConfiguration', cache.name);
+        this.commentBlock(sb,
+            'Start up node with specified configuration.',
+            '',
+            '@param args Command line arguments, none required.',
+            '@throws Exception If failed.'
+        );
+        sb.startBlock('public static void main(String[] args) throws Exception {');
 
-                    sb.append(`ignite.getOrCreateCache(${shortFactoryCls}.${cache.name}(), ${shortFactoryCls}.${nearCacheMtd}());`);
-                });
-            }
-            else
-                sb.append(`Ignition.start(${cfgRef});`);
+        if (demo) {
+            sb.startBlock('try {');
+            sb.append('// Start H2 database server.');
+            sb.append('Server.createTcpServer("-tcpDaemon").start();');
+            sb.endBlock('}');
+            sb.startBlock('catch (SQLException ignore) {');
+            sb.append('// No-op.');
+            sb.endBlock('}');
 
-            if (demo) {
-                sb.emptyLine();
+            sb.emptyLine();
+        }
 
-                sb.append('prepareDemoData();');
+        if ((_.nonEmpty(clientNearCaches) || demo) && shortFactoryCls) {
+            sb.append(`Ignite ignite = Ignition.start(${cfgRef});`);
 
+            _.forEach(clientNearCaches, (cache, idx) => {
                 sb.emptyLine();
 
-                sb.append('runDemo(ignite);');
-            }
+                if (idx === 0)
+                    sb.append('// Demo of near cache creation on client node.');
 
-            sb.endBlock('}');
-
-            sb.endBlock('}');
+                const nearCacheMtd = this.javaTypes.toJavaName('nearConfiguration', cache.name);
 
-            return sb.asString();
+                sb.append(`ignite.getOrCreateCache(${shortFactoryCls}.${this.javaTypes.toJavaName('cache', cache.name)}(), ${shortFactoryCls}.${nearCacheMtd}());`);
+            });
         }
+        else
+            sb.append(`Ignition.start(${cfgRef});`);
 
-        /**
-         * Function to generate java class for load caches.
-         *
-         * @param caches Caches to load.
-         * @param pkg Class package name.
-         * @param clsName Class name.
-         * @param {String} cfgRef Config.
-         */
-        static loadCaches(caches, pkg, clsName, cfgRef) {
-            const sb = new StringBuilder();
-
-            sb.append(`package ${pkg};`)
-                .emptyLine();
-
-            const imports = ['org.apache.ignite.Ignition', 'org.apache.ignite.Ignite'];
-
-            _.forEach(this._prepareImports(imports), (cls) => sb.append(`import ${cls};`));
+        if (demo) {
             sb.emptyLine();
 
-            this.mainComment(sb);
-            sb.startBlock(`public class ${clsName} {`);
+            sb.append('prepareDemoData();');
 
-            this.commentBlock(sb,
-                '<p>',
-                'Utility to load caches from database.',
-                '<p>',
-                'How to use:',
-                '<ul>',
-                '    <li>Start cluster.</li>',
-                '    <li>Start this utility and wait while load complete.</li>',
-                '</ul>',
-                '',
-                '@param args Command line arguments, none required.',
-                '@throws Exception If failed.'
-            );
-            sb.startBlock('public static void main(String[] args) throws Exception {');
+            sb.emptyLine();
 
-            sb.startBlock(`try (Ignite ignite = Ignition.start(${cfgRef})) {`);
+            sb.append('runDemo(ignite);');
+        }
 
-            sb.append('System.out.println(">>> Loading caches...");');
+        sb.endBlock('}');
 
-            sb.emptyLine();
+        sb.endBlock('}');
 
-            _.forEach(caches, (cache) => {
-                sb.append('System.out.println(">>> Loading cache: ' + cache.name + '");');
-                sb.append('ignite.cache("' + cache.name + '").loadCache(null);');
+        return sb.asString();
+    }
 
-                sb.emptyLine();
-            });
+    /**
+     * Function to generate java class for load caches.
+     *
+     * @param caches Caches to load.
+     * @param pkg Class package name.
+     * @param clsName Class name.
+     * @param {String} cfgRef Config.
+     */
+    static loadCaches(caches, pkg, clsName, cfgRef) {
+        const sb = new StringBuilder();
+
+        sb.append(`package ${pkg};`)
+            .emptyLine();
+
+        const imports = ['org.apache.ignite.Ignition', 'org.apache.ignite.Ignite'];
+
+        _.forEach(this._prepareImports(imports), (cls) => sb.append(`import ${cls};`));
+        sb.emptyLine();
+
+        this.mainComment(sb);
+        sb.startBlock(`public class ${clsName} {`);
+
+        this.commentBlock(sb,
+            '<p>',
+            'Utility to load caches from database.',
+            '<p>',
+            'How to use:',
+            '<ul>',
+            '    <li>Start cluster.</li>',
+            '    <li>Start this utility and wait while load complete.</li>',
+            '</ul>',
+            '',
+            '@param args Command line arguments, none required.',
+            '@throws Exception If failed.'
+        );
+        sb.startBlock('public static void main(String[] args) throws Exception {');
+
+        sb.startBlock(`try (Ignite ignite = Ignition.start(${cfgRef})) {`);
+
+        sb.append('System.out.println(">>> Loading caches...");');
+
+        sb.emptyLine();
+
+        _.forEach(caches, (cache) => {
+            sb.append('System.out.println(">>> Loading cache: ' + cache.name + '");');
+            sb.append('ignite.cache("' + cache.name + '").loadCache(null);');
 
-            sb.append('System.out.println(">>> All caches loaded!");');
+            sb.emptyLine();
+        });
 
-            sb.endBlock('}');
+        sb.append('System.out.println(">>> All caches loaded!");');
 
-            sb.endBlock('}');
+        sb.endBlock('}');
 
-            sb.endBlock('}');
+        sb.endBlock('}');
 
-            return sb.asString();
-        }
+        sb.endBlock('}');
 
-        /**
-         * Checks if cluster has demo types.
-         *
-         * @param cluster Cluster to check.
-         * @param demo Is demo enabled.
-         * @returns {boolean} True if cluster has caches with demo types.
-         */
-        static isDemoConfigured(cluster, demo) {
-            return demo && _.find(cluster.caches, (cache) => _.find(cache.domains, (domain) => _.find(PREDEFINED_QUERIES, (desc) => domain.valueType.toUpperCase().endsWith(desc.type))));
-        }
+        return sb.asString();
     }
 
-    return JavaTransformer;
-}];
+    /**
+     * Checks if cluster has demo types.
+     *
+     * @param cluster Cluster to check.
+     * @param demo Is demo enabled.
+     * @returns {boolean} True if cluster has caches with demo types.
+     */
+    static isDemoConfigured(cluster, demo) {
+        return demo && _.find(cluster.caches, (cache) => _.find(cache.domains, (domain) => _.find(PREDEFINED_QUERIES, (desc) => domain.valueType.toUpperCase().endsWith(desc.type))));
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Pom.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js
similarity index 97%
rename from modules/web-console/frontend/app/modules/configuration/generator/Pom.service.js
rename to modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js
index db58532187780..2e017610e0e81 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Pom.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js
@@ -23,7 +23,7 @@ import POM_DEPENDENCIES from 'app/data/pom-dependencies.json';
 /**
  * Pom file generation entry point.
  */
-class GeneratorPom {
+export default class IgniteMavenGenerator {
     escapeId(s) {
         if (typeof (s) !== 'string')
             return s;
@@ -184,6 +184,9 @@ class GeneratorPom {
         this.addDependency(deps, 'org.apache.ignite', 'ignite-indexing', version);
         this.addDependency(deps, 'org.apache.ignite', 'ignite-rest-http', version);
 
+        if (_.get(cluster, 'deploymentSpi.kind') === 'URI')
+            this.addDependency(deps, 'org.apache.ignite', 'ignite-urideploy', version);
+
         let dep = POM_DEPENDENCIES[cluster.discovery.kind];
 
         if (dep)
@@ -229,5 +232,3 @@ class GeneratorPom {
         return sb;
     }
 }
-
-export default ['GeneratorPom', GeneratorPom];
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Properties.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Properties.service.js
index 49b4aa69df949..8a6a4715db439 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Properties.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Properties.service.js
@@ -20,7 +20,7 @@ import StringBuilder from './StringBuilder';
 /**
  * Properties generation entry point.
  */
-export default class PropertiesGenerator {
+export default class IgnitePropertiesGenerator {
     _collectProperties(bean) {
         const props = [];
 
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Readme.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Readme.service.js
index 7043807fb818c..0aa34ee2878b9 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Readme.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Readme.service.js
@@ -20,7 +20,7 @@ import StringBuilder from './StringBuilder';
 /**
  * Properties generation entry point.
  */
-export default class ReadmeGenerator {
+export default class IgniteReadmeGenerator {
     header(sb) {
         sb.append('Content of this folder was generated by Apache Ignite Web Console');
         sb.append('=================================================================');
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/SharpTransformer.service.js b/modules/web-console/frontend/app/modules/configuration/generator/SharpTransformer.service.js
index 19043f6f8a57a..6e6bffe92298c 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/SharpTransformer.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/SharpTransformer.service.js
@@ -19,225 +19,238 @@ import _ from 'lodash';
 import AbstractTransformer from './AbstractTransformer';
 import StringBuilder from './StringBuilder';
 
-export default ['JavaTypes', 'IgnitePlatformGenerator', (JavaTypes, generator) => {
-    return class SharpTransformer extends AbstractTransformer {
-        static generator = generator;
-
-        static commentBlock(sb, ...lines) {
-            _.forEach(lines, (line) => sb.append(`// ${line}`));
-        }
-
-        static doc(sb, ...lines) {
-            sb.append('/// <summary>');
-            _.forEach(lines, (line) => sb.append(`/// ${line}`));
-            sb.append('/// </summary>');
-        }
-
-        static mainComment(sb) {
-            return this.doc(sb, sb.generatedBy());
-        }
-
-        /**
-         *
-         * @param {Array.<String>} sb
-         * @param {Bean} bean
-         */
-        static _defineBean(sb, bean) {
-            const shortClsName = JavaTypes.shortClassName(bean.clsName);
-
-            sb.append(`var ${bean.id} = new ${shortClsName}();`);
-        }
-
-        /**
-         * @param {StringBuilder} sb
-         * @param {Bean} parent
-         * @param {Bean} propertyName
-         * @param {String|Bean} value
-         * @private
-         */
-        static _setProperty(sb, parent, propertyName, value) {
-            sb.append(`${parent.id}.${_.upperFirst(propertyName)} = ${value};`);
-        }
-
-        /**
-         *
-         * @param {StringBuilder} sb
-         * @param {Bean} parent
-         * @param {String} propertyName
-         * @param {Bean} bean
-         * @private
-         */
-        static _setBeanProperty(sb, parent, propertyName, bean) {
-            sb.append(`${parent.id}.${_.upperFirst(propertyName)} = ${bean.id};`);
-        }
-
-        static _toObject(clsName, val) {
-            const items = _.isArray(val) ? val : [val];
-
-            return _.map(items, (item, idx) => {
-                if (_.isNil(item))
-                    return 'null';
-
-                const shortClsName = JavaTypes.shortClassName(clsName);
-
-                switch (shortClsName) {
-                    // case 'byte':
-                    //     return `(byte) ${item}`;
-                    // case 'Serializable':
-                    case 'String':
-                        if (items.length > 1)
-                            return `"${item}"${idx !== items.length - 1 ? ' +' : ''}`;
-
-                        return `"${item}"`;
-                    // case 'Path':
-                    //     return `"${item.replace(/\\/g, '\\\\')}"`;
-                    // case 'Class':
-                    //     return `${this.shortClassName(item)}.class`;
-                    // case 'UUID':
-                    //     return `UUID.fromString("${item}")`;
-                    // case 'PropertyChar':
-                    //     return `props.getProperty("${item}").toCharArray()`;
-                    // case 'Property':
-                    //     return `props.getProperty("${item}")`;
-                    // case 'Bean':
-                    //     if (item.isComplex())
-                    //         return item.id;
+import ConfigurationGenerator from './ConfigurationGenerator';
+
+import ClusterDefaults from './defaults/Cluster.service';
+import CacheDefaults from './defaults/Cache.service';
+import IGFSDefaults from './defaults/IGFS.service';
+
+import JavaTypes from '../../../services/JavaTypes.service';
+
+const generator = new ConfigurationGenerator();
+
+const clusterDflts = new ClusterDefaults();
+const cacheDflts = new CacheDefaults();
+const igfsDflts = new IGFSDefaults();
+
+const javaTypes = new JavaTypes(clusterDflts, cacheDflts, igfsDflts);
+
+export default class SharpTransformer extends AbstractTransformer {
+    static generator = generator;
+
+    static commentBlock(sb, ...lines) {
+        _.forEach(lines, (line) => sb.append(`// ${line}`));
+    }
+
+    static doc(sb, ...lines) {
+        sb.append('/// <summary>');
+        _.forEach(lines, (line) => sb.append(`/// ${line}`));
+        sb.append('/// </summary>');
+    }
+
+    static mainComment(sb) {
+        return this.doc(sb, sb.generatedBy());
+    }
+
+    /**
+     *
+     * @param {Array.<String>} sb
+     * @param {Bean} bean
+     */
+    static _defineBean(sb, bean) {
+        const shortClsName = javaTypes.shortClassName(bean.clsName);
+
+        sb.append(`var ${bean.id} = new ${shortClsName}();`);
+    }
+
+    /**
+     * @param {StringBuilder} sb
+     * @param {Bean} parent
+     * @param {Bean} propertyName
+     * @param {String|Bean} value
+     * @private
+     */
+    static _setProperty(sb, parent, propertyName, value) {
+        sb.append(`${parent.id}.${_.upperFirst(propertyName)} = ${value};`);
+    }
+
+    /**
+     *
+     * @param {StringBuilder} sb
+     * @param {Bean} parent
+     * @param {String} propertyName
+     * @param {Bean} bean
+     * @private
+     */
+    static _setBeanProperty(sb, parent, propertyName, bean) {
+        sb.append(`${parent.id}.${_.upperFirst(propertyName)} = ${bean.id};`);
+    }
+
+    static _toObject(clsName, val) {
+        const items = _.isArray(val) ? val : [val];
+
+        return _.map(items, (item, idx) => {
+            if (_.isNil(item))
+                return 'null';
+
+            const shortClsName = javaTypes.shortClassName(clsName);
+
+            switch (shortClsName) {
+                // case 'byte':
+                //     return `(byte) ${item}`;
+                // case 'Serializable':
+                case 'String':
+                    if (items.length > 1)
+                        return `"${item}"${idx !== items.length - 1 ? ' +' : ''}`;
+
+                    return `"${item}"`;
+                // case 'Path':
+                //     return `"${item.replace(/\\/g, '\\\\')}"`;
+                // case 'Class':
+                //     return `${this.shortClassName(item)}.class`;
+                // case 'UUID':
+                //     return `UUID.fromString("${item}")`;
+                // case 'PropertyChar':
+                //     return `props.getProperty("${item}").toCharArray()`;
+                // case 'Property':
+                //     return `props.getProperty("${item}")`;
+                // case 'Bean':
+                //     if (item.isComplex())
+                //         return item.id;
+                //
+                //     return this._newBean(item);
+                default:
+                    if (javaTypes.nonEnum(shortClsName))
+                        return item;
+
+                    return `${shortClsName}.${item}`;
+            }
+        });
+    }
+
+    /**
+     *
+     * @param {StringBuilder} sb
+     * @param {Bean} bean
+     * @returns {Array}
+     */
+    static _setProperties(sb = new StringBuilder(), bean) {
+        _.forEach(bean.properties, (prop) => {
+            switch (prop.clsName) {
+                case 'ICollection':
+                    // const implClsName = JavaTypes.shortClassName(prop.implClsName);
+
+                    const colTypeClsName = javaTypes.shortClassName(prop.typeClsName);
+
+                    if (colTypeClsName === 'String') {
+                        const items = this._toObject(colTypeClsName, prop.items);
+
+                        sb.append(`${bean.id}.${_.upperFirst(prop.name)} = new {${items.join(', ')}};`);
+                    }
+                    // else {
+                    //     if (_.includes(vars, prop.id))
+                    //         sb.append(`${prop.id} = new ${implClsName}<>();`);
+                    //     else {
+                    //         vars.push(prop.id);
                     //
-                    //     return this._newBean(item);
-                    default:
-                        if (JavaTypes.nonEnum(shortClsName))
-                            return item;
-
-                        return `${shortClsName}.${item}`;
-                }
-            });
-        }
-
-        /**
-         *
-         * @param {StringBuilder} sb
-         * @param {Bean} bean
-         * @returns {Array}
-         */
-        static _setProperties(sb = new StringBuilder(), bean) {
-            _.forEach(bean.properties, (prop) => {
-                switch (prop.clsName) {
-                    case 'ICollection':
-                        // const implClsName = JavaTypes.shortClassName(prop.implClsName);
-
-                        const colTypeClsName = JavaTypes.shortClassName(prop.typeClsName);
-
-                        if (colTypeClsName === 'String') {
-                            const items = this._toObject(colTypeClsName, prop.items);
-
-                            sb.append(`${bean.id}.${_.upperFirst(prop.name)} = new {${items.join(', ')}};`);
-                        }
-                        // else {
-                        //     if (_.includes(vars, prop.id))
-                        //         sb.append(`${prop.id} = new ${implClsName}<>();`);
-                        //     else {
-                        //         vars.push(prop.id);
-                        //
-                        //         sb.append(`${clsName}<${colTypeClsName}> ${prop.id} = new ${implClsName}<>();`);
-                        //     }
-                        //
-                        //     sb.emptyLine();
-                        //
-                        //     if (nonBean) {
-                        //         const items = this._toObject(colTypeClsName, prop.items);
-                        //
-                        //         _.forEach(items, (item) => {
-                        //             sb.append(`${prop.id}.add("${item}");`);
-                        //
-                        //             sb.emptyLine();
-                        //         });
-                        //     }
-                        //     else {
-                        //         _.forEach(prop.items, (item) => {
-                        //             this.constructBean(sb, item, vars, limitLines);
-                        //
-                        //             sb.append(`${prop.id}.add(${item.id});`);
-                        //
-                        //             sb.emptyLine();
-                        //         });
-                        //
-                        //         this._setProperty(sb, bean.id, prop.name, prop.id);
-                        //     }
-                        // }
-
-                        break;
-
-                    case 'Bean':
-                        const nestedBean = prop.value;
-
-                        this._defineBean(sb, nestedBean);
-
-                        sb.emptyLine();
-
-                        this._setProperties(sb, nestedBean);
-
-                        sb.emptyLine();
-
-                        this._setBeanProperty(sb, bean, prop.name, nestedBean);
-
-                        break;
-                    default:
-                        this._setProperty(sb, bean, prop.name, this._toObject(prop.clsName, prop.value));
-                }
-            });
-
-            return sb;
-        }
-
-        /**
-         * Build Java startup class with configuration.
-         *
-         * @param {Bean} cfg
-         * @param pkg Package name.
-         * @param clsName Class name for generate factory class otherwise generate code snippet.
-         * @param clientNearCfg Optional near cache configuration for client node.
-         * @returns {String}
-         */
-        static toClassFile(cfg, pkg, clsName) {
-            const sb = new StringBuilder();
-
-            sb.startBlock(`namespace ${pkg}`, '{');
-
-            _.forEach(_.sortBy(cfg.collectClasses()), (cls) => sb.append(`using ${cls};`));
-            sb.emptyLine();
-
-
-            this.mainComment(sb);
-            sb.startBlock(`public class ${clsName}`, '{');
-
-            this.doc(sb, 'Configure grid.');
-            sb.startBlock('public static IgniteConfiguration CreateConfiguration()', '{');
-
-            this._defineBean(sb, cfg);
-
-            sb.emptyLine();
-
-            this._setProperties(sb, cfg);
+                    //         sb.append(`${clsName}<${colTypeClsName}> ${prop.id} = new ${implClsName}<>();`);
+                    //     }
+                    //
+                    //     sb.emptyLine();
+                    //
+                    //     if (nonBean) {
+                    //         const items = this._toObject(colTypeClsName, prop.items);
+                    //
+                    //         _.forEach(items, (item) => {
+                    //             sb.append(`${prop.id}.add("${item}");`);
+                    //
+                    //             sb.emptyLine();
+                    //         });
+                    //     }
+                    //     else {
+                    //         _.forEach(prop.items, (item) => {
+                    //             this.constructBean(sb, item, vars, limitLines);
+                    //
+                    //             sb.append(`${prop.id}.add(${item.id});`);
+                    //
+                    //             sb.emptyLine();
+                    //         });
+                    //
+                    //         this._setProperty(sb, bean.id, prop.name, prop.id);
+                    //     }
+                    // }
+
+                    break;
+
+                case 'Bean':
+                    const nestedBean = prop.value;
+
+                    this._defineBean(sb, nestedBean);
+
+                    sb.emptyLine();
+
+                    this._setProperties(sb, nestedBean);
+
+                    sb.emptyLine();
+
+                    this._setBeanProperty(sb, bean, prop.name, nestedBean);
+
+                    break;
+                default:
+                    this._setProperty(sb, bean, prop.name, this._toObject(prop.clsName, prop.value));
+            }
+        });
+
+        return sb;
+    }
+
+    /**
+     * Build Java startup class with configuration.
+     *
+     * @param {Bean} cfg
+     * @param pkg Package name.
+     * @param clsName Class name for generate factory class otherwise generate code snippet.
+     * @returns {String}
+     */
+    static toClassFile(cfg, pkg, clsName) {
+        const sb = new StringBuilder();
+
+        sb.startBlock(`namespace ${pkg}`, '{');
+
+        _.forEach(_.sortBy(cfg.collectClasses()), (cls) => sb.append(`using ${cls};`));
+        sb.emptyLine();
+
+
+        this.mainComment(sb);
+        sb.startBlock(`public class ${clsName}`, '{');
+
+        this.doc(sb, 'Configure grid.');
+        sb.startBlock('public static IgniteConfiguration CreateConfiguration()', '{');
+
+        this._defineBean(sb, cfg);
+
+        sb.emptyLine();
+
+        this._setProperties(sb, cfg);
 
-            sb.emptyLine();
+        sb.emptyLine();
 
-            sb.append(`return ${cfg.id};`);
+        sb.append(`return ${cfg.id};`);
 
-            sb.endBlock('}');
+        sb.endBlock('}');
 
-            sb.endBlock('}');
+        sb.endBlock('}');
 
-            sb.endBlock('}');
+        sb.endBlock('}');
 
-            return sb.asString();
-        }
+        return sb.asString();
+    }
 
-        static generateSection(bean) {
-            const sb = new StringBuilder();
+    static generateSection(bean) {
+        const sb = new StringBuilder();
 
-            this._setProperties(sb, bean);
+        this._setProperties(sb, bean);
 
-            return sb.asString();
-        }
-    };
-}];
+        return sb.asString();
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/SpringTransformer.service.js b/modules/web-console/frontend/app/modules/configuration/generator/SpringTransformer.service.js
index 73df25e000e53..b2345759886f8 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/SpringTransformer.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/SpringTransformer.service.js
@@ -20,314 +20,311 @@ import _ from 'lodash';
 import AbstractTransformer from './AbstractTransformer';
 import StringBuilder from './StringBuilder';
 
-const escapeXml = (str) => {
-    return str.replace(/&/g, '&amp;')
-        .replace(/"/g, '&quot;')
-        .replace(/'/g, '&apos;')
-        .replace(/>/g, '&gt;')
-        .replace(/</g, '&lt;');
-};
-
-export default ['JavaTypes', 'igniteEventGroups', 'IgniteConfigurationGenerator', (JavaTypes, eventGroups, generator) => {
-    return class SpringTransformer extends AbstractTransformer {
-        static generator = generator;
-
-        static commentBlock(sb, ...lines) {
-            if (lines.length > 1) {
-                sb.append('<!--');
-
-                _.forEach(lines, (line) => sb.append(`  ${line}`));
-
-                sb.append('-->');
-            }
-            else
-                sb.append(`<!-- ${_.head(lines)} -->`);
+export default class IgniteSpringTransformer extends AbstractTransformer {
+    static escapeXml(str) {
+        return str.replace(/&/g, '&amp;')
+            .replace(/"/g, '&quot;')
+            .replace(/'/g, '&apos;')
+            .replace(/>/g, '&gt;')
+            .replace(/</g, '&lt;');
+    }
+
+    static commentBlock(sb, ...lines) {
+        if (lines.length > 1) {
+            sb.append('<!--');
+
+            _.forEach(lines, (line) => sb.append(`  ${line}`));
+
+            sb.append('-->');
         }
+        else
+            sb.append(`<!-- ${_.head(lines)} -->`);
+    }
 
-        static appendBean(sb, bean, appendId) {
-            const beanTags = [];
-
-            if (appendId)
-                beanTags.push(`id="${bean.id}"`);
-
-            beanTags.push(`class="${bean.clsName}"`);
-
-            if (bean.factoryMtd)
-                beanTags.push(`factory-method="${bean.factoryMtd}"`);
-
-            sb.startBlock(`<bean ${beanTags.join(' ')}>`);
-
-            _.forEach(bean.arguments, (arg) => {
-                if (arg.clsName === 'MAP') {
-                    sb.startBlock('<constructor-arg>');
-                    this._constructMap(sb, arg);
-                    sb.endBlock('</constructor-arg>');
-                }
-                else if (_.isNil(arg.value)) {
-                    sb.startBlock('<constructor-arg>');
-                    sb.append('<null/>');
-                    sb.endBlock('</constructor-arg>');
-                }
-                else if (arg.constant) {
-                    sb.startBlock('<constructor-arg>');
-                    sb.append(`<util:constant static-field="${arg.clsName}.${arg.value}"/>`);
-                    sb.endBlock('</constructor-arg>');
-                }
-                else if (arg.clsName === 'BEAN') {
-                    sb.startBlock('<constructor-arg>');
-                    this.appendBean(sb, arg.value);
-                    sb.endBlock('</constructor-arg>');
-                }
-                else
-                    sb.append(`<constructor-arg value="${this._toObject(arg.clsName, arg.value)}"/>`);
-            });
+    static appendBean(sb, bean, appendId) {
+        const beanTags = [];
 
-            this._setProperties(sb, bean);
+        if (appendId)
+            beanTags.push(`id="${bean.id}"`);
 
-            sb.endBlock('</bean>');
-        }
+        beanTags.push(`class="${bean.clsName}"`);
 
-        static _toObject(clsName, items) {
-            return _.map(_.isArray(items) ? items : [items], (item) => {
-                switch (clsName) {
-                    case 'PROPERTY':
-                    case 'PROPERTY_CHAR':
-                    case 'PROPERTY_INT':
-                        return `\${${item}}`;
-                    case 'java.lang.Class':
-                        return JavaTypes.fullClassName(item);
-                    case 'long':
-                        return `${item}L`;
-                    case 'java.lang.String':
-                        return escapeXml(item);
-                    default:
-                        return item;
-                }
-            });
-        }
+        if (bean.factoryMtd)
+            beanTags.push(`factory-method="${bean.factoryMtd}"`);
 
-        static _isBean(clsName) {
-            return JavaTypes.nonBuiltInClass(clsName) && JavaTypes.nonEnum(clsName) && _.includes(clsName, '.');
-        }
+        sb.startBlock(`<bean ${beanTags.join(' ')}>`);
 
-        static _setCollection(sb, prop) {
-            sb.startBlock(`<property name="${prop.name}">`);
-            sb.startBlock('<list>');
+        _.forEach(bean.arguments, (arg) => {
+            if (arg.clsName === 'MAP') {
+                sb.startBlock('<constructor-arg>');
+                this._constructMap(sb, arg);
+                sb.endBlock('</constructor-arg>');
+            }
+            else if (_.isNil(arg.value)) {
+                sb.startBlock('<constructor-arg>');
+                sb.append('<null/>');
+                sb.endBlock('</constructor-arg>');
+            }
+            else if (arg.constant) {
+                sb.startBlock('<constructor-arg>');
+                sb.append(`<util:constant static-field="${arg.clsName}.${arg.value}"/>`);
+                sb.endBlock('</constructor-arg>');
+            }
+            else if (arg.clsName === 'BEAN') {
+                sb.startBlock('<constructor-arg>');
+                this.appendBean(sb, arg.value);
+                sb.endBlock('</constructor-arg>');
+            }
+            else
+                sb.append(`<constructor-arg value="${this._toObject(arg.clsName, arg.value)}"/>`);
+        });
+
+        this._setProperties(sb, bean);
+
+        sb.endBlock('</bean>');
+    }
+
+    static _toObject(clsName, items) {
+        return _.map(_.isArray(items) ? items : [items], (item) => {
+            switch (clsName) {
+                case 'PROPERTY':
+                case 'PROPERTY_CHAR':
+                case 'PROPERTY_INT':
+                    return `\${${item}}`;
+                case 'java.lang.Class':
+                    return this.javaTypes.fullClassName(item);
+                case 'long':
+                    return `${item}L`;
+                case 'java.lang.String':
+                case 'PATH':
+                    return this.escapeXml(item);
+                default:
+                    return item;
+            }
+        });
+    }
 
-            _.forEach(prop.items, (item, idx) => {
-                if (this._isBean(prop.typeClsName)) {
-                    if (idx !== 0)
-                        sb.emptyLine();
+    static _isBean(clsName) {
+        return this.javaTypes.nonBuiltInClass(clsName) && this.javaTypes.nonEnum(clsName) && _.includes(clsName, '.');
+    }
 
-                    this.appendBean(sb, item);
-                }
-                else
-                    sb.append(`<value>${item}</value>`);
-            });
+    static _setCollection(sb, prop) {
+        sb.startBlock(`<property name="${prop.name}">`);
+        sb.startBlock('<list>');
 
-            sb.endBlock('</list>');
-            sb.endBlock('</property>');
-        }
+        _.forEach(prop.items, (item, idx) => {
+            if (this._isBean(prop.typeClsName)) {
+                if (idx !== 0)
+                    sb.emptyLine();
 
-        static _constructMap(sb, map) {
-            sb.startBlock('<map>');
+                this.appendBean(sb, item);
+            }
+            else
+                sb.append(`<value>${item}</value>`);
+        });
 
-            _.forEach(map.entries, (entry) => {
-                const key = entry[map.keyField];
-                const val = entry[map.valField];
+        sb.endBlock('</list>');
+        sb.endBlock('</property>');
+    }
 
-                const isKeyBean = this._isBean(map.keyClsName);
-                const isValBean = this._isBean(map.valClsName);
+    static _constructMap(sb, map) {
+        sb.startBlock('<map>');
 
+        _.forEach(map.entries, (entry) => {
+            const key = entry[map.keyField];
+            const val = entry[map.valField];
 
-                if (isKeyBean || isValBean) {
-                    sb.startBlock('<entry>');
+            const isKeyBean = this._isBean(map.keyClsName);
+            const isValBean = this._isBean(map.valClsName);
 
-                    sb.startBlock('<key>');
-                    if (isKeyBean)
-                        this.appendBean(sb, key);
-                    else
-                        sb.append(this._toObject(map.keyClsName, key));
-                    sb.endBlock('</key>');
 
-                    sb.startBlock('<value>');
-                    if (isValBean)
-                        this.appendBean(sb, val);
-                    else
-                        sb.append(this._toObject(map.valClsName, val));
-                    sb.endBlock('</value>');
+            if (isKeyBean || isValBean) {
+                sb.startBlock('<entry>');
 
-                    sb.endBlock('</entry>');
-                }
+                sb.startBlock('<key>');
+                if (isKeyBean)
+                    this.appendBean(sb, key);
                 else
-                    sb.append(`<entry key="${this._toObject(map.keyClsName, key)}" value="${this._toObject(map.valClsName, val)}"/>`);
-            });
+                    sb.append(this._toObject(map.keyClsName, key));
+                sb.endBlock('</key>');
 
-            sb.endBlock('</map>');
-        }
+                sb.startBlock('<value>');
+                if (isValBean)
+                    this.appendBean(sb, val);
+                else
+                    sb.append(this._toObject(map.valClsName, val));
+                sb.endBlock('</value>');
 
-        /**
-         *
-         * @param {StringBuilder} sb
-         * @param {Bean} bean
-         * @returns {StringBuilder}
-         */
-        static _setProperties(sb, bean) {
-            _.forEach(bean.properties, (prop, idx) => {
-                switch (prop.clsName) {
-                    case 'DATA_SOURCE':
-                        const valAttr = prop.name === 'dataSource' ? 'ref' : 'value';
+                sb.endBlock('</entry>');
+            }
+            else
+                sb.append(`<entry key="${this._toObject(map.keyClsName, key)}" value="${this._toObject(map.valClsName, val)}"/>`);
+        });
 
-                        sb.append(`<property name="${prop.name}" ${valAttr}="${prop.id}"/>`);
+        sb.endBlock('</map>');
+    }
 
-                        break;
-                    case 'EVENT_TYPES':
-                        sb.startBlock(`<property name="${prop.name}">`);
+    /**
+     *
+     * @param {StringBuilder} sb
+     * @param {Bean} bean
+     * @returns {StringBuilder}
+     */
+    static _setProperties(sb, bean) {
+        _.forEach(bean.properties, (prop, idx) => {
+            switch (prop.clsName) {
+                case 'DATA_SOURCE':
+                    const valAttr = prop.name === 'dataSource' ? 'ref' : 'value';
 
-                        if (prop.eventTypes.length === 1) {
-                            const evtGrp = _.find(eventGroups, {value: _.head(prop.eventTypes)});
+                    sb.append(`<property name="${prop.name}" ${valAttr}="${prop.id}"/>`);
 
-                            evtGrp && sb.append(`<util:constant static-field="${evtGrp.class}.${evtGrp.value}"/>`);
-                        }
-                        else {
-                            sb.startBlock('<list>');
+                    break;
+                case 'EVENT_TYPES':
+                    sb.startBlock(`<property name="${prop.name}">`);
 
-                            _.forEach(prop.eventTypes, (item, ix) => {
-                                ix > 0 && sb.emptyLine();
+                    if (prop.eventTypes.length === 1) {
+                        const evtGrp = _.find(this.eventGroups, {value: _.head(prop.eventTypes)});
 
-                                const evtGrp = _.find(eventGroups, {value: item});
+                        evtGrp && sb.append(`<util:constant static-field="${evtGrp.class}.${evtGrp.value}"/>`);
+                    }
+                    else {
+                        sb.startBlock('<list>');
 
-                                if (evtGrp) {
-                                    sb.append(`<!-- EventType.${item} -->`);
+                        _.forEach(prop.eventTypes, (item, ix) => {
+                            ix > 0 && sb.emptyLine();
 
-                                    _.forEach(evtGrp.events, (event) =>
-                                        sb.append(`<util:constant static-field="${evtGrp.class}.${event}"/>`));
-                                }
-                            });
+                            const evtGrp = _.find(this.eventGroups, {value: item});
 
-                            sb.endBlock('</list>');
-                        }
+                            if (evtGrp) {
+                                sb.append(`<!-- EventType.${item} -->`);
 
-                        sb.endBlock('</property>');
+                                _.forEach(evtGrp.events, (event) =>
+                                    sb.append(`<util:constant static-field="${evtGrp.class}.${event}"/>`));
+                            }
+                        });
 
-                        break;
-                    case 'ARRAY':
-                    case 'COLLECTION':
-                        this._setCollection(sb, prop);
+                        sb.endBlock('</list>');
+                    }
 
-                        break;
-                    case 'MAP':
-                        sb.startBlock(`<property name="${prop.name}">`);
+                    sb.endBlock('</property>');
 
-                        this._constructMap(sb, prop);
+                    break;
+                case 'ARRAY':
+                case 'COLLECTION':
+                    this._setCollection(sb, prop);
 
-                        sb.endBlock('</property>');
+                    break;
+                case 'MAP':
+                    sb.startBlock(`<property name="${prop.name}">`);
 
-                        break;
-                    case 'java.util.Properties':
-                        sb.startBlock(`<property name="${prop.name}">`);
-                        sb.startBlock('<props>');
+                    this._constructMap(sb, prop);
 
-                        _.forEach(prop.entries, (entry) => {
-                            sb.append(`<prop key="${entry.name}">${entry.value}</prop>`);
-                        });
+                    sb.endBlock('</property>');
 
-                        sb.endBlock('</props>');
-                        sb.endBlock('</property>');
+                    break;
+                case 'java.util.Properties':
+                    sb.startBlock(`<property name="${prop.name}">`);
+                    sb.startBlock('<props>');
 
-                        break;
-                    case 'BEAN':
-                        sb.startBlock(`<property name="${prop.name}">`);
+                    _.forEach(prop.entries, (entry) => {
+                        sb.append(`<prop key="${entry.name}">${entry.value}</prop>`);
+                    });
 
-                        this.appendBean(sb, prop.value);
+                    sb.endBlock('</props>');
+                    sb.endBlock('</property>');
 
-                        sb.endBlock('</property>');
+                    break;
+                case 'BEAN':
+                    sb.startBlock(`<property name="${prop.name}">`);
 
-                        break;
-                    default:
-                        sb.append(`<property name="${prop.name}" value="${this._toObject(prop.clsName, prop.value)}"/>`);
-                }
+                    this.appendBean(sb, prop.value);
 
-                this._emptyLineIfNeeded(sb, bean.properties, idx);
-            });
+                    sb.endBlock('</property>');
 
-            return sb;
-        }
+                    break;
+                default:
+                    sb.append(`<property name="${prop.name}" value="${this._toObject(prop.clsName, prop.value)}"/>`);
+            }
 
-        /**
-         * Build final XML.
-         *
-         * @param {Bean} cfg Ignite configuration.
-         * @param {Boolean} clientNearCaches
-         * @returns {StringBuilder}
-         */
-        static igniteConfiguration(cfg, clientNearCaches) {
-            const sb = new StringBuilder();
-
-            // 0. Add header.
-            sb.append('<?xml version="1.0" encoding="UTF-8"?>');
-            sb.emptyLine();
+            this._emptyLineIfNeeded(sb, bean.properties, idx);
+        });
+
+        return sb;
+    }
+
+    /**
+     * Build final XML.
+     *
+     * @param {Bean} cfg Ignite configuration.
+     * @param {Boolean} clientNearCaches
+     * @returns {StringBuilder}
+     */
+    static igniteConfiguration(cfg, clientNearCaches) {
+        const sb = new StringBuilder();
+
+        // 0. Add header.
+        sb.append('<?xml version="1.0" encoding="UTF-8"?>');
+        sb.emptyLine();
+
+        this.mainComment(sb);
+        sb.emptyLine();
+
+        // 1. Start beans section.
+        sb.startBlock([
+            '<beans xmlns="http://www.springframework.org/schema/beans"',
+            '       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"',
+            '       xmlns:util="http://www.springframework.org/schema/util"',
+            '       xsi:schemaLocation="http://www.springframework.org/schema/beans',
+            '                           http://www.springframework.org/schema/beans/spring-beans.xsd',
+            '                           http://www.springframework.org/schema/util',
+            '                           http://www.springframework.org/schema/util/spring-util.xsd">']);
+
+        // 2. Add external property file
+        if (this.hasProperties(cfg)) {
+            this.commentBlock(sb, 'Load external properties file.');
+
+            sb.startBlock('<bean id="placeholderConfig" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">');
+            sb.append('<property name="location" value="classpath:secret.properties"/>');
+            sb.endBlock('</bean>');
 
-            this.mainComment(sb);
             sb.emptyLine();
+        }
 
-            // 1. Start beans section.
-            sb.startBlock([
-                '<beans xmlns="http://www.springframework.org/schema/beans"',
-                '       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"',
-                '       xmlns:util="http://www.springframework.org/schema/util"',
-                '       xsi:schemaLocation="http://www.springframework.org/schema/beans',
-                '                           http://www.springframework.org/schema/beans/spring-beans.xsd',
-                '                           http://www.springframework.org/schema/util',
-                '                           http://www.springframework.org/schema/util/spring-util.xsd">']);
+        // 3. Add data sources.
+        const dataSources = this.collectDataSources(cfg);
 
-            // 2. Add external property file
-            if (this.hasProperties(cfg)) {
-                this.commentBlock(sb, 'Load external properties file.');
+        if (dataSources.length) {
+            this.commentBlock(sb, 'Data source beans will be initialized from external properties file.');
 
-                sb.startBlock('<bean id="placeholderConfig" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">');
-                sb.append('<property name="location" value="classpath:secret.properties"/>');
-                sb.endBlock('</bean>');
+            _.forEach(dataSources, (ds) => {
+                this.appendBean(sb, ds, true);
 
                 sb.emptyLine();
-            }
-
-            // 3. Add data sources.
-            const dataSources = this.collectDataSources(cfg);
-
-            if (dataSources.length) {
-                this.commentBlock(sb, 'Data source beans will be initialized from external properties file.');
-
-                _.forEach(dataSources, (ds) => {
-                    this.appendBean(sb, ds, true);
-
-                    sb.emptyLine();
-                });
-            }
+            });
+        }
 
-            _.forEach(clientNearCaches, (cache) => {
-                this.commentBlock(sb, 'Configuration of near cache for cache "' + cache.name + '"');
+        _.forEach(clientNearCaches, (cache) => {
+            this.commentBlock(sb, `Configuration of near cache for cache "${cache.name}"`);
 
-                this.appendBean(sb, generator.cacheNearClient(cache), true);
+            this.appendBean(sb, this.generator.cacheNearClient(cache), true);
 
-                sb.emptyLine();
-            });
+            sb.emptyLine();
+        });
 
-            // 3. Add main content.
-            this.appendBean(sb, cfg);
+        // 3. Add main content.
+        this.appendBean(sb, cfg);
 
-            // 4. Close beans section.
-            sb.endBlock('</beans>');
+        // 4. Close beans section.
+        sb.endBlock('</beans>');
 
-            return sb;
-        }
+        return sb;
+    }
 
-        static cluster(cluster, client) {
-            const cfg = generator.igniteConfiguration(cluster, client);
+    static cluster(cluster, client) {
+        const cfg = this.generator.igniteConfiguration(cluster, client);
 
-            const clientNearCaches = client ? _.filter(cluster.caches, (cache) => _.get(cache, 'clientNearConfiguration.enabled')) : [];
+        const clientNearCaches = client ? _.filter(cluster.caches, (cache) => _.get(cache, 'clientNearConfiguration.enabled')) : [];
 
-            return this.igniteConfiguration(cfg, clientNearCaches);
-        }
-    };
-}];
+        return this.igniteConfiguration(cfg, clientNearCaches);
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/defaults/cache.platform.provider.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.platform.service.js
similarity index 88%
rename from modules/web-console/frontend/app/modules/configuration/generator/defaults/cache.platform.provider.js
rename to modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.platform.service.js
index f06e11b5f57be..eeac3a03f7901 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/defaults/cache.platform.provider.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.platform.service.js
@@ -49,12 +49,8 @@ const DFLT_CACHE = {
     }
 };
 
-export default function() {
-    this.append = (dflts) => {
-        _.merge(DFLT_CACHE, dflts);
-    };
-
-    this.$get = ['igniteCacheDefaults', (cacheDefaults) => {
-        return _.merge({}, cacheDefaults, DFLT_CACHE);
-    }];
+export default class IgniteCachePlatformDefaults {
+    constructor() {
+        Object.assign(this, DFLT_CACHE);
+    }
 }
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/defaults/cache.provider.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.service.js
similarity index 95%
rename from modules/web-console/frontend/app/modules/configuration/generator/defaults/cache.provider.js
rename to modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.service.js
index f50e493268412..14b315f4fdde1 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/defaults/cache.provider.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.service.js
@@ -15,8 +15,6 @@
  * limitations under the License.
  */
 
-import _ from 'lodash';
-
 const DFLT_CACHE = {
     cacheMode: {
         clsName: 'org.apache.ignite.cache.CacheMode'
@@ -126,12 +124,8 @@ const DFLT_CACHE = {
     }
 };
 
-export default function() {
-    this.append = (dflts) => {
-        _.merge(DFLT_CACHE, dflts);
-    };
-
-    this.$get = [() => {
-        return DFLT_CACHE;
-    }];
+export default class IgniteCacheDefaults {
+    constructor() {
+        Object.assign(this, DFLT_CACHE);
+    }
 }
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/defaults/cluster.platform.provider.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.platform.service.js
similarity index 84%
rename from modules/web-console/frontend/app/modules/configuration/generator/defaults/cluster.platform.provider.js
rename to modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.platform.service.js
index 582426ec8e7b4..b7019511355e9 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/defaults/cluster.platform.provider.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.platform.service.js
@@ -15,8 +15,6 @@
  * limitations under the License.
  */
 
-import _ from 'lodash';
-
 const enumValueMapper = (val) => _.capitalize(val);
 
 const DFLT_CLUSTER = {
@@ -38,12 +36,8 @@ const DFLT_CLUSTER = {
     }
 };
 
-export default function() {
-    this.append = (dflts) => {
-        _.merge(DFLT_CLUSTER, dflts);
-    };
-
-    this.$get = ['igniteClusterDefaults', (clusterDefaults) => {
-        return _.merge({}, clusterDefaults, DFLT_CLUSTER);
-    }];
+export default class IgniteClusterPlatformDefaults {
+    constructor() {
+        Object.assign(this, DFLT_CLUSTER);
+    }
 }
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/defaults/cluster.provider.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
similarity index 98%
rename from modules/web-console/frontend/app/modules/configuration/generator/defaults/cluster.provider.js
rename to modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
index 726581de810ea..6333ef96afcd1 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/defaults/cluster.provider.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
@@ -282,12 +282,8 @@ const DFLT_CLUSTER = {
     }
 };
 
-export default function() {
-    this.append = (dflts) => {
-        _.merge(DFLT_CLUSTER, dflts);
-    };
-
-    this.$get = [() => {
-        return DFLT_CLUSTER;
-    }];
+export default class IgniteClusterDefaults {
+    constructor() {
+        Object.assign(this, DFLT_CLUSTER);
+    }
 }
diff --git a/modules/web-console/frontend/app/modules/configuration/EventGroups.provider.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Event-groups.service.js
similarity index 78%
rename from modules/web-console/frontend/app/modules/configuration/EventGroups.provider.js
rename to modules/web-console/frontend/app/modules/configuration/generator/defaults/Event-groups.service.js
index 61f31885a9691..315da1f41fb1b 100644
--- a/modules/web-console/frontend/app/modules/configuration/EventGroups.provider.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Event-groups.service.js
@@ -15,16 +15,13 @@
  * limitations under the License.
  */
 
-// Events groups.
-import GROUPS from 'app/data/event-types.json';
-
-export default ['igniteEventGroups', function() {
-    const groups = GROUPS;
+import _ from 'lodash';
 
-    this.push = (data) => groups.push(data);
-
-    this.$get = [() => {
-        return groups;
-    }];
-}];
+// Events groups.
+import EVENT_GROUPS from 'app/data/event-groups.json';
 
+export default class IgniteEventGroups {
+    constructor() {
+        return _.clone(EVENT_GROUPS);
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/defaults/igfs.provider.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/IGFS.service.js
similarity index 92%
rename from modules/web-console/frontend/app/modules/configuration/generator/defaults/igfs.provider.js
rename to modules/web-console/frontend/app/modules/configuration/generator/defaults/IGFS.service.js
index c556336a3953b..985a56e10761f 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/defaults/igfs.provider.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/IGFS.service.js
@@ -57,12 +57,8 @@ const DFLT_IGFS = {
     }
 };
 
-export default function() {
-    this.append = (dflts) => {
-        _.merge(DFLT_IGFS, dflts);
-    };
-
-    this.$get = [() => {
-        return DFLT_IGFS;
-    }];
+export default class IgniteIGFSDefaults {
+    constructor() {
+        Object.assign(this, DFLT_IGFS);
+    }
 }
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/generator-common.js b/modules/web-console/frontend/app/modules/configuration/generator/generator-common.js
deleted file mode 100644
index d502c8a5c4605..0000000000000
--- a/modules/web-console/frontend/app/modules/configuration/generator/generator-common.js
+++ /dev/null
@@ -1,625 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Entry point for common functions for code generation.
-const $generatorCommon = {};
-
-// Add leading zero.
-$generatorCommon.addLeadingZero = function(numberStr, minSize) {
-    if (typeof (numberStr) !== 'string')
-        numberStr = String(numberStr);
-
-    while (numberStr.length < minSize)
-        numberStr = '0' + numberStr;
-
-    return numberStr;
-};
-
-// Format date to string.
-$generatorCommon.formatDate = function(date) {
-    const dd = $generatorCommon.addLeadingZero(date.getDate(), 2);
-    const mm = $generatorCommon.addLeadingZero(date.getMonth() + 1, 2);
-
-    const yyyy = date.getFullYear();
-
-    return mm + '/' + dd + '/' + yyyy + ' ' + $generatorCommon.addLeadingZero(date.getHours(), 2) + ':' + $generatorCommon.addLeadingZero(date.getMinutes(), 2);
-};
-
-/**
- * Generate title comment for XML, Java, ... files.
- *
- * @param sbj {string} What is generated.
- * @returns {string} Text to add as title comment in generated java class.
- */
-$generatorCommon.mainComment = function mainComment(sbj) {
-    return 'This ' + sbj + ' was generated by Ignite Web Console (' + $generatorCommon.formatDate(new Date()) + ')';
-};
-
-// Create result holder with service functions and properties for XML and java code generation.
-$generatorCommon.builder = function(deep) {
-    if (_.isNil($generatorCommon.JavaTypes))
-        $generatorCommon.JavaTypes = angular.element(document.getElementById('app')).injector().get('JavaTypes');
-
-    const res = [];
-
-    res.deep = deep || 0;
-    res.needEmptyLine = false;
-    res.lineStart = true;
-    res.datasources = [];
-    res.imports = {};
-    res.staticImports = {};
-    res.vars = {};
-
-    res.safeDeep = 0;
-    res.safeNeedEmptyLine = false;
-    res.safeImports = {};
-    res.safeDatasources = [];
-    res.safePoint = -1;
-
-    res.mergeProps = function(fromRes) {
-        if ($generatorCommon.isDefinedAndNotEmpty(fromRes)) {
-            res.datasources = fromRes.datasources;
-
-            angular.extend(res.imports, fromRes.imports);
-            angular.extend(res.staticImports, fromRes.staticImports);
-            angular.extend(res.vars, fromRes.vars);
-        }
-    };
-
-    res.mergeLines = function(fromRes) {
-        if ($generatorCommon.isDefinedAndNotEmpty(fromRes)) {
-            if (res.needEmptyLine)
-                res.push('');
-
-            _.forEach(fromRes, function(line) {
-                res.append(line);
-            });
-        }
-    };
-
-    res.startSafeBlock = function() {
-        res.safeDeep = this.deep;
-        this.safeNeedEmptyLine = this.needEmptyLine;
-        this.safeImports = _.cloneDeep(this.imports);
-        this.safeStaticImports = _.cloneDeep(this.staticImports);
-        this.safeDatasources = this.datasources.slice();
-        this.safePoint = this.length;
-    };
-
-    res.rollbackSafeBlock = function() {
-        if (this.safePoint >= 0) {
-            this.splice(this.safePoint, this.length - this.safePoint);
-
-            this.deep = res.safeDeep;
-            this.needEmptyLine = this.safeNeedEmptyLine;
-            this.datasources = this.safeDatasources;
-            this.imports = this.safeImports;
-            this.staticImports = this.safeStaticImports;
-            this.safePoint = -1;
-        }
-    };
-
-    res.asString = function() {
-        return this.join('\n');
-    };
-
-    res.append = function(s) {
-        this.push((this.lineStart ? _.repeat('    ', this.deep) : '') + s);
-
-        return this;
-    };
-
-    res.line = function(s) {
-        if (s) {
-            if (res.needEmptyLine)
-                res.push('');
-
-            res.append(s);
-        }
-
-        res.needEmptyLine = false;
-
-        res.lineStart = true;
-
-        return res;
-    };
-
-    res.startBlock = function(s) {
-        if (s) {
-            if (this.needEmptyLine)
-                this.push('');
-
-            this.append(s);
-        }
-
-        this.needEmptyLine = false;
-
-        this.lineStart = true;
-
-        this.deep++;
-
-        return this;
-    };
-
-    res.endBlock = function(s) {
-        this.deep--;
-
-        if (s)
-            this.append(s);
-
-        this.lineStart = true;
-
-        return this;
-    };
-
-    res.softEmptyLine = function() {
-        this.needEmptyLine = this.length > 0;
-    };
-
-    res.emptyLineIfNeeded = function() {
-        if (this.needEmptyLine) {
-            this.push('');
-            this.lineStart = true;
-
-            this.needEmptyLine = false;
-        }
-    };
-
-    /**
-     * Add class to imports.
-     *
-     * @param clsName Full class name.
-     * @returns {String} Short class name or full class name in case of names conflict.
-     */
-    res.importClass = function(clsName) {
-        if ($generatorCommon.JavaTypes.isJavaPrimitive(clsName))
-            return clsName;
-
-        const fullClassName = $generatorCommon.JavaTypes.fullClassName(clsName);
-
-        const dotIdx = fullClassName.lastIndexOf('.');
-
-        const shortName = dotIdx > 0 ? fullClassName.substr(dotIdx + 1) : fullClassName;
-
-        if (this.imports[shortName]) {
-            if (this.imports[shortName] !== fullClassName)
-                return fullClassName; // Short class names conflict. Return full name.
-        }
-        else
-            this.imports[shortName] = fullClassName;
-
-        return shortName;
-    };
-
-    /**
-     * Add class to imports.
-     *
-     * @param member Static member.
-     * @returns {String} Short class name or full class name in case of names conflict.
-     */
-    res.importStatic = function(member) {
-        const dotIdx = member.lastIndexOf('.');
-
-        const shortName = dotIdx > 0 ? member.substr(dotIdx + 1) : member;
-
-        if (this.staticImports[shortName]) {
-            if (this.staticImports[shortName] !== member)
-                return member; // Short class names conflict. Return full name.
-        }
-        else
-            this.staticImports[shortName] = member;
-
-        return shortName;
-    };
-
-    /**
-     * @returns String with "java imports" section.
-     */
-    res.generateImports = function() {
-        const genImports = [];
-
-        for (const clsName in this.imports) {
-            if (this.imports.hasOwnProperty(clsName) && this.imports[clsName].lastIndexOf('java.lang.', 0) !== 0)
-                genImports.push('import ' + this.imports[clsName] + ';');
-        }
-
-        genImports.sort();
-
-        return genImports.join('\n');
-    };
-
-    /**
-     * @returns String with "java imports" section.
-     */
-    res.generateStaticImports = function() {
-        const statImports = [];
-
-        for (const clsName in this.staticImports) {
-            if (this.staticImports.hasOwnProperty(clsName) && this.staticImports[clsName].lastIndexOf('java.lang.', 0) !== 0)
-                statImports.push('import static ' + this.staticImports[clsName] + ';');
-        }
-
-        statImports.sort();
-
-        return statImports.join('\n');
-    };
-
-    return res;
-};
-
-// Eviction policies code generation descriptors.
-$generatorCommon.EVICTION_POLICIES = {
-    LRU: {
-        className: 'org.apache.ignite.cache.eviction.lru.LruEvictionPolicy',
-        fields: {batchSize: {dflt: 1}, maxMemorySize: null, maxSize: {dflt: 100000}}
-    },
-    FIFO: {
-        className: 'org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy',
-        fields: {batchSize: {dflt: 1}, maxMemorySize: null, maxSize: {dflt: 100000}}
-    },
-    SORTED: {
-        className: 'org.apache.ignite.cache.eviction.sorted.SortedEvictionPolicy',
-        fields: {batchSize: {dflt: 1}, maxMemorySize: null, maxSize: {dflt: 100000}}
-    }
-};
-
-// Marshaller code generation descriptors.
-$generatorCommon.MARSHALLERS = {
-    OptimizedMarshaller: {
-        className: 'org.apache.ignite.marshaller.optimized.OptimizedMarshaller',
-        fields: {poolSize: null, requireSerializable: null }
-    },
-    JdkMarshaller: {
-        className: 'org.apache.ignite.marshaller.jdk.JdkMarshaller',
-        fields: {}
-    }
-};
-
-// Pairs of supported databases and their JDBC dialects.
-$generatorCommon.JDBC_DIALECTS = {
-    Generic: 'org.apache.ignite.cache.store.jdbc.dialect.BasicJdbcDialect',
-    Oracle: 'org.apache.ignite.cache.store.jdbc.dialect.OracleDialect',
-    DB2: 'org.apache.ignite.cache.store.jdbc.dialect.DB2Dialect',
-    SQLServer: 'org.apache.ignite.cache.store.jdbc.dialect.SQLServerDialect',
-    MySQL: 'org.apache.ignite.cache.store.jdbc.dialect.MySQLDialect',
-    PostgreSQL: 'org.apache.ignite.cache.store.jdbc.dialect.BasicJdbcDialect',
-    H2: 'org.apache.ignite.cache.store.jdbc.dialect.H2Dialect'
-};
-
-// Return JDBC dialect full class name for specified database.
-$generatorCommon.jdbcDialectClassName = function(db) {
-    const dialectClsName = $generatorCommon.JDBC_DIALECTS[db];
-
-    return dialectClsName ? dialectClsName : 'Unknown database: ' + db;
-};
-
-// Generate default data cache for specified igfs instance.
-$generatorCommon.igfsDataCache = function(igfs) {
-    return {
-        name: igfs.name + '-data',
-        cacheMode: 'PARTITIONED',
-        atomicityMode: 'TRANSACTIONAL',
-        writeSynchronizationMode: 'FULL_SYNC',
-        backups: 0,
-        igfsAffinnityGroupSize: igfs.affinnityGroupSize || 512
-    };
-};
-
-// Generate default meta cache for specified igfs instance.
-$generatorCommon.igfsMetaCache = function(igfs) {
-    return {
-        name: igfs.name + '-meta',
-        cacheMode: 'REPLICATED',
-        atomicityMode: 'TRANSACTIONAL',
-        writeSynchronizationMode: 'FULL_SYNC'
-    };
-};
-
-// Pairs of supported databases and their data sources.
-$generatorCommon.DATA_SOURCES = {
-    Generic: 'com.mchange.v2.c3p0.ComboPooledDataSource',
-    Oracle: 'oracle.jdbc.pool.OracleDataSource',
-    DB2: 'com.ibm.db2.jcc.DB2DataSource',
-    SQLServer: 'com.microsoft.sqlserver.jdbc.SQLServerDataSource',
-    MySQL: 'com.mysql.jdbc.jdbc2.optional.MysqlDataSource',
-    PostgreSQL: 'org.postgresql.ds.PGPoolingDataSource',
-    H2: 'org.h2.jdbcx.JdbcDataSource'
-};
-
-// Return data source full class name for specified database.
-$generatorCommon.dataSourceClassName = function(db) {
-    const dsClsName = $generatorCommon.DATA_SOURCES[db];
-
-    return dsClsName ? dsClsName : 'Unknown database: ' + db;
-};
-
-// Store factories code generation descriptors.
-$generatorCommon.STORE_FACTORIES = {
-    CacheJdbcPojoStoreFactory: {
-        className: 'org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory',
-        suffix: 'JdbcPojo',
-        fields: {
-            configuration: {type: 'bean'}
-        }
-    },
-    CacheJdbcBlobStoreFactory: {
-        className: 'org.apache.ignite.cache.store.jdbc.CacheJdbcBlobStoreFactory',
-        suffix: 'JdbcBlob',
-        fields: {
-            initSchema: null,
-            createTableQuery: null,
-            loadQuery: null,
-            insertQuery: null,
-            updateQuery: null,
-            deleteQuery: null
-        }
-    },
-    CacheHibernateBlobStoreFactory: {
-        className: 'org.apache.ignite.cache.store.hibernate.CacheHibernateBlobStoreFactory',
-        suffix: 'Hibernate',
-        fields: {hibernateProperties: {type: 'propertiesAsList', propVarName: 'props'}}
-    }
-};
-
-// Swap space SPI code generation descriptor.
-$generatorCommon.SWAP_SPACE_SPI = {
-    className: 'org.apache.ignite.spi.swapspace.file.FileSwapSpaceSpi',
-    fields: {
-        baseDirectory: {type: 'path'},
-        readStripesNumber: null,
-        maximumSparsity: {type: 'float'},
-        maxWriteQueueSize: null,
-        writeBufferSize: null
-    }
-};
-
-// Transaction configuration code generation descriptor.
-$generatorCommon.TRANSACTION_CONFIGURATION = {
-    className: 'org.apache.ignite.configuration.TransactionConfiguration',
-    fields: {
-        defaultTxConcurrency: {type: 'enum', enumClass: 'org.apache.ignite.transactions.TransactionConcurrency', dflt: 'PESSIMISTIC'},
-        defaultTxIsolation: {type: 'enum', enumClass: 'org.apache.ignite.transactions.TransactionIsolation', dflt: 'REPEATABLE_READ'},
-        defaultTxTimeout: {dflt: 0},
-        pessimisticTxLogLinger: {dflt: 10000},
-        pessimisticTxLogSize: null,
-        txSerializableEnabled: null,
-        txManagerFactory: {type: 'bean'}
-    }
-};
-
-// SSL configuration code generation descriptor.
-$generatorCommon.SSL_CONFIGURATION_TRUST_FILE_FACTORY = {
-    className: 'org.apache.ignite.ssl.SslContextFactory',
-    fields: {
-        keyAlgorithm: null,
-        keyStoreFilePath: {type: 'path'},
-        keyStorePassword: {type: 'raw'},
-        keyStoreType: null,
-        protocol: null,
-        trustStoreFilePath: {type: 'path'},
-        trustStorePassword: {type: 'raw'},
-        trustStoreType: null
-    }
-};
-
-// SSL configuration code generation descriptor.
-$generatorCommon.SSL_CONFIGURATION_TRUST_MANAGER_FACTORY = {
-    className: 'org.apache.ignite.ssl.SslContextFactory',
-    fields: {
-        keyAlgorithm: null,
-        keyStoreFilePath: {type: 'path'},
-        keyStorePassword: {type: 'raw'},
-        keyStoreType: null,
-        protocol: null,
-        trustManagers: {type: 'array'}
-    }
-};
-
-// Communication configuration code generation descriptor.
-$generatorCommon.CONNECTOR_CONFIGURATION = {
-    className: 'org.apache.ignite.configuration.ConnectorConfiguration',
-    fields: {
-        jettyPath: null,
-        host: null,
-        port: {dflt: 11211},
-        portRange: {dflt: 100},
-        idleTimeout: {dflt: 7000},
-        idleQueryCursorTimeout: {dflt: 600000},
-        idleQueryCursorCheckFrequency: {dflt: 60000},
-        receiveBufferSize: {dflt: 32768},
-        sendBufferSize: {dflt: 32768},
-        sendQueueLimit: {dflt: 0},
-        directBuffer: {dflt: false},
-        noDelay: {dflt: true},
-        selectorCount: null,
-        threadPoolSize: null,
-        messageInterceptor: {type: 'bean'},
-        secretKey: null,
-        sslEnabled: {dflt: false}
-    }
-};
-
-// Communication configuration code generation descriptor.
-$generatorCommon.COMMUNICATION_CONFIGURATION = {
-    className: 'org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi',
-    fields: {
-        listener: {type: 'bean'},
-        localAddress: null,
-        localPort: {dflt: 47100},
-        localPortRange: {dflt: 100},
-        sharedMemoryPort: {dflt: 48100},
-        directBuffer: {dflt: false},
-        directSendBuffer: {dflt: false},
-        idleConnectionTimeout: {dflt: 30000},
-        connectTimeout: {dflt: 5000},
-        maxConnectTimeout: {dflt: 600000},
-        reconnectCount: {dflt: 10},
-        socketSendBuffer: {dflt: 32768},
-        socketReceiveBuffer: {dflt: 32768},
-        messageQueueLimit: {dflt: 1024},
-        slowClientQueueLimit: null,
-        tcpNoDelay: {dflt: true},
-        ackSendThreshold: {dflt: 16},
-        unacknowledgedMessagesBufferSize: {dflt: 0},
-        socketWriteTimeout: {dflt: 2000},
-        selectorsCount: null,
-        addressResolver: {type: 'bean'}
-    }
-};
-
-// Communication configuration code generation descriptor.
-$generatorCommon.IGFS_IPC_CONFIGURATION = {
-    className: 'org.apache.ignite.igfs.IgfsIpcEndpointConfiguration',
-    fields: {
-        type: {type: 'enum', enumClass: 'org.apache.ignite.igfs.IgfsIpcEndpointType'},
-        host: {dflt: '127.0.0.1'},
-        port: {dflt: 10500},
-        memorySize: {dflt: 262144},
-        tokenDirectoryPath: {dflt: 'ipc/shmem'},
-        threadCount: null
-    }
-};
-
-$generatorCommon.ODBC_CONFIGURATION = {
-    className: 'org.apache.ignite.configuration.OdbcConfiguration',
-    fields: {
-        endpointAddress: {dflt: '0.0.0.0:10800..10810'},
-        maxOpenCursors: {dflt: 128}
-    }
-};
-
-// Check that cache has datasource.
-$generatorCommon.cacheHasDatasource = function(cache) {
-    if (cache.cacheStoreFactory && cache.cacheStoreFactory.kind) {
-        const storeFactory = cache.cacheStoreFactory[cache.cacheStoreFactory.kind];
-
-        return !!(storeFactory && (storeFactory.connectVia ? (storeFactory.connectVia === 'DataSource' ? storeFactory.dialect : false) : storeFactory.dialect)); // eslint-disable-line no-nested-ternary
-    }
-
-    return false;
-};
-
-$generatorCommon.secretPropertiesNeeded = function(cluster) {
-    return !_.isNil(_.find(cluster.caches, $generatorCommon.cacheHasDatasource)) || cluster.sslEnabled;
-};
-
-// Check that binary is configured.
-$generatorCommon.binaryIsDefined = function(binary) {
-    return binary && ($generatorCommon.isDefinedAndNotEmpty(binary.idMapper) || $generatorCommon.isDefinedAndNotEmpty(binary.nameMapper) ||
-        $generatorCommon.isDefinedAndNotEmpty(binary.serializer) || $generatorCommon.isDefinedAndNotEmpty(binary.typeConfigurations) ||
-        (!_.isNil(binary.compactFooter) && !binary.compactFooter));
-};
-
-// Extract domain model metadata location.
-$generatorCommon.domainQueryMetadata = function(domain) {
-    return domain.queryMetadata ? domain.queryMetadata : 'Configuration';
-};
-
-/**
- * @param {Object} obj Object to check.
- * @param {Array<String>} props Array of properties names.
- * @returns {boolean} 'true' if
- */
-$generatorCommon.hasAtLeastOneProperty = function(obj, props) {
-    return obj && props && _.findIndex(props, (prop) => !_.isNil(obj[prop])) >= 0;
-};
-
-/**
- * Convert some name to valid java name.
- *
- * @param prefix To append to java name.
- * @param name to convert.
- * @returns {string} Valid java name.
- */
-$generatorCommon.toJavaName = function(prefix, name) {
-    const javaName = name ? name.replace(/[^A-Za-z_0-9]+/g, '_') : 'dflt';
-
-    return prefix + javaName.charAt(0).toLocaleUpperCase() + javaName.slice(1);
-};
-
-/**
- * @param v Value to check.
- * @returns {boolean} 'true' if value defined and not empty string.
- */
-$generatorCommon.isDefinedAndNotEmpty = function(v) {
-    let defined = !_.isNil(v);
-
-    if (defined && (_.isString(v) || _.isArray(v)))
-        defined = v.length > 0;
-
-    return defined;
-};
-
-/**
- * @param {Object} obj Object to check.
- * @param {Array<String>} props Properties names.
- * @returns {boolean} 'true' if object contains at least one from specified properties.
- */
-$generatorCommon.hasProperty = function(obj, props) {
-    for (const propName in props) {
-        if (props.hasOwnProperty(propName)) {
-            if (obj[propName])
-                return true;
-        }
-    }
-
-    return false;
-};
-
-/**
- * Get class for selected implementation of Failover SPI.
- *
- * @param spi Failover SPI configuration.
- * @returns {*} Class for selected implementation of Failover SPI.
- */
-$generatorCommon.failoverSpiClass = function(spi) {
-    switch (spi.kind) {
-        case 'JobStealing': return 'org.apache.ignite.spi.failover.jobstealing.JobStealingFailoverSpi';
-        case 'Never': return 'org.apache.ignite.spi.failover.never.NeverFailoverSpi';
-        case 'Always': return 'org.apache.ignite.spi.failover.always.AlwaysFailoverSpi';
-        case 'Custom': return _.get(spi, 'Custom.class');
-        default: return 'Unknown';
-    }
-};
-
-$generatorCommon.loggerConfigured = function(logger) {
-    if (logger && logger.kind) {
-        const log = logger[logger.kind];
-
-        switch (logger.kind) {
-            case 'Log4j2': return log && $generatorCommon.isDefinedAndNotEmpty(log.path);
-
-            case 'Log4j':
-                if (!log || !log.mode)
-                    return false;
-
-                if (log.mode === 'Path')
-                    return $generatorCommon.isDefinedAndNotEmpty(log.path);
-
-                return true;
-
-            case 'Custom': return log && $generatorCommon.isDefinedAndNotEmpty(log.class);
-
-            default:
-                return true;
-        }
-    }
-
-    return false;
-};
-
-export default $generatorCommon;
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/generator-java.js b/modules/web-console/frontend/app/modules/configuration/generator/generator-java.js
deleted file mode 100644
index 296b942c42ec0..0000000000000
--- a/modules/web-console/frontend/app/modules/configuration/generator/generator-java.js
+++ /dev/null
@@ -1,3617 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Java generation entry point.
-const $generatorJava = {};
-
-/**
- * Translate some value to valid java code.
- *
- * @param val Value to convert.
- * @param type Value type.
- * @returns {*} String with value that will be valid for java.
- */
-$generatorJava.toJavaCode = function(val, type) {
-    if (val === null)
-        return 'null';
-
-    if (type === 'raw')
-        return val;
-
-    if (type === 'class')
-        return val + '.class';
-
-    if (type === 'float')
-        return val + 'f';
-
-    if (type === 'path')
-        return '"' + val.replace(/\\/g, '\\\\') + '"';
-
-    if (type)
-        return type + '.' + val;
-
-    if (typeof (val) === 'string')
-        return '"' + val.replace('"', '\\"') + '"';
-
-    if (typeof (val) === 'number' || typeof (val) === 'boolean')
-        return String(val);
-
-    return 'Unknown type: ' + typeof (val) + ' (' + val + ')';
-};
-
-/**
- * @param propName Property name.
- * @param setterName Optional concrete setter name.
- * @returns Property setter with name by java conventions.
- */
-$generatorJava.setterName = function(propName, setterName) {
-    return setterName ? setterName : $generatorCommon.toJavaName('set', propName);
-};
-
-// Add constructor argument
-$generatorJava.constructorArg = function(obj, propName, dflt, notFirst, opt) {
-    const v = (obj ? obj[propName] : null) || dflt;
-
-    if ($generatorCommon.isDefinedAndNotEmpty(v))
-        return (notFirst ? ', ' : '') + $generatorJava.toJavaCode(v);
-    else if (!opt)
-        return notFirst ? ', null' : 'null';
-
-    return '';
-};
-
-/**
- * Add variable declaration.
- *
- * @param res Resulting output with generated code.
- * @param varName Variable name.
- * @param varFullType Variable full class name to be added to imports.
- * @param varFullActualType Variable actual full class name to be added to imports.
- * @param varFullGenericType1 Optional full class name of first generic.
- * @param varFullGenericType2 Optional full class name of second generic.
- * @param subClass If 'true' then variable will be declared as anonymous subclass.
- */
-$generatorJava.declareVariable = function(res, varName, varFullType, varFullActualType, varFullGenericType1, varFullGenericType2, subClass) {
-    res.emptyLineIfNeeded();
-
-    const varType = res.importClass(varFullType);
-
-    const varNew = !res.vars[varName];
-
-    if (varNew)
-        res.vars[varName] = true;
-
-    if (varFullActualType && varFullGenericType1) {
-        const varActualType = res.importClass(varFullActualType);
-        const varGenericType1 = res.importClass(varFullGenericType1);
-        let varGenericType2 = null;
-
-        if (varFullGenericType2)
-            varGenericType2 = res.importClass(varFullGenericType2);
-
-        res.line((varNew ? (varType + '<' + varGenericType1 + (varGenericType2 ? ', ' + varGenericType2 : '') + '> ') : '') +
-            varName + ' = new ' + varActualType + '<>();');
-    }
-    else
-        res.line((varNew ? (varType + ' ') : '') + varName + ' = new ' + varType + '()' + (subClass ? ' {' : ';'));
-
-    if (!subClass)
-        res.needEmptyLine = true;
-
-    return varName;
-};
-
-/**
- * Add local variable declaration.
- *
- * @param res Resulting output with generated code.
- * @param varName Variable name.
- * @param varFullType Variable full class name to be added to imports.
- */
-$generatorJava.declareVariableLocal = function(res, varName, varFullType) {
-    const varType = res.importClass(varFullType);
-
-    res.line(varType + ' ' + varName + ' = new ' + varType + '();');
-
-    res.needEmptyLine = true;
-};
-
-/**
- * Add custom variable declaration.
- *
- * @param res Resulting output with generated code.
- * @param varName Variable name.
- * @param varFullType Variable full class name to be added to imports.
- * @param varExpr Custom variable creation expression.
- * @param modifier Additional variable modifier.
- */
-$generatorJava.declareVariableCustom = function(res, varName, varFullType, varExpr, modifier) {
-    const varType = res.importClass(varFullType);
-
-    const varNew = !res.vars[varName];
-
-    if (varNew)
-        res.vars[varName] = true;
-
-    res.line((varNew ? ((modifier ? modifier + ' ' : '') + varType + ' ') : '') + varName + ' = ' + varExpr + ';');
-
-    res.needEmptyLine = true;
-};
-
-/**
- * Add array variable declaration.
- *
- * @param res Resulting output with generated code.
- * @param varName Variable name.
- * @param varFullType Variable full class name to be added to imports.
- * @param length Array length.
- */
-$generatorJava.declareVariableArray = function(res, varName, varFullType, length) {
-    const varType = res.importClass(varFullType);
-
-    const varNew = !res.vars[varName];
-
-    if (varNew)
-        res.vars[varName] = true;
-
-    res.line((varNew ? (varType + '[] ') : '') + varName + ' = new ' + varType + '[' + length + '];');
-
-    res.needEmptyLine = true;
-};
-
-/**
- * Clear list of declared variables.
- *
- * @param res
- */
-$generatorJava.resetVariables = function(res) {
-    res.vars = {};
-};
-
-/**
- * Add property via setter / property name.
- *
- * @param res Resulting output with generated code.
- * @param varName Variable name.
- * @param obj Source object with data.
- * @param propName Property name to take from source object.
- * @param dataType Optional info about property data type.
- * @param setterName Optional special setter name.
- * @param dflt Optional default value.
- */
-$generatorJava.property = function(res, varName, obj, propName, dataType, setterName, dflt) {
-    if (!_.isNil(obj)) {
-        const val = obj[propName];
-
-        if ($generatorCommon.isDefinedAndNotEmpty(val)) {
-            const missDflt = _.isNil(dflt);
-
-            // Add to result if no default provided or value not equals to default.
-            if (missDflt || (!missDflt && val !== dflt)) {
-                res.line(varName + '.' + $generatorJava.setterName(propName, setterName) +
-                    '(' + $generatorJava.toJavaCode(val, dataType) + ');');
-
-                return true;
-            }
-        }
-    }
-
-    return false;
-};
-
-/**
- * Add enum property via setter / property name.
- *
- * @param res Resulting output with generated code.
- * @param varName Variable name.
- * @param obj Source object with data.
- * @param propName Property name to take from source object.
- * @param dataType Name of enum class
- * @param setterName Optional special setter name.
- * @param dflt Optional default value.
- */
-$generatorJava.enumProperty = function(res, varName, obj, propName, dataType, setterName, dflt) {
-    const val = obj[propName];
-
-    if ($generatorCommon.isDefinedAndNotEmpty(val)) {
-        const missDflt = _.isNil(dflt);
-
-        // Add to result if no default provided or value not equals to default.
-        if (missDflt || (!missDflt && val !== dflt)) {
-            res.line(varName + '.' + $generatorJava.setterName(propName, setterName) +
-                '(' + $generatorJava.toJavaCode(val, dataType ? res.importClass(dataType) : null) + ');');
-
-            return true;
-        }
-    }
-
-    return false;
-};
-
-// Add property for class name.
-$generatorJava.classNameProperty = function(res, varName, obj, propName) {
-    const val = obj[propName];
-
-    if (!_.isNil(val)) {
-        res.line(varName + '.' + $generatorJava.setterName(propName) +
-            '("' + $generatorCommon.JavaTypes.fullClassName(val) + '");');
-    }
-};
-
-/**
- * Add list property.
- *
- * @param res Resulting output with generated code.
- * @param varName Variable name.
- * @param obj Source object with data.
- * @param propName Property name to take from source object.
- * @param dataType Optional data type.
- * @param setterName Optional setter name.
- */
-$generatorJava.listProperty = function(res, varName, obj, propName, dataType, setterName) {
-    const val = obj[propName];
-
-    if (val && val.length > 0) {
-        res.emptyLineIfNeeded();
-
-        res.importClass('java.util.Arrays');
-
-        $generatorJava.fxVarArgs(res, varName + '.' + $generatorJava.setterName(propName, setterName), false,
-            _.map(val, (v) => $generatorJava.toJavaCode(v, dataType)), '(Arrays.asList(', '))');
-
-        res.needEmptyLine = true;
-    }
-};
-
-/**
- * Add function with varargs arguments.
- *
- * @param res Resulting output with generated code.
- * @param fx Function name.
- * @param quote Whether to quote arguments.
- * @param args Array with arguments.
- * @param startBlock Optional start block string.
- * @param endBlock Optional end block string.
- * @param startQuote Start quote string.
- * @param endQuote End quote string.
- */
-$generatorJava.fxVarArgs = function(res, fx, quote, args, startBlock = '(', endBlock = ')', startQuote = '"', endQuote = '"') {
-    const quoteArg = (arg) => quote ? startQuote + arg + endQuote : arg;
-
-    if (args.length === 1)
-        res.append(fx + startBlock + quoteArg(args[0]) + endBlock + ';');
-    else {
-        res.startBlock(fx + startBlock);
-
-        const len = args.length - 1;
-
-        _.forEach(args, (arg, ix) => res.line(quoteArg(arg) + (ix < len ? ', ' : '')));
-
-        res.endBlock(endBlock + ';');
-    }
-};
-
-/**
- * Add array property.
- *
- * @param res Resulting output with generated code.
- * @param varName Variable name.
- * @param obj Source object with data.
- * @param propName Property name to take from source object.
- * @param setterName Optional setter name.
- */
-$generatorJava.arrayProperty = function(res, varName, obj, propName, setterName) {
-    const val = obj[propName];
-
-    if (val && val.length > 0) {
-        res.emptyLineIfNeeded();
-
-        $generatorJava.fxVarArgs(res, varName + '.' + $generatorJava.setterName(propName, setterName), false,
-            _.map(val, (v) => 'new ' + res.importClass(v) + '()'), '({ ', ' });');
-
-        res.needEmptyLine = true;
-    }
-};
-
-/**
- * Add multi-param property (setter with several arguments).
- *
- * @param res Resulting output with generated code.
- * @param varName Variable name.
- * @param obj Source object with data.
- * @param propName Property name to take from source object.
- * @param dataType Optional data type.
- * @param setterName Optional setter name.
- */
-$generatorJava.multiparamProperty = function(res, varName, obj, propName, dataType, setterName) {
-    const val = obj[propName];
-
-    if (val && val.length > 0) {
-        $generatorJava.fxVarArgs(res, varName + '.' + $generatorJava.setterName(propName, setterName), false,
-            _.map(val, (v) => $generatorJava.toJavaCode(dataType === 'class' ? res.importClass(v) : v, dataType)));
-    }
-};
-
-/**
- * Add complex bean.
- *
- * @param res Resulting output with generated code.
- * @param varName Variable name.
- * @param bean
- * @param beanPropName Bean property name.
- * @param beanVarName
- * @param beanClass Bean class.
- * @param props
- * @param createBeanAlthoughNoProps If 'true' then create empty bean.
- */
-$generatorJava.beanProperty = function(res, varName, bean, beanPropName, beanVarName, beanClass, props, createBeanAlthoughNoProps) {
-    if (bean && $generatorCommon.hasProperty(bean, props)) {
-        res.emptyLineIfNeeded();
-
-        $generatorJava.declareVariable(res, beanVarName, beanClass);
-
-        _.forIn(props, function(descr, propName) {
-            if (props.hasOwnProperty(propName)) {
-                if (descr) {
-                    switch (descr.type) {
-                        case 'list':
-                            $generatorJava.listProperty(res, beanVarName, bean, propName, descr.elementsType, descr.setterName);
-                            break;
-
-                        case 'array':
-                            $generatorJava.arrayProperty(res, beanVarName, bean, propName, descr.setterName);
-                            break;
-
-                        case 'enum':
-                            $generatorJava.enumProperty(res, beanVarName, bean, propName, descr.enumClass, descr.setterName, descr.dflt);
-                            break;
-
-                        case 'float':
-                            $generatorJava.property(res, beanVarName, bean, propName, 'float', descr.setterName);
-                            break;
-
-                        case 'path':
-                            $generatorJava.property(res, beanVarName, bean, propName, 'path', descr.setterName);
-                            break;
-
-                        case 'raw':
-                            $generatorJava.property(res, beanVarName, bean, propName, 'raw', descr.setterName);
-                            break;
-
-                        case 'propertiesAsList':
-                            const val = bean[propName];
-
-                            if (val && val.length > 0) {
-                                $generatorJava.declareVariable(res, descr.propVarName, 'java.util.Properties');
-
-                                _.forEach(val, function(nameAndValue) {
-                                    const eqIndex = nameAndValue.indexOf('=');
-
-                                    if (eqIndex >= 0) {
-                                        res.line(descr.propVarName + '.setProperty(' +
-                                            '"' + nameAndValue.substring(0, eqIndex) + '", ' +
-                                            '"' + nameAndValue.substr(eqIndex + 1) + '");');
-                                    }
-                                });
-
-                                res.needEmptyLine = true;
-
-                                res.line(beanVarName + '.' + $generatorJava.setterName(propName) + '(' + descr.propVarName + ');');
-                            }
-                            break;
-
-                        case 'bean':
-                            if ($generatorCommon.isDefinedAndNotEmpty(bean[propName]))
-                                res.line(beanVarName + '.' + $generatorJava.setterName(propName) + '(new ' + res.importClass(bean[propName]) + '());');
-
-                            break;
-
-                        default:
-                            $generatorJava.property(res, beanVarName, bean, propName, null, descr.setterName, descr.dflt);
-                    }
-                }
-                else
-                    $generatorJava.property(res, beanVarName, bean, propName);
-            }
-        });
-
-        res.needEmptyLine = true;
-
-        res.line(varName + '.' + $generatorJava.setterName(beanPropName) + '(' + beanVarName + ');');
-
-        res.needEmptyLine = true;
-    }
-    else if (createBeanAlthoughNoProps) {
-        res.emptyLineIfNeeded();
-        res.line(varName + '.' + $generatorJava.setterName(beanPropName) + '(new ' + res.importClass(beanClass) + '());');
-
-        res.needEmptyLine = true;
-    }
-};
-
-/**
- * Add eviction policy.
- *
- * @param res Resulting output with generated code.
- * @param varName Current using variable name.
- * @param evtPlc Data to add.
- * @param propName Name in source data.
- */
-$generatorJava.evictionPolicy = function(res, varName, evtPlc, propName) {
-    if (evtPlc && evtPlc.kind) {
-        const evictionPolicyDesc = $generatorCommon.EVICTION_POLICIES[evtPlc.kind];
-
-        const obj = evtPlc[evtPlc.kind.toUpperCase()];
-
-        $generatorJava.beanProperty(res, varName, obj, propName, propName,
-            evictionPolicyDesc.className, evictionPolicyDesc.fields, true);
-    }
-};
-
-// Generate cluster general group.
-$generatorJava.clusterGeneral = function(cluster, clientNearCfg, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorJava.declareVariable(res, 'cfg', 'org.apache.ignite.configuration.IgniteConfiguration');
-
-    $generatorJava.property(res, 'cfg', cluster, 'name', null, 'setGridName');
-    res.needEmptyLine = true;
-
-    $generatorJava.property(res, 'cfg', cluster, 'localHost');
-    res.needEmptyLine = true;
-
-    if (clientNearCfg) {
-        res.line('cfg.setClientMode(true);');
-
-        res.needEmptyLine = true;
-    }
-
-    if (cluster.discovery) {
-        const d = cluster.discovery;
-
-        $generatorJava.declareVariable(res, 'discovery', 'org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi');
-
-        switch (d.kind) {
-            case 'Multicast':
-                $generatorJava.beanProperty(res, 'discovery', d.Multicast, 'ipFinder', 'ipFinder',
-                    'org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder',
-                    {
-                        multicastGroup: null,
-                        multicastPort: null,
-                        responseWaitTime: null,
-                        addressRequestAttempts: null,
-                        localAddress: null,
-                        addresses: {type: 'list'}
-                    }, true);
-
-                break;
-
-            case 'Vm':
-                $generatorJava.beanProperty(res, 'discovery', d.Vm, 'ipFinder', 'ipFinder',
-                    'org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder',
-                    {addresses: {type: 'list'}}, true);
-
-                break;
-
-            case 'S3':
-                $generatorJava.beanProperty(res, 'discovery', d.S3, 'ipFinder', 'ipFinder',
-                    'org.apache.ignite.spi.discovery.tcp.ipfinder.s3.TcpDiscoveryS3IpFinder', {bucketName: null}, true);
-
-                break;
-
-            case 'Cloud':
-                $generatorJava.beanProperty(res, 'discovery', d.Cloud, 'ipFinder', 'ipFinder',
-                    'org.apache.ignite.spi.discovery.tcp.ipfinder.cloud.TcpDiscoveryCloudIpFinder',
-                    {
-                        credential: null,
-                        credentialPath: null,
-                        identity: null,
-                        provider: null,
-                        regions: {type: 'list'},
-                        zones: {type: 'list'}
-                    }, true);
-
-                break;
-
-            case 'GoogleStorage':
-                $generatorJava.beanProperty(res, 'discovery', d.GoogleStorage, 'ipFinder', 'ipFinder',
-                    'org.apache.ignite.spi.discovery.tcp.ipfinder.gce.TcpDiscoveryGoogleStorageIpFinder',
-                    {
-                        projectName: null,
-                        bucketName: null,
-                        serviceAccountP12FilePath: null,
-                        serviceAccountId: null
-                    }, true);
-
-                break;
-
-            case 'Jdbc':
-                $generatorJava.declareVariable(res, 'ipFinder',
-                    'org.apache.ignite.spi.discovery.tcp.ipfinder.jdbc.TcpDiscoveryJdbcIpFinder');
-                $generatorJava.property(res, 'ipFinder', d.Jdbc, 'initSchema');
-
-                const datasource = d.Jdbc;
-                if (datasource.dataSourceBean && datasource.dialect) {
-                    res.needEmptyLine = !datasource.initSchema;
-
-                    res.line('ipFinder.setDataSource(DataSources.INSTANCE_' + datasource.dataSourceBean + ');');
-                }
-
-                res.needEmptyLine = true;
-
-                res.line('discovery.setIpFinder(ipFinder);');
-
-                break;
-
-            case 'SharedFs':
-                $generatorJava.beanProperty(res, 'discovery', d.SharedFs, 'ipFinder', 'ipFinder',
-                    'org.apache.ignite.spi.discovery.tcp.ipfinder.sharedfs.TcpDiscoverySharedFsIpFinder', {path: null}, true);
-
-                break;
-
-            case 'ZooKeeper':
-                const finderVar = 'ipFinder';
-
-                $generatorJava.declareVariable(res, 'ipFinder', 'org.apache.ignite.spi.discovery.tcp.ipfinder.zk.TcpDiscoveryZookeeperIpFinder');
-
-                if (d.ZooKeeper) {
-                    if ($generatorCommon.isDefinedAndNotEmpty(d.ZooKeeper.curator))
-                        res.line(finderVar + '.setCurator(new ' + res.importClass(d.ZooKeeper.curator) + '());');
-
-                    $generatorJava.property(res, finderVar, d.ZooKeeper, 'zkConnectionString');
-
-                    if (d.ZooKeeper.retryPolicy && d.ZooKeeper.retryPolicy.kind) {
-                        const kind = d.ZooKeeper.retryPolicy.kind;
-                        const retryPolicy = d.ZooKeeper.retryPolicy[kind];
-
-                        switch (kind) {
-                            case 'ExponentialBackoff':
-                                res.line(finderVar + '.setRetryPolicy(new ' + res.importClass('org.apache.curator.retry.ExponentialBackoffRetry') + '(' +
-                                    $generatorJava.constructorArg(retryPolicy, 'baseSleepTimeMs', 1000) +
-                                    $generatorJava.constructorArg(retryPolicy, 'maxRetries', 10, true) +
-                                    $generatorJava.constructorArg(retryPolicy, 'maxSleepMs', null, true, true) + '));');
-
-                                break;
-
-                            case 'BoundedExponentialBackoff':
-                                res.line(finderVar + '.setRetryPolicy(new ' + res.importClass('org.apache.curator.retry.BoundedExponentialBackoffRetry') + '(' +
-                                    $generatorJava.constructorArg(retryPolicy, 'baseSleepTimeMs', 1000) +
-                                    $generatorJava.constructorArg(retryPolicy, 'maxSleepTimeMs', 2147483647, true) +
-                                    $generatorJava.constructorArg(retryPolicy, 'maxRetries', 10, true) + '));');
-
-                                break;
-
-                            case 'UntilElapsed':
-                                res.line(finderVar + '.setRetryPolicy(new ' + res.importClass('org.apache.curator.retry.RetryUntilElapsed') + '(' +
-                                    $generatorJava.constructorArg(retryPolicy, 'maxElapsedTimeMs', 60000) +
-                                    $generatorJava.constructorArg(retryPolicy, 'sleepMsBetweenRetries', 1000, true) + '));');
-
-                                break;
-
-                            case 'NTimes':
-                                res.line(finderVar + '.setRetryPolicy(new ' + res.importClass('org.apache.curator.retry.RetryNTimes') + '(' +
-                                    $generatorJava.constructorArg(retryPolicy, 'n', 10) +
-                                    $generatorJava.constructorArg(retryPolicy, 'sleepMsBetweenRetries', 1000, true) + '));');
-
-                                break;
-
-                            case 'OneTime':
-                                res.line(finderVar + '.setRetryPolicy(new ' + res.importClass('org.apache.curator.retry.RetryOneTime') + '(' +
-                                    $generatorJava.constructorArg(retryPolicy, 'sleepMsBetweenRetry', 1000) + '));');
-
-                                break;
-
-                            case 'Forever':
-                                res.line(finderVar + '.setRetryPolicy(new ' + res.importClass('org.apache.curator.retry.RetryForever') + '(' +
-                                    $generatorJava.constructorArg(retryPolicy, 'retryIntervalMs', 1000) + '));');
-
-                                break;
-
-                            case 'Custom':
-                                if (retryPolicy && $generatorCommon.isDefinedAndNotEmpty(retryPolicy.className))
-                                    res.line(finderVar + '.setRetryPolicy(new ' + res.importClass(retryPolicy.className) + '());');
-
-                                break;
-
-                            default:
-                        }
-                    }
-
-                    $generatorJava.property(res, finderVar, d.ZooKeeper, 'basePath', null, null, '/services');
-                    $generatorJava.property(res, finderVar, d.ZooKeeper, 'serviceName', null, null, 'ignite');
-                    $generatorJava.property(res, finderVar, d.ZooKeeper, 'allowDuplicateRegistrations', null, null, false);
-                }
-
-                res.line('discovery.setIpFinder(ipFinder);');
-
-                break;
-
-            default:
-                res.line('Unknown discovery kind: ' + d.kind);
-        }
-
-        res.needEmptyLine = false;
-
-        $generatorJava.clusterDiscovery(d, res);
-
-        res.emptyLineIfNeeded();
-
-        res.line('cfg.setDiscoverySpi(discovery);');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate atomics group.
-$generatorJava.clusterAtomics = function(atomics, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.hasAtLeastOneProperty(atomics, ['cacheMode', 'atomicSequenceReserveSize', 'backups'])) {
-        res.startSafeBlock();
-
-        $generatorJava.declareVariable(res, 'atomicCfg', 'org.apache.ignite.configuration.AtomicConfiguration');
-
-        $generatorJava.enumProperty(res, 'atomicCfg', atomics, 'cacheMode', 'org.apache.ignite.cache.CacheMode', null, 'PARTITIONED');
-
-        const cacheMode = atomics.cacheMode ? atomics.cacheMode : 'PARTITIONED';
-
-        let hasData = cacheMode !== 'PARTITIONED';
-
-        hasData = $generatorJava.property(res, 'atomicCfg', atomics, 'atomicSequenceReserveSize', null, null, 1000) || hasData;
-
-        if (cacheMode === 'PARTITIONED')
-            hasData = $generatorJava.property(res, 'atomicCfg', atomics, 'backups', null, null, 0) || hasData;
-
-        res.needEmptyLine = true;
-
-        res.line('cfg.setAtomicConfiguration(atomicCfg);');
-
-        res.needEmptyLine = true;
-
-        if (!hasData)
-            res.rollbackSafeBlock();
-    }
-
-    return res;
-};
-
-// Generate binary group.
-$generatorJava.clusterBinary = function(binary, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.binaryIsDefined(binary)) {
-        const varName = 'binary';
-
-        $generatorJava.declareVariable(res, varName, 'org.apache.ignite.configuration.BinaryConfiguration');
-
-        if ($generatorCommon.isDefinedAndNotEmpty(binary.idMapper))
-            res.line(varName + '.setIdMapper(new ' + res.importClass(binary.idMapper) + '());');
-
-        if ($generatorCommon.isDefinedAndNotEmpty(binary.nameMapper))
-            res.line(varName + '.setNameMapper(new ' + res.importClass(binary.nameMapper) + '());');
-
-        if ($generatorCommon.isDefinedAndNotEmpty(binary.serializer))
-            res.line(varName + '.setSerializer(new ' + res.importClass(binary.serializer) + '());');
-
-        res.needEmptyLine = $generatorCommon.isDefinedAndNotEmpty(binary.idMapper) || $generatorCommon.isDefinedAndNotEmpty(binary.serializer);
-
-        if ($generatorCommon.isDefinedAndNotEmpty(binary.typeConfigurations)) {
-            const arrVar = 'types';
-
-            $generatorJava.declareVariable(res, arrVar, 'java.util.Collection', 'java.util.ArrayList', 'org.apache.ignite.binary.BinaryTypeConfiguration');
-
-            _.forEach(binary.typeConfigurations, function(type) {
-                if ($generatorCommon.isDefinedAndNotEmpty(type.typeName))
-                    res.line(arrVar + '.add(' + $generatorJava.binaryTypeFunctionName(type.typeName) + '());'); // TODO IGNITE-2269 Replace using of separated methods for binary type configurations to extended constructors.
-            });
-
-            res.needEmptyLine = true;
-
-            res.line(varName + '.setTypeConfigurations(' + arrVar + ');');
-
-            res.needEmptyLine = true;
-        }
-
-        $generatorJava.property(res, varName, binary, 'compactFooter', null, null, true);
-
-        res.needEmptyLine = true;
-
-        res.line('cfg.setBinaryConfiguration(' + varName + ');');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate cache key configurations.
-$generatorJava.clusterCacheKeyConfiguration = function(keyCfgs, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    keyCfgs = _.filter(keyCfgs, (cfg) => cfg.typeName && cfg.affinityKeyFieldName);
-
-    if (_.isEmpty(keyCfgs))
-        return res;
-
-    $generatorJava.declareVariableArray(res, 'keyConfigurations', 'org.apache.ignite.cache.CacheKeyConfiguration', keyCfgs.length);
-
-    const cacheKeyCfg = res.importClass('org.apache.ignite.cache.CacheKeyConfiguration');
-
-    _.forEach(keyCfgs, (cfg, idx) => {
-        res.needEmptyLine = true;
-
-        res.line(`keyConfigurations[${idx}] = new ${cacheKeyCfg}("${cfg.typeName}", "${cfg.affinityKeyFieldName}");`);
-
-        res.needEmptyLine = true;
-    });
-
-    res.line('cfg.setCacheKeyConfiguration(keyConfigurations);');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// TODO IGNITE-2269 Remove specified methods after implamentation of extended constructors.
-// Construct binary type configuration factory method name.
-$generatorJava.binaryTypeFunctionName = function(typeName) {
-    const dotIdx = typeName.lastIndexOf('.');
-
-    const shortName = dotIdx > 0 ? typeName.substr(dotIdx + 1) : typeName;
-
-    return $generatorCommon.toJavaName('binaryType', shortName);
-};
-
-// TODO IGNITE-2269 Remove specified methods after implamentation of extended constructors.
-// Generate factory method for specified BinaryTypeConfiguration.
-$generatorJava.binaryTypeConfiguration = function(type, res) {
-    const typeName = type.typeName;
-
-    res.line('/**');
-    res.line(' * Create binary type configuration for ' + typeName + '.');
-    res.line(' *');
-    res.line(' * @return Configured binary type.');
-    res.line(' */');
-    res.startBlock('private static BinaryTypeConfiguration ' + $generatorJava.binaryTypeFunctionName(typeName) + '() {');
-
-    $generatorJava.resetVariables(res);
-
-    const typeVar = 'typeCfg';
-
-    $generatorJava.declareVariable(res, typeVar, 'org.apache.ignite.binary.BinaryTypeConfiguration');
-
-    $generatorJava.property(res, typeVar, type, 'typeName');
-
-    if ($generatorCommon.isDefinedAndNotEmpty(type.idMapper))
-        res.line(typeVar + '.setIdMapper(new ' + res.importClass(type.idMapper) + '());');
-
-    if ($generatorCommon.isDefinedAndNotEmpty(type.nameMapper))
-        res.line(typeVar + '.setNameMapper(new ' + res.importClass(type.nameMapper) + '());');
-
-    if ($generatorCommon.isDefinedAndNotEmpty(type.serializer))
-        res.line(typeVar + '.setSerializer(new ' + res.importClass(type.serializer) + '());');
-
-    $generatorJava.property(res, typeVar, type, 'enum', null, null, false);
-
-    res.needEmptyLine = true;
-
-    res.line('return ' + typeVar + ';');
-    res.endBlock('}');
-
-    res.needEmptyLine = true;
-};
-
-// TODO IGNITE-2269 Remove specified methods after implamentation of extended constructors.
-// Generates binary type configuration factory methods.
-$generatorJava.binaryTypeConfigurations = function(binary, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!_.isNil(binary)) {
-        _.forEach(binary.typeConfigurations, function(type) {
-            $generatorJava.binaryTypeConfiguration(type, res);
-        });
-    }
-
-    return res;
-};
-
-// Generate collision group.
-$generatorJava.clusterCollision = function(collision, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (collision && collision.kind && collision.kind !== 'Noop') {
-        const spi = collision[collision.kind];
-
-        if (collision.kind !== 'Custom' || (spi && $generatorCommon.isDefinedAndNotEmpty(spi.class))) {
-            const varName = 'collisionSpi';
-
-            switch (collision.kind) {
-                case 'JobStealing':
-                    $generatorJava.declareVariable(res, varName, 'org.apache.ignite.spi.collision.jobstealing.JobStealingCollisionSpi');
-
-                    $generatorJava.property(res, varName, spi, 'activeJobsThreshold', null, null, 95);
-                    $generatorJava.property(res, varName, spi, 'waitJobsThreshold', null, null, 0);
-                    $generatorJava.property(res, varName, spi, 'messageExpireTime', null, null, 1000);
-                    $generatorJava.property(res, varName, spi, 'maximumStealingAttempts', null, null, 5);
-                    $generatorJava.property(res, varName, spi, 'stealingEnabled', null, null, true);
-
-                    if ($generatorCommon.isDefinedAndNotEmpty(spi.externalCollisionListener)) {
-                        res.line(varName + '.' + $generatorJava.setterName('externalCollisionListener') +
-                            '(new ' + res.importClass(spi.externalCollisionListener) + '());');
-                    }
-
-                    if ($generatorCommon.isDefinedAndNotEmpty(spi.stealingAttributes)) {
-                        const stealingAttrsVar = 'stealingAttrs';
-
-                        res.needEmptyLine = true;
-
-                        $generatorJava.declareVariable(res, stealingAttrsVar, 'java.util.Map', 'java.util.HashMap', 'String', 'java.io.Serializable');
-
-                        _.forEach(spi.stealingAttributes, function(attr) {
-                            res.line(stealingAttrsVar + '.put("' + attr.name + '", "' + attr.value + '");');
-                        });
-
-                        res.needEmptyLine = true;
-
-                        res.line(varName + '.setStealingAttributes(' + stealingAttrsVar + ');');
-                    }
-
-                    break;
-
-                case 'FifoQueue':
-                    $generatorJava.declareVariable(res, varName, 'org.apache.ignite.spi.collision.fifoqueue.FifoQueueCollisionSpi');
-
-                    $generatorJava.property(res, varName, spi, 'parallelJobsNumber');
-                    $generatorJava.property(res, varName, spi, 'waitingJobsNumber');
-
-                    break;
-
-                case 'PriorityQueue':
-                    $generatorJava.declareVariable(res, varName, 'org.apache.ignite.spi.collision.priorityqueue.PriorityQueueCollisionSpi');
-
-                    $generatorJava.property(res, varName, spi, 'parallelJobsNumber');
-                    $generatorJava.property(res, varName, spi, 'waitingJobsNumber');
-                    $generatorJava.property(res, varName, spi, 'priorityAttributeKey', null, null, 'grid.task.priority');
-                    $generatorJava.property(res, varName, spi, 'jobPriorityAttributeKey', null, null, 'grid.job.priority');
-                    $generatorJava.property(res, varName, spi, 'defaultPriority', null, null, 0);
-                    $generatorJava.property(res, varName, spi, 'starvationIncrement', null, null, 1);
-                    $generatorJava.property(res, varName, spi, 'starvationPreventionEnabled', null, null, true);
-
-                    break;
-
-                case 'Custom':
-                    $generatorJava.declareVariable(res, varName, spi.class);
-
-                    break;
-
-                default:
-            }
-
-            res.needEmptyLine = true;
-
-            res.line('cfg.setCollisionSpi(' + varName + ');');
-
-            res.needEmptyLine = true;
-        }
-    }
-
-    return res;
-};
-
-// Generate communication group.
-$generatorJava.clusterCommunication = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    const cfg = $generatorCommon.COMMUNICATION_CONFIGURATION;
-
-    $generatorJava.beanProperty(res, 'cfg', cluster.communication, 'communicationSpi', 'commSpi', cfg.className, cfg.fields);
-
-    res.needEmptyLine = false;
-
-    $generatorJava.property(res, 'cfg', cluster, 'networkTimeout', null, null, 5000);
-    $generatorJava.property(res, 'cfg', cluster, 'networkSendRetryDelay', null, null, 1000);
-    $generatorJava.property(res, 'cfg', cluster, 'networkSendRetryCount', null, null, 3);
-    $generatorJava.property(res, 'cfg', cluster, 'segmentCheckFrequency');
-    $generatorJava.property(res, 'cfg', cluster, 'waitForSegmentOnStart', null, null, false);
-    $generatorJava.property(res, 'cfg', cluster, 'discoveryStartupDelay', null, null, 60000);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate REST access group.
-$generatorJava.clusterConnector = function(connector, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!_.isNil(connector) && connector.enabled) {
-        const cfg = _.cloneDeep($generatorCommon.CONNECTOR_CONFIGURATION);
-
-        if (connector.sslEnabled) {
-            cfg.fields.sslClientAuth = {dflt: false};
-            cfg.fields.sslFactory = {type: 'bean'};
-        }
-
-        $generatorJava.beanProperty(res, 'cfg', connector, 'connectorConfiguration', 'clientCfg',
-            cfg.className, cfg.fields, true);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate deployment group.
-$generatorJava.clusterDeployment = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorJava.enumProperty(res, 'cfg', cluster, 'deploymentMode', 'org.apache.ignite.configuration.DeploymentMode', null, 'SHARED');
-
-    res.softEmptyLine();
-
-    const p2pEnabled = cluster.peerClassLoadingEnabled;
-
-    if (!_.isNil(p2pEnabled)) {
-        $generatorJava.property(res, 'cfg', cluster, 'peerClassLoadingEnabled', null, null, false);
-
-        if (p2pEnabled) {
-            $generatorJava.property(res, 'cfg', cluster, 'peerClassLoadingMissedResourcesCacheSize', null, null, 100);
-            $generatorJava.property(res, 'cfg', cluster, 'peerClassLoadingThreadPoolSize', null, null, 2);
-            $generatorJava.multiparamProperty(res, 'cfg', cluster, 'peerClassLoadingLocalClassPathExclude');
-        }
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate discovery group.
-$generatorJava.clusterDiscovery = function(disco, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (disco) {
-        $generatorJava.property(res, 'discovery', disco, 'localAddress');
-        $generatorJava.property(res, 'discovery', disco, 'localPort', null, null, 47500);
-        $generatorJava.property(res, 'discovery', disco, 'localPortRange', null, null, 100);
-
-        if ($generatorCommon.isDefinedAndNotEmpty(disco.addressResolver)) {
-            $generatorJava.beanProperty(res, 'discovery', disco, 'addressResolver', 'addressResolver', disco.addressResolver, {}, true);
-            res.needEmptyLine = false;
-        }
-
-        $generatorJava.property(res, 'discovery', disco, 'socketTimeout', null, null, 5000);
-        $generatorJava.property(res, 'discovery', disco, 'ackTimeout', null, null, 5000);
-        $generatorJava.property(res, 'discovery', disco, 'maxAckTimeout', null, null, 600000);
-        $generatorJava.property(res, 'discovery', disco, 'networkTimeout', null, null, 5000);
-        $generatorJava.property(res, 'discovery', disco, 'joinTimeout', null, null, 0);
-        $generatorJava.property(res, 'discovery', disco, 'threadPriority', null, null, 10);
-        $generatorJava.property(res, 'discovery', disco, 'heartbeatFrequency', null, null, 2000);
-        $generatorJava.property(res, 'discovery', disco, 'maxMissedHeartbeats', null, null, 1);
-        $generatorJava.property(res, 'discovery', disco, 'maxMissedClientHeartbeats', null, null, 5);
-        $generatorJava.property(res, 'discovery', disco, 'topHistorySize', null, null, 1000);
-
-        if ($generatorCommon.isDefinedAndNotEmpty(disco.listener)) {
-            $generatorJava.beanProperty(res, 'discovery', disco, 'listener', 'listener', disco.listener, {}, true);
-            res.needEmptyLine = false;
-        }
-
-        if ($generatorCommon.isDefinedAndNotEmpty(disco.dataExchange)) {
-            $generatorJava.beanProperty(res, 'discovery', disco, 'dataExchange', 'dataExchange', disco.dataExchange, {}, true);
-            res.needEmptyLine = false;
-        }
-
-        if ($generatorCommon.isDefinedAndNotEmpty(disco.metricsProvider)) {
-            $generatorJava.beanProperty(res, 'discovery', disco, 'metricsProvider', 'metricsProvider', disco.metricsProvider, {}, true);
-            res.needEmptyLine = false;
-        }
-
-        $generatorJava.property(res, 'discovery', disco, 'reconnectCount', null, null, 10);
-        $generatorJava.property(res, 'discovery', disco, 'statisticsPrintFrequency', null, null, 0);
-        $generatorJava.property(res, 'discovery', disco, 'ipFinderCleanFrequency', null, null, 60000);
-
-        if ($generatorCommon.isDefinedAndNotEmpty(disco.authenticator)) {
-            $generatorJava.beanProperty(res, 'discovery', disco, 'authenticator', 'authenticator', disco.authenticator, {}, true);
-            res.needEmptyLine = false;
-        }
-
-        $generatorJava.property(res, 'discovery', disco, 'forceServerMode', null, null, false);
-        $generatorJava.property(res, 'discovery', disco, 'clientReconnectDisabled', null, null, false);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate events group.
-$generatorJava.clusterEvents = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (cluster.includeEventTypes && cluster.includeEventTypes.length > 0) {
-        res.emptyLineIfNeeded();
-
-        const evtGrps = angular.element(document.getElementById('app')).injector().get('igniteEventGroups');
-
-        if (cluster.includeEventTypes.length === 1) {
-            const evtGrp = _.find(evtGrps, {value: cluster.includeEventTypes[0]});
-            const evts = res.importStatic(evtGrp.class + '.' + evtGrp.value);
-
-            res.line('cfg.setIncludeEventTypes(' + evts + ');');
-        }
-        else {
-            _.forEach(cluster.includeEventTypes, function(value, ix) {
-                const evtGrp = _.find(evtGrps, {value});
-                const evts = res.importStatic(evtGrp.class + '.' + evtGrp.value);
-
-                if (ix === 0)
-                    res.line('int[] events = new int[' + evts + '.length');
-                else
-                    res.line('    + ' + evts + '.length');
-            });
-
-            res.line('];');
-
-            res.needEmptyLine = true;
-
-            res.line('int k = 0;');
-
-            _.forEach(cluster.includeEventTypes, function(value, idx) {
-                res.needEmptyLine = true;
-
-                const evtGrp = _.find(evtGrps, {value});
-                const evts = res.importStatic(evtGrp.class + '.' + value);
-
-                res.line('System.arraycopy(' + evts + ', 0, events, k, ' + evts + '.length);');
-
-                if (idx < cluster.includeEventTypes.length - 1)
-                    res.line('k += ' + evts + '.length;');
-            });
-
-            res.needEmptyLine = true;
-
-            res.line('cfg.setIncludeEventTypes(events);');
-        }
-
-        res.needEmptyLine = true;
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate failover group.
-$generatorJava.clusterFailover = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.isDefinedAndNotEmpty(cluster.failoverSpi) && _.findIndex(cluster.failoverSpi, function(spi) {
-        return $generatorCommon.isDefinedAndNotEmpty(spi.kind) && (spi.kind !== 'Custom' || $generatorCommon.isDefinedAndNotEmpty(_.get(spi, spi.kind + '.class')));
-    }) >= 0) {
-        const arrayVarName = 'failoverSpiList';
-
-        $generatorJava.declareVariable(res, arrayVarName, 'java.util.List', 'java.util.ArrayList', 'org.apache.ignite.spi.failover.FailoverSpi');
-
-        _.forEach(cluster.failoverSpi, function(spi) {
-            if (spi.kind && (spi.kind !== 'Custom' || $generatorCommon.isDefinedAndNotEmpty(_.get(spi, spi.kind + '.class')))) {
-                const varName = 'failoverSpi';
-
-                const maxAttempts = _.get(spi, spi.kind + '.maximumFailoverAttempts');
-
-                if ((spi.kind === 'JobStealing' || spi.kind === 'Always') && $generatorCommon.isDefinedAndNotEmpty(maxAttempts) && maxAttempts !== 5) {
-                    const spiCls = res.importClass($generatorCommon.failoverSpiClass(spi));
-
-                    $generatorJava.declareVariableCustom(res, varName, 'org.apache.ignite.spi.failover.FailoverSpi', 'new ' + spiCls + '()');
-
-                    if ($generatorCommon.isDefinedAndNotEmpty(spi[spi.kind].maximumFailoverAttempts))
-                        res.line('((' + spiCls + ') ' + varName + ').setMaximumFailoverAttempts(' + spi[spi.kind].maximumFailoverAttempts + ');');
-
-                    res.needEmptyLine = true;
-
-                    res.line(arrayVarName + '.add(' + varName + ');');
-                }
-                else
-                    res.line(arrayVarName + '.add(new ' + res.importClass($generatorCommon.failoverSpiClass(spi)) + '());');
-
-                res.needEmptyLine = true;
-            }
-        });
-
-        res.line('cfg.setFailoverSpi(' + arrayVarName + '.toArray(new FailoverSpi[' + arrayVarName + '.size()]));');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate marshaller group.
-$generatorJava.clusterLogger = function(logger, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.loggerConfigured(logger)) {
-        const varName = 'logger';
-
-        const log = logger[logger.kind];
-
-        switch (logger.kind) {
-            case 'Log4j2':
-                $generatorJava.declareVariableCustom(res, varName, 'org.apache.ignite.logger.log4j2.Log4J2Logger',
-                    'new Log4J2Logger(' + $generatorJava.toJavaCode(log.path, 'path') + ')');
-
-                res.needEmptyLine = true;
-
-                if ($generatorCommon.isDefinedAndNotEmpty(log.level))
-                    res.line(varName + '.setLevel(' + res.importClass('org.apache.logging.log4j.Level') + '.' + log.level + ');');
-
-                break;
-
-            case 'Null':
-                $generatorJava.declareVariable(res, varName, 'org.apache.ignite.logger.NullLogger');
-
-                break;
-
-            case 'Java':
-                $generatorJava.declareVariable(res, varName, 'org.apache.ignite.logger.java.JavaLogger');
-
-                break;
-
-            case 'JCL':
-                $generatorJava.declareVariable(res, varName, 'org.apache.ignite.logger.jcl.JclLogger');
-
-                break;
-
-            case 'SLF4J':
-                $generatorJava.declareVariable(res, varName, 'org.apache.ignite.logger.slf4j.Slf4jLogger');
-
-                break;
-
-            case 'Log4j':
-                if (log.mode === 'Default')
-                    $generatorJava.declareVariable(res, varName, 'org.apache.ignite.logger.log4j.Log4JLogger');
-                else {
-                    $generatorJava.declareVariableCustom(res, varName, 'org.apache.ignite.logger.log4j.Log4JLogger',
-                        'new Log4JLogger(' + $generatorJava.toJavaCode(log.path, 'path') + ')');
-                }
-
-                if ($generatorCommon.isDefinedAndNotEmpty(log.level))
-                    res.line(varName + '.setLevel(' + res.importClass('org.apache.log4j.Level') + '.' + log.level + ');');
-
-                break;
-
-            case 'Custom':
-                $generatorJava.declareVariable(res, varName, log.class);
-
-                break;
-
-            default:
-        }
-
-        res.needEmptyLine = true;
-
-        res.line('cfg.setGridLogger(' + varName + ');');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate marshaller group.
-$generatorJava.clusterMarshaller = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    const marshaller = cluster.marshaller;
-
-    if (marshaller && marshaller.kind) {
-        const marshallerDesc = $generatorCommon.MARSHALLERS[marshaller.kind];
-
-        $generatorJava.beanProperty(res, 'cfg', marshaller[marshaller.kind], 'marshaller', 'marshaller',
-            marshallerDesc.className, marshallerDesc.fields, true);
-
-        $generatorJava.beanProperty(res, 'marshaller', marshaller[marshaller.kind], marshallerDesc.className, marshallerDesc.fields, true);
-    }
-
-    $generatorJava.property(res, 'cfg', cluster, 'marshalLocalJobs', null, null, false);
-    $generatorJava.property(res, 'cfg', cluster, 'marshallerCacheKeepAliveTime', null, null, 10000);
-    $generatorJava.property(res, 'cfg', cluster, 'marshallerCacheThreadPoolSize', null, 'setMarshallerCachePoolSize');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate metrics group.
-$generatorJava.clusterMetrics = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorJava.property(res, 'cfg', cluster, 'metricsExpireTime');
-    $generatorJava.property(res, 'cfg', cluster, 'metricsHistorySize', null, null, 10000);
-    $generatorJava.property(res, 'cfg', cluster, 'metricsLogFrequency', null, null, 60000);
-    $generatorJava.property(res, 'cfg', cluster, 'metricsUpdateFrequency', null, null, 2000);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate swap group.
-$generatorJava.clusterSwap = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (cluster.swapSpaceSpi && cluster.swapSpaceSpi.kind === 'FileSwapSpaceSpi') {
-        $generatorJava.beanProperty(res, 'cfg', cluster.swapSpaceSpi.FileSwapSpaceSpi, 'swapSpaceSpi', 'swapSpi',
-            $generatorCommon.SWAP_SPACE_SPI.className, $generatorCommon.SWAP_SPACE_SPI.fields, true);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate time group.
-$generatorJava.clusterTime = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorJava.property(res, 'cfg', cluster, 'clockSyncSamples', null, null, 8);
-    $generatorJava.property(res, 'cfg', cluster, 'clockSyncFrequency', null, null, 120000);
-    $generatorJava.property(res, 'cfg', cluster, 'timeServerPortBase', null, null, 31100);
-    $generatorJava.property(res, 'cfg', cluster, 'timeServerPortRange', null, null, 100);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate ODBC configuration group.
-$generatorJava.clusterODBC = function(odbc, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (odbc && odbc.odbcEnabled) {
-        $generatorJava.beanProperty(res, 'cfg', odbc, 'odbcConfiguration', 'odbcConfiguration',
-            $generatorCommon.ODBC_CONFIGURATION.className, $generatorCommon.ODBC_CONFIGURATION.fields, true);
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate thread pools group.
-$generatorJava.clusterPools = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorJava.property(res, 'cfg', cluster, 'publicThreadPoolSize');
-    $generatorJava.property(res, 'cfg', cluster, 'systemThreadPoolSize');
-    $generatorJava.property(res, 'cfg', cluster, 'managementThreadPoolSize');
-    $generatorJava.property(res, 'cfg', cluster, 'igfsThreadPoolSize');
-    $generatorJava.property(res, 'cfg', cluster, 'rebalanceThreadPoolSize');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate transactions group.
-$generatorJava.clusterTransactions = function(transactionConfiguration, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorJava.beanProperty(res, 'cfg', transactionConfiguration, 'transactionConfiguration',
-        'transactionConfiguration', $generatorCommon.TRANSACTION_CONFIGURATION.className,
-        $generatorCommon.TRANSACTION_CONFIGURATION.fields, false);
-
-    return res;
-};
-
-// Generate user attributes group.
-$generatorJava.clusterUserAttributes = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.isDefinedAndNotEmpty(cluster.attributes)) {
-        $generatorJava.declareVariable(res, 'attributes', 'java.util.Map', 'java.util.HashMap', 'java.lang.String', 'java.lang.String');
-
-        _.forEach(cluster.attributes, function(attr) {
-            res.line('attributes.put("' + attr.name + '", "' + attr.value + '");');
-        });
-
-        res.needEmptyLine = true;
-
-        res.line('cfg.setUserAttributes(attributes);');
-
-        res.needEmptyLine = true;
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-
-// Generate cache general group.
-$generatorJava.cacheGeneral = function(cache, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('cache', cache);
-
-    $generatorJava.property(res, varName, cache, 'name');
-
-    $generatorJava.enumProperty(res, varName, cache, 'cacheMode', 'org.apache.ignite.cache.CacheMode');
-    $generatorJava.enumProperty(res, varName, cache, 'atomicityMode', 'org.apache.ignite.cache.CacheAtomicityMode');
-
-    if (cache.cacheMode === 'PARTITIONED' && $generatorJava.property(res, varName, cache, 'backups'))
-        $generatorJava.property(res, varName, cache, 'readFromBackup');
-
-    $generatorJava.property(res, varName, cache, 'copyOnRead');
-
-    if (cache.cacheMode === 'PARTITIONED' && cache.atomicityMode === 'TRANSACTIONAL')
-        $generatorJava.property(res, varName, cache, 'invalidate');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache memory group.
-$generatorJava.cacheMemory = function(cache, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('cache', cache);
-
-    $generatorJava.enumProperty(res, varName, cache, 'memoryMode', 'org.apache.ignite.cache.CacheMemoryMode', null, 'ONHEAP_TIERED');
-
-    if (cache.memoryMode !== 'OFFHEAP_VALUES')
-        $generatorJava.property(res, varName, cache, 'offHeapMaxMemory', null, null, -1);
-
-    res.softEmptyLine();
-
-    $generatorJava.evictionPolicy(res, varName, cache.evictionPolicy, 'evictionPolicy');
-
-    $generatorJava.property(res, varName, cache, 'startSize', null, null, 1500000);
-    $generatorJava.property(res, varName, cache, 'swapEnabled', null, null, false);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache query & indexing group.
-$generatorJava.cacheQuery = function(cache, domains, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('cache', cache);
-
-    $generatorJava.property(res, varName, cache, 'sqlSchema');
-    $generatorJava.property(res, varName, cache, 'sqlOnheapRowCacheSize', null, null, 10240);
-    $generatorJava.property(res, varName, cache, 'longQueryWarningTimeout', null, null, 3000);
-
-    const indexedTypes = _.reduce(domains, (acc, domain) => {
-        if (domain.queryMetadata === 'Annotations') {
-            acc.push(domain.keyType);
-            acc.push(domain.valueType);
-        }
-
-        return acc;
-    }, []);
-
-    if (indexedTypes.length > 0) {
-        res.softEmptyLine();
-
-        $generatorJava.multiparamProperty(res, varName, {indexedTypes}, 'indexedTypes', 'class');
-    }
-
-    res.softEmptyLine();
-
-    $generatorJava.multiparamProperty(res, varName, cache, 'sqlFunctionClasses', 'class');
-
-    res.softEmptyLine();
-
-    $generatorJava.property(res, varName, cache, 'snapshotableIndex', null, null, false);
-    $generatorJava.property(res, varName, cache, 'sqlEscapeAll', null, null, false);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-/**
- * Generate cache store datasource.
- *
- * @param storeFactory Factory to generate data source for.
- * @param res Resulting output with generated code.
- */
-$generatorJava.cacheStoreDataSource = function(storeFactory, res) {
-    const dialect = storeFactory.connectVia ? (storeFactory.connectVia === 'DataSource' ? storeFactory.dialect : null) : storeFactory.dialect;
-
-    if (dialect) {
-        const varName = 'dataSource';
-
-        const dataSourceBean = storeFactory.dataSourceBean;
-
-        const varType = res.importClass($generatorCommon.dataSourceClassName(dialect));
-
-        res.line('public static final ' + varType + ' INSTANCE_' + dataSourceBean + ' = create' + dataSourceBean + '();');
-
-        res.needEmptyLine = true;
-
-        res.startBlock('private static ' + varType + ' create' + dataSourceBean + '() {');
-        if (dialect === 'Oracle')
-            res.startBlock('try {');
-
-        $generatorJava.resetVariables(res);
-
-        $generatorJava.declareVariable(res, varName, varType);
-
-        switch (dialect) {
-            case 'Generic':
-                res.line(varName + '.setJdbcUrl(props.getProperty("' + dataSourceBean + '.jdbc.url"));');
-
-                break;
-
-            case 'DB2':
-                res.line(varName + '.setServerName(props.getProperty("' + dataSourceBean + '.jdbc.server_name"));');
-                res.line(varName + '.setPortNumber(Integer.valueOf(props.getProperty("' + dataSourceBean + '.jdbc.port_number")));');
-                res.line(varName + '.setDatabaseName(props.getProperty("' + dataSourceBean + '.jdbc.database_name"));');
-                res.line(varName + '.setDriverType(Integer.valueOf(props.getProperty("' + dataSourceBean + '.jdbc.driver_type")));');
-
-                break;
-
-            case 'PostgreSQL':
-                res.line(varName + '.setUrl(props.getProperty("' + dataSourceBean + '.jdbc.url"));');
-
-                break;
-
-            default:
-                res.line(varName + '.setURL(props.getProperty("' + dataSourceBean + '.jdbc.url"));');
-        }
-
-        res.line(varName + '.setUser(props.getProperty("' + dataSourceBean + '.jdbc.username"));');
-        res.line(varName + '.setPassword(props.getProperty("' + dataSourceBean + '.jdbc.password"));');
-
-        res.needEmptyLine = true;
-
-        res.line('return dataSource;');
-
-        if (dialect === 'Oracle') {
-            res.endBlock('}');
-            res.startBlock('catch (' + res.importClass('java.sql.SQLException') + ' ex) {');
-            res.line('throw new Error(ex);');
-            res.endBlock('}');
-        }
-
-        res.endBlock('}');
-
-        res.needEmptyLine = true;
-
-        return dataSourceBean;
-    }
-
-    return null;
-};
-
-$generatorJava.clusterDataSources = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    const datasources = [];
-
-    let storeFound = false;
-
-    function startSourcesFunction() {
-        if (!storeFound) {
-            res.line('/** Helper class for datasource creation. */');
-            res.startBlock('public static class DataSources {');
-
-            storeFound = true;
-        }
-    }
-
-    _.forEach(cluster.caches, function(cache) {
-        const factoryKind = cache.cacheStoreFactory.kind;
-
-        const storeFactory = cache.cacheStoreFactory[factoryKind];
-
-        if (storeFactory) {
-            const beanClassName = $generatorJava.dataSourceClassName(res, storeFactory);
-
-            if (beanClassName && !_.includes(datasources, beanClassName)) {
-                datasources.push(beanClassName);
-
-                if (factoryKind === 'CacheJdbcPojoStoreFactory' || factoryKind === 'CacheJdbcBlobStoreFactory') {
-                    startSourcesFunction();
-
-                    $generatorJava.cacheStoreDataSource(storeFactory, res);
-                }
-            }
-        }
-    });
-
-    if (cluster.discovery.kind === 'Jdbc') {
-        const datasource = cluster.discovery.Jdbc;
-
-        if (datasource.dataSourceBean && datasource.dialect) {
-            const beanClassName = $generatorJava.dataSourceClassName(res, datasource);
-
-            if (beanClassName && !_.includes(datasources, beanClassName)) {
-                startSourcesFunction();
-
-                $generatorJava.cacheStoreDataSource(datasource, res);
-            }
-        }
-    }
-
-    if (storeFound)
-        res.endBlock('}');
-
-    return res;
-};
-
-/**
- * Generate cache store group.
- *
- * @param cache Cache descriptor.
- * @param domains Domain model descriptors.
- * @param cacheVarName Cache variable name.
- * @param res Resulting output with generated code.
- * @returns {*} Java code for cache store configuration.
- */
-$generatorJava.cacheStore = function(cache, domains, cacheVarName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!cacheVarName)
-        cacheVarName = $generatorJava.nextVariableName('cache', cache);
-
-    if (cache.cacheStoreFactory && cache.cacheStoreFactory.kind) {
-        const factoryKind = cache.cacheStoreFactory.kind;
-
-        const storeFactory = cache.cacheStoreFactory[factoryKind];
-
-        if (storeFactory) {
-            const storeFactoryDesc = $generatorCommon.STORE_FACTORIES[factoryKind];
-
-            const varName = 'storeFactory' + storeFactoryDesc.suffix;
-
-            if (factoryKind === 'CacheJdbcPojoStoreFactory') {
-                // Generate POJO store factory.
-                $generatorJava.declareVariable(res, varName, 'org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory', null, null, null, true);
-                res.deep++;
-
-                res.line('/** {@inheritDoc} */');
-                res.startBlock('@Override public ' + res.importClass('org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore') + ' create() {');
-
-                res.line('setDataSource(DataSources.INSTANCE_' + storeFactory.dataSourceBean + ');');
-
-                res.needEmptyLine = true;
-
-                res.line('return super.create();');
-                res.endBlock('}');
-                res.endBlock('};');
-
-                res.needEmptyLine = true;
-
-                res.line(varName + '.setDialect(new ' +
-                    res.importClass($generatorCommon.jdbcDialectClassName(storeFactory.dialect)) + '());');
-
-                res.needEmptyLine = true;
-
-                if (storeFactory.sqlEscapeAll) {
-                    res.line(varName + '.setSqlEscapeAll(true);');
-
-                    res.needEmptyLine = true;
-                }
-
-                const domainConfigs = _.filter(domains, function(domain) {
-                    return $generatorCommon.domainQueryMetadata(domain) === 'Configuration' &&
-                        $generatorCommon.isDefinedAndNotEmpty(domain.databaseTable);
-                });
-
-                if ($generatorCommon.isDefinedAndNotEmpty(domainConfigs)) {
-                    $generatorJava.declareVariable(res, 'jdbcTypes', 'java.util.Collection', 'java.util.ArrayList', 'org.apache.ignite.cache.store.jdbc.JdbcType');
-
-                    res.needEmptyLine = true;
-
-                    _.forEach(domainConfigs, function(domain) {
-                        if ($generatorCommon.isDefinedAndNotEmpty(domain.databaseTable))
-                            res.line('jdbcTypes.add(jdbcType' + $generatorJava.extractType(domain.valueType) + '(' + cacheVarName + '.getName()));');
-                    });
-
-                    res.needEmptyLine = true;
-
-                    res.line(varName + '.setTypes(jdbcTypes.toArray(new JdbcType[jdbcTypes.size()]));');
-
-                    res.needEmptyLine = true;
-                }
-
-                res.line(cacheVarName + '.setCacheStoreFactory(' + varName + ');');
-            }
-            else if (factoryKind === 'CacheJdbcBlobStoreFactory') {
-                // Generate POJO store factory.
-                $generatorJava.declareVariable(res, varName, 'org.apache.ignite.cache.store.jdbc.CacheJdbcBlobStoreFactory', null, null, null, storeFactory.connectVia === 'DataSource');
-
-                if (storeFactory.connectVia === 'DataSource') {
-                    res.deep++;
-
-                    res.line('/** {@inheritDoc} */');
-                    res.startBlock('@Override public ' + res.importClass('org.apache.ignite.cache.store.jdbc.CacheJdbcBlobStore') + ' create() {');
-
-                    res.line('setDataSource(DataSources.INSTANCE_' + storeFactory.dataSourceBean + ');');
-
-                    res.needEmptyLine = true;
-
-                    res.line('return super.create();');
-                    res.endBlock('}');
-                    res.endBlock('};');
-
-                    res.needEmptyLine = true;
-
-                    $generatorJava.property(res, varName, storeFactory, 'initSchema');
-                    $generatorJava.property(res, varName, storeFactory, 'createTableQuery');
-                    $generatorJava.property(res, varName, storeFactory, 'loadQuery');
-                    $generatorJava.property(res, varName, storeFactory, 'insertQuery');
-                    $generatorJava.property(res, varName, storeFactory, 'updateQuery');
-                    $generatorJava.property(res, varName, storeFactory, 'deleteQuery');
-                }
-                else {
-                    $generatorJava.property(res, varName, storeFactory, 'connectionUrl');
-
-                    if (storeFactory.user) {
-                        $generatorJava.property(res, varName, storeFactory, 'user');
-                        res.line(varName + '.setPassword(props.getProperty("ds.' + storeFactory.user + '.password"));');
-                    }
-                }
-
-                res.needEmptyLine = true;
-
-                res.line(cacheVarName + '.setCacheStoreFactory(' + varName + ');');
-            }
-            else
-                $generatorJava.beanProperty(res, cacheVarName, storeFactory, 'cacheStoreFactory', varName, storeFactoryDesc.className, storeFactoryDesc.fields, true);
-
-            res.needEmptyLine = true;
-        }
-    }
-
-    res.softEmptyLine();
-
-    $generatorJava.property(res, cacheVarName, cache, 'storeKeepBinary', null, null, false);
-    $generatorJava.property(res, cacheVarName, cache, 'loadPreviousValue', null, null, false);
-    $generatorJava.property(res, cacheVarName, cache, 'readThrough', null, null, false);
-    $generatorJava.property(res, cacheVarName, cache, 'writeThrough', null, null, false);
-
-    res.softEmptyLine();
-
-    if (cache.writeBehindEnabled) {
-        $generatorJava.property(res, cacheVarName, cache, 'writeBehindEnabled', null, null, false);
-        $generatorJava.property(res, cacheVarName, cache, 'writeBehindBatchSize', null, null, 512);
-        $generatorJava.property(res, cacheVarName, cache, 'writeBehindFlushSize', null, null, 10240);
-        $generatorJava.property(res, cacheVarName, cache, 'writeBehindFlushFrequency', null, null, 5000);
-        $generatorJava.property(res, cacheVarName, cache, 'writeBehindFlushThreadCount', null, null, 1);
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache node filter group.
-$generatorJava.cacheNodeFilter = function(cache, igfss, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('cache', cache);
-
-    switch (_.get(cache, 'nodeFilter.kind')) {
-        case 'IGFS':
-            const foundIgfs = _.find(igfss, (igfs) => igfs._id === cache.nodeFilter.IGFS.igfs);
-
-            if (foundIgfs) {
-                const predClsName = res.importClass('org.apache.ignite.internal.processors.igfs.IgfsNodePredicate');
-
-                res.line(`${varName}.setNodeFilter(new ${predClsName}("${foundIgfs.name}"));`);
-            }
-
-            break;
-
-        case 'OnNodes':
-            const nodes = cache.nodeFilter.OnNodes.nodeIds;
-
-            if ($generatorCommon.isDefinedAndNotEmpty(nodes)) {
-                const startQuote = res.importClass('java.util.UUID') + '.fromString("';
-
-                $generatorJava.fxVarArgs(res, varName + '.setNodeFilter(new ' +
-                    res.importClass('org.apache.ignite.internal.util.lang.GridNodePredicate'), true, nodes, '(', '))',
-                    startQuote, '")');
-            }
-
-            break;
-
-        case 'Custom':
-            res.line(varName + '.setNodeFilter(new ' + res.importClass(cache.nodeFilter.Custom.className) + '());');
-
-            break;
-
-        default: break;
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache concurrency group.
-$generatorJava.cacheConcurrency = function(cache, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('cache', cache);
-
-    $generatorJava.property(res, varName, cache, 'maxConcurrentAsyncOperations', null, null, 500);
-    $generatorJava.property(res, varName, cache, 'defaultLockTimeout', null, null, 0);
-    $generatorJava.enumProperty(res, varName, cache, 'atomicWriteOrderMode', 'org.apache.ignite.cache.CacheAtomicWriteOrderMode');
-    $generatorJava.enumProperty(res, varName, cache, 'writeSynchronizationMode', 'org.apache.ignite.cache.CacheWriteSynchronizationMode', null, 'PRIMARY_SYNC');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache rebalance group.
-$generatorJava.cacheRebalance = function(cache, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('cache', cache);
-
-    if (cache.cacheMode !== 'LOCAL') {
-        $generatorJava.enumProperty(res, varName, cache, 'rebalanceMode', 'org.apache.ignite.cache.CacheRebalanceMode', null, 'ASYNC');
-        $generatorJava.property(res, varName, cache, 'rebalanceThreadPoolSize', null, null, 1);
-        $generatorJava.property(res, varName, cache, 'rebalanceBatchSize', null, null, 524288);
-        $generatorJava.property(res, varName, cache, 'rebalanceBatchesPrefetchCount', null, null, 2);
-        $generatorJava.property(res, varName, cache, 'rebalanceOrder', null, null, 0);
-        $generatorJava.property(res, varName, cache, 'rebalanceDelay', null, null, 0);
-        $generatorJava.property(res, varName, cache, 'rebalanceTimeout', null, null, 10000);
-        $generatorJava.property(res, varName, cache, 'rebalanceThrottle', null, null, 0);
-    }
-
-    res.softEmptyLine();
-
-    if (cache.igfsAffinnityGroupSize) {
-        res.line(varName + '.setAffinityMapper(new ' + res.importClass('org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper') + '(' + cache.igfsAffinnityGroupSize + '));');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate cache server near cache group.
-$generatorJava.cacheServerNearCache = function(cache, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('cache', cache);
-
-    if (cache.cacheMode === 'PARTITIONED' && cache.nearCacheEnabled) {
-        res.needEmptyLine = true;
-
-        if (cache.nearConfiguration) {
-            $generatorJava.declareVariable(res, 'nearCfg', 'org.apache.ignite.configuration.NearCacheConfiguration');
-
-            res.needEmptyLine = true;
-
-            if (cache.nearConfiguration.nearStartSize) {
-                $generatorJava.property(res, 'nearCfg', cache.nearConfiguration, 'nearStartSize', null, null, 375000);
-
-                res.needEmptyLine = true;
-            }
-
-            if (cache.nearConfiguration.nearEvictionPolicy && cache.nearConfiguration.nearEvictionPolicy.kind) {
-                $generatorJava.evictionPolicy(res, 'nearCfg', cache.nearConfiguration.nearEvictionPolicy, 'nearEvictionPolicy');
-
-                res.needEmptyLine = true;
-            }
-
-            res.line(varName + '.setNearConfiguration(nearCfg);');
-
-            res.needEmptyLine = true;
-        }
-    }
-
-    return res;
-};
-
-// Generate cache statistics group.
-$generatorJava.cacheStatistics = function(cache, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('cache', cache);
-
-    $generatorJava.property(res, varName, cache, 'statisticsEnabled', null, null, false);
-    $generatorJava.property(res, varName, cache, 'managementEnabled', null, null, false);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate domain model query fields.
-$generatorJava.domainModelQueryFields = function(res, domain) {
-    const fields = domain.fields;
-
-    if (fields && fields.length > 0) {
-        $generatorJava.declareVariable(res, 'fields', 'java.util.LinkedHashMap', 'java.util.LinkedHashMap', 'java.lang.String', 'java.lang.String');
-
-        _.forEach(fields, function(field) {
-            res.line('fields.put("' + field.name + '", "' + $generatorCommon.JavaTypes.fullClassName(field.className) + '");');
-        });
-
-        res.needEmptyLine = true;
-
-        res.line('qryMeta.setFields(fields);');
-
-        res.needEmptyLine = true;
-    }
-};
-
-// Generate domain model query aliases.
-$generatorJava.domainModelQueryAliases = function(res, domain) {
-    const aliases = domain.aliases;
-
-    if (aliases && aliases.length > 0) {
-        $generatorJava.declareVariable(res, 'aliases', 'java.util.Map', 'java.util.HashMap', 'java.lang.String', 'java.lang.String');
-
-        _.forEach(aliases, function(alias) {
-            res.line('aliases.put("' + alias.field + '", "' + alias.alias + '");');
-        });
-
-        res.needEmptyLine = true;
-
-        res.line('qryMeta.setAliases(aliases);');
-
-        res.needEmptyLine = true;
-    }
-};
-
-// Generate domain model indexes.
-$generatorJava.domainModelQueryIndexes = function(res, domain) {
-    const indexes = domain.indexes;
-
-    if (indexes && indexes.length > 0) {
-        res.needEmptyLine = true;
-
-        $generatorJava.declareVariable(res, 'indexes', 'java.util.List', 'java.util.ArrayList', 'org.apache.ignite.cache.QueryIndex');
-
-        _.forEach(indexes, function(index) {
-            const fields = index.fields;
-
-            // One row generation for 1 field index.
-            if (fields && fields.length === 1) {
-                const field = index.fields[0];
-
-                res.line('indexes.add(new ' + res.importClass('org.apache.ignite.cache.QueryIndex') +
-                    '("' + field.name + '", ' +
-                    res.importClass('org.apache.ignite.cache.QueryIndexType') + '.' + index.indexType + ', ' +
-                    field.direction + ', "' + index.name + '"));');
-            }
-            else {
-                res.needEmptyLine = true;
-
-                $generatorJava.declareVariable(res, 'index', 'org.apache.ignite.cache.QueryIndex');
-
-                $generatorJava.property(res, 'index', index, 'name');
-                $generatorJava.enumProperty(res, 'index', index, 'indexType', 'org.apache.ignite.cache.QueryIndexType');
-
-                res.needEmptyLine = true;
-
-                if (fields && fields.length > 0) {
-                    $generatorJava.declareVariable(res, 'indFlds', 'java.util.LinkedHashMap', 'java.util.LinkedHashMap', 'String', 'Boolean');
-
-                    _.forEach(fields, function(field) {
-                        res.line('indFlds.put("' + field.name + '", ' + field.direction + ');');
-                    });
-
-                    res.needEmptyLine = true;
-
-                    res.line('index.setFields(indFlds);');
-
-                    res.needEmptyLine = true;
-                }
-
-                res.line('indexes.add(index);');
-            }
-        });
-
-        res.needEmptyLine = true;
-
-        res.line('qryMeta.setIndexes(indexes);');
-
-        res.needEmptyLine = true;
-    }
-};
-
-// Generate domain model db fields.
-$generatorJava.domainModelDatabaseFields = function(res, domain, fieldProperty) {
-    const dbFields = domain[fieldProperty];
-
-    if (dbFields && dbFields.length > 0) {
-        res.needEmptyLine = true;
-
-        res.importClass('java.sql.Types');
-
-        res.startBlock('jdbcType.' + $generatorCommon.toJavaName('set', fieldProperty) + '(');
-
-        const lastIx = dbFields.length - 1;
-
-        res.importClass('org.apache.ignite.cache.store.jdbc.JdbcTypeField');
-
-        _.forEach(dbFields, function(field, ix) {
-            res.line('new JdbcTypeField(' +
-                'Types.' + field.databaseFieldType + ', ' + '"' + field.databaseFieldName + '", ' +
-                res.importClass(field.javaFieldType) + '.class, ' + '"' + field.javaFieldName + '"' + ')' + (ix < lastIx ? ',' : ''));
-        });
-
-        res.endBlock(');');
-
-        res.needEmptyLine = true;
-    }
-};
-
-// Generate domain model general group.
-$generatorJava.domainModelGeneral = function(domain, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    switch ($generatorCommon.domainQueryMetadata(domain)) {
-        case 'Annotations':
-            if ($generatorCommon.isDefinedAndNotEmpty(domain.keyType) || $generatorCommon.isDefinedAndNotEmpty(domain.valueType)) {
-                const types = [];
-
-                if ($generatorCommon.isDefinedAndNotEmpty(domain.keyType))
-                    types.push($generatorJava.toJavaCode(res.importClass(domain.keyType), 'class'));
-                else
-                    types.push('???');
-
-                if ($generatorCommon.isDefinedAndNotEmpty(domain.valueType))
-                    types.push($generatorJava.toJavaCode(res.importClass(domain.valueType), 'class'));
-                else
-                    types.push('???');
-
-                if ($generatorCommon.isDefinedAndNotEmpty(types))
-                    $generatorJava.fxVarArgs(res, 'cache.setIndexedTypes', false, types);
-            }
-
-            break;
-
-        case 'Configuration':
-            $generatorJava.classNameProperty(res, 'jdbcTypes', domain, 'keyType');
-            $generatorJava.property(res, 'jdbcTypes', domain, 'valueType');
-
-            if ($generatorCommon.isDefinedAndNotEmpty(domain.fields)) {
-                res.needEmptyLine = true;
-
-                $generatorJava.classNameProperty(res, 'qryMeta', domain, 'keyType');
-                $generatorJava.property(res, 'qryMeta', domain, 'valueType');
-            }
-
-            break;
-
-        default:
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate domain model for query group.
-$generatorJava.domainModelQuery = function(domain, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.domainQueryMetadata(domain) === 'Configuration') {
-        $generatorJava.domainModelQueryFields(res, domain);
-        $generatorJava.domainModelQueryAliases(res, domain);
-        $generatorJava.domainModelQueryIndexes(res, domain);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate domain model for store group.
-$generatorJava.domainStore = function(domain, withTypes, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorJava.property(res, 'jdbcType', domain, 'databaseSchema');
-    $generatorJava.property(res, 'jdbcType', domain, 'databaseTable');
-
-    if (withTypes) {
-        $generatorJava.classNameProperty(res, 'jdbcType', domain, 'keyType');
-        $generatorJava.property(res, 'jdbcType', domain, 'valueType');
-    }
-
-    $generatorJava.domainModelDatabaseFields(res, domain, 'keyFields');
-    $generatorJava.domainModelDatabaseFields(res, domain, 'valueFields');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate domain model configs.
-$generatorJava.cacheDomains = function(domains, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    const domainConfigs = _.filter(domains, function(domain) {
-        return $generatorCommon.domainQueryMetadata(domain) === 'Configuration' &&
-            $generatorCommon.isDefinedAndNotEmpty(domain.fields);
-    });
-
-    // Generate domain model configs.
-    if ($generatorCommon.isDefinedAndNotEmpty(domainConfigs)) {
-        $generatorJava.declareVariable(res, 'queryEntities', 'java.util.Collection', 'java.util.ArrayList', 'org.apache.ignite.cache.QueryEntity');
-
-        _.forEach(domainConfigs, function(domain) {
-            if ($generatorCommon.isDefinedAndNotEmpty(domain.fields))
-                res.line('queryEntities.add(queryEntity' + $generatorJava.extractType(domain.valueType) + '());');
-        });
-
-        res.needEmptyLine = true;
-
-        res.line(varName + '.setQueryEntities(queryEntities);');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate cache configs.
-$generatorJava.cache = function(cache, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorJava.cacheGeneral(cache, varName, res);
-    $generatorJava.cacheMemory(cache, varName, res);
-    $generatorJava.cacheQuery(cache, cache.domains, varName, res);
-    $generatorJava.cacheStore(cache, cache.domains, varName, res);
-
-    const igfs = _.get(cache, 'nodeFilter.IGFS.instance');
-
-    $generatorJava.cacheNodeFilter(cache, igfs ? [igfs] : [], varName, res);
-    $generatorJava.cacheConcurrency(cache, varName, res);
-    $generatorJava.cacheRebalance(cache, varName, res);
-    $generatorJava.cacheServerNearCache(cache, varName, res);
-    $generatorJava.cacheStatistics(cache, varName, res);
-    $generatorJava.cacheDomains(cache.domains, varName, res);
-};
-
-// Generation of cache domain model in separate methods.
-$generatorJava.clusterDomains = function(caches, res) {
-    const domains = [];
-
-    const typeVarName = 'jdbcType';
-    const metaVarName = 'qryMeta';
-
-    _.forEach(caches, function(cache) {
-        _.forEach(cache.domains, function(domain) {
-            if (_.isNil(_.find(domains, function(m) {
-                return m === domain.valueType;
-            }))) {
-                $generatorJava.resetVariables(res);
-
-                const type = $generatorJava.extractType(domain.valueType);
-
-                if ($generatorCommon.isDefinedAndNotEmpty(domain.databaseTable)) {
-                    res.line('/**');
-                    res.line(' * Create JDBC type for ' + type + '.');
-                    res.line(' *');
-                    res.line(' * @param cacheName Cache name.');
-                    res.line(' * @return Configured JDBC type.');
-                    res.line(' */');
-                    res.startBlock('private static JdbcType jdbcType' + type + '(String cacheName) {');
-
-                    $generatorJava.declareVariable(res, typeVarName, 'org.apache.ignite.cache.store.jdbc.JdbcType');
-
-                    res.needEmptyLine = true;
-
-                    res.line(typeVarName + '.setCacheName(cacheName);');
-
-                    $generatorJava.domainStore(domain, true, res);
-
-                    res.needEmptyLine = true;
-
-                    res.line('return ' + typeVarName + ';');
-                    res.endBlock('}');
-
-                    res.needEmptyLine = true;
-                }
-
-                if ($generatorCommon.domainQueryMetadata(domain) === 'Configuration' &&
-                    $generatorCommon.isDefinedAndNotEmpty(domain.fields)) {
-                    res.line('/**');
-                    res.line(' * Create SQL Query descriptor for ' + type + '.');
-                    res.line(' *');
-                    res.line(' * @return Configured query entity.');
-                    res.line(' */');
-                    res.startBlock('private static QueryEntity queryEntity' + type + '() {');
-
-                    $generatorJava.declareVariable(res, metaVarName, 'org.apache.ignite.cache.QueryEntity');
-
-                    $generatorJava.classNameProperty(res, metaVarName, domain, 'keyType');
-                    $generatorJava.property(res, metaVarName, domain, 'valueType');
-
-                    res.needEmptyLine = true;
-
-                    $generatorJava.domainModelQuery(domain, res);
-
-                    res.emptyLineIfNeeded();
-                    res.line('return ' + metaVarName + ';');
-
-                    res.needEmptyLine = true;
-
-                    res.endBlock('}');
-                }
-
-                domains.push(domain.valueType);
-            }
-        });
-    });
-};
-
-/**
- * @param prefix Variable prefix.
- * @param obj Object to process.
- * @param names Known names to generate next unique name.
- */
-$generatorJava.nextVariableName = function(prefix, obj, names) {
-    let nextName = $generatorCommon.toJavaName(prefix, obj.name);
-
-    let ix = 0;
-
-    const checkNextName = (name) => name === nextName + (ix === 0 ? '' : '_' + ix);
-
-    while (_.find(names, (name) => checkNextName(name)))
-        ix++;
-
-    if (ix > 0)
-        nextName = nextName + '_' + ix;
-
-    return nextName;
-};
-
-// Generate cluster caches.
-$generatorJava.clusterCaches = function(caches, igfss, isSrvCfg, res) {
-    function clusterCache(cache, names) {
-        res.emptyLineIfNeeded();
-
-        const cacheName = $generatorJava.nextVariableName('cache', cache, names);
-
-        $generatorJava.resetVariables(res);
-
-        const hasDatasource = $generatorCommon.cacheHasDatasource(cache);
-
-        res.line('/**');
-        res.line(' * Create configuration for cache "' + cache.name + '".');
-        res.line(' *');
-        res.line(' * @return Configured cache.');
-
-        if (hasDatasource)
-            res.line(' * @throws Exception if failed to create cache configuration.');
-
-        res.line(' */');
-        res.startBlock('public static CacheConfiguration ' + cacheName + '()' + (hasDatasource ? ' throws Exception' : '') + ' {');
-
-        $generatorJava.declareVariable(res, cacheName, 'org.apache.ignite.configuration.CacheConfiguration');
-
-        $generatorJava.cache(cache, cacheName, res);
-
-        res.line('return ' + cacheName + ';');
-        res.endBlock('}');
-
-        names.push(cacheName);
-
-        res.needEmptyLine = true;
-    }
-
-    if (!res)
-        res = $generatorCommon.builder();
-
-    const names = [];
-
-    if ($generatorCommon.isDefinedAndNotEmpty(caches)) {
-        res.emptyLineIfNeeded();
-
-        _.forEach(caches, function(cache) {
-            clusterCache(cache, names);
-        });
-
-        res.needEmptyLine = true;
-    }
-
-    if (isSrvCfg && $generatorCommon.isDefinedAndNotEmpty(igfss)) {
-        res.emptyLineIfNeeded();
-
-        _.forEach(igfss, function(igfs) {
-            clusterCache($generatorCommon.igfsDataCache(igfs), names);
-            clusterCache($generatorCommon.igfsMetaCache(igfs), names);
-        });
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate cluster caches.
-$generatorJava.clusterCacheUse = function(caches, igfss, res) {
-    function clusterCacheInvoke(cache, names) {
-        names.push($generatorJava.nextVariableName('cache', cache, names));
-    }
-
-    if (!res)
-        res = $generatorCommon.builder();
-
-    const cacheNames = [];
-
-    _.forEach(caches, function(cache) {
-        clusterCacheInvoke(cache, cacheNames);
-    });
-
-    const igfsNames = [];
-
-    _.forEach(igfss, function(igfs) {
-        clusterCacheInvoke($generatorCommon.igfsDataCache(igfs), igfsNames);
-        clusterCacheInvoke($generatorCommon.igfsMetaCache(igfs), igfsNames);
-    });
-
-    const allCacheNames = cacheNames.concat(igfsNames);
-
-    if (allCacheNames.length) {
-        res.line('cfg.setCacheConfiguration(' + allCacheNames.join('(), ') + '());');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Get class name from fully specified class path.
-$generatorJava.extractType = function(fullType) {
-    return fullType.substring(fullType.lastIndexOf('.') + 1);
-};
-
-/**
- * Generate java class code.
- *
- * @param domain Domain model object.
- * @param key If 'true' then key class should be generated.
- * @param pkg Package name.
- * @param useConstructor If 'true' then empty and full constructors should be generated.
- * @param includeKeyFields If 'true' then include key fields into value POJO.
- * @param res Resulting output with generated code.
- */
-$generatorJava.javaClassCode = function(domain, key, pkg, useConstructor, includeKeyFields, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    const type = $generatorJava.extractType(key ? domain.keyType : domain.valueType);
-
-    // Class comment.
-    res.line('/**');
-    res.line(' * ' + type + ' definition.');
-    res.line(' *');
-    res.line(' * ' + $generatorCommon.mainComment('POJO'));
-    res.line(' */');
-
-    res.startBlock('public class ' + type + ' implements ' + res.importClass('java.io.Serializable') + ' {');
-
-    res.line('/** */');
-    res.line('private static final long serialVersionUID = 0L;');
-    res.needEmptyLine = true;
-
-    const allFields = (key || includeKeyFields) ? domain.keyFields.slice() : [];
-
-    if (!key) {
-        _.forEach(domain.valueFields, (valFld) => {
-            if (_.findIndex(allFields, (fld) => fld.javaFieldName === valFld.javaFieldName) < 0)
-                allFields.push(valFld);
-        });
-    }
-
-    // Generate allFields declaration.
-    _.forEach(allFields, function(field) {
-        const fldName = field.javaFieldName;
-
-        res.line('/** Value for ' + fldName + '. */');
-
-        res.line('private ' + res.importClass(field.javaFieldType) + ' ' + fldName + ';');
-
-        res.needEmptyLine = true;
-    });
-
-    // Generate constructors.
-    if (useConstructor) {
-        res.line('/**');
-        res.line(' * Empty constructor.');
-        res.line(' */');
-        res.startBlock('public ' + type + '() {');
-        res.line('// No-op.');
-        res.endBlock('}');
-
-        res.needEmptyLine = true;
-
-        res.line('/**');
-        res.line(' * Full constructor.');
-        res.line(' */');
-        res.startBlock('public ' + type + '(');
-
-        _.forEach(allFields, function(field, idx) {
-            res.line(res.importClass(field.javaFieldType) + ' ' + field.javaFieldName + (idx < allFields.length - 1 ? ',' : ''));
-        });
-
-        res.endBlock(') {');
-
-        res.startBlock();
-
-        _.forEach(allFields, (field) => res.line('this.' + field.javaFieldName + ' = ' + field.javaFieldName + ';'));
-
-        res.endBlock('}');
-
-        res.needEmptyLine = true;
-    }
-
-    // Generate getters and setters methods.
-    _.forEach(allFields, function(field) {
-        const fldName = field.javaFieldName;
-
-        const fldType = res.importClass(field.javaFieldType);
-
-        res.line('/**');
-        res.line(' * Gets ' + fldName + '.');
-        res.line(' *');
-        res.line(' * @return Value for ' + fldName + '.');
-        res.line(' */');
-        res.startBlock('public ' + fldType + ' ' + $generatorCommon.toJavaName('get', fldName) + '() {');
-        res.line('return ' + fldName + ';');
-        res.endBlock('}');
-
-        res.needEmptyLine = true;
-
-        res.line('/**');
-        res.line(' * Sets ' + fldName + '.');
-        res.line(' *');
-        res.line(' * @param ' + fldName + ' New value for ' + fldName + '.');
-        res.line(' */');
-        res.startBlock('public void ' + $generatorCommon.toJavaName('set', fldName) + '(' + fldType + ' ' + fldName + ') {');
-        res.line('this.' + fldName + ' = ' + fldName + ';');
-        res.endBlock('}');
-
-        res.needEmptyLine = true;
-    });
-
-    // Generate equals() method.
-    res.line('/** {@inheritDoc} */');
-    res.startBlock('@Override public boolean equals(Object o) {');
-    res.startBlock('if (this == o)');
-    res.line('return true;');
-    res.endBlock();
-    res.append('');
-
-    res.startBlock('if (!(o instanceof ' + type + '))');
-    res.line('return false;');
-    res.endBlock();
-
-    res.needEmptyLine = true;
-
-    res.line(type + ' that = (' + type + ')o;');
-
-    _.forEach(allFields, function(field) {
-        res.needEmptyLine = true;
-
-        const javaName = field.javaFieldName;
-        const javaType = field.javaFieldType;
-
-        if ($generatorCommon.JavaTypes.isJavaPrimitive(javaType)) {
-            if (javaType === 'float')
-                res.startBlock('if (Float.compare(' + javaName + ', that.' + javaName + ') != 0)');
-            else if (javaType === 'double')
-                res.startBlock('if (Double.compare(' + javaName + ', that.' + javaName + ') != 0)');
-            else
-                res.startBlock('if (' + javaName + ' != that.' + javaName + ')');
-        }
-        else
-            res.startBlock('if (' + javaName + ' != null ? !' + javaName + '.equals(that.' + javaName + ') : that.' + javaName + ' != null)');
-
-        res.line('return false;');
-        res.endBlock();
-    });
-
-    res.needEmptyLine = true;
-
-    res.line('return true;');
-    res.endBlock('}');
-
-    res.needEmptyLine = true;
-
-    // Generate hashCode() method.
-    res.line('/** {@inheritDoc} */');
-    res.startBlock('@Override public int hashCode() {');
-
-    let first = true;
-    let tempVar = false;
-
-    _.forEach(allFields, function(field) {
-        const javaName = field.javaFieldName;
-        const javaType = field.javaFieldType;
-
-        if (!first)
-            res.needEmptyLine = true;
-
-        if ($generatorCommon.JavaTypes.isJavaPrimitive(javaType)) {
-            if (javaType === 'boolean')
-                res.line(first ? 'int res = ' + javaName + ' ? 1 : 0;' : 'res = 31 * res + (' + javaName + ' ? 1 : 0);');
-            else if (javaType === 'byte' || javaType === 'short')
-                res.line(first ? 'int res = (int)' + javaName + ';' : 'res = 31 * res + (int)' + javaName + ';');
-            else if (javaType === 'int')
-                res.line(first ? 'int res = ' + javaName + ';' : 'res = 31 * res + ' + javaName + ';');
-            else if (javaType === 'long') {
-                res.line(first
-                    ? 'int res = (int)(' + javaName + ' ^ (' + javaName + ' >>> 32));'
-                    : 'res = 31 * res + (int)(' + javaName + ' ^ (' + javaName + ' >>> 32));');
-            }
-            else if (javaType === 'float') {
-                res.line(first
-                    ? 'int res = ' + javaName + ' != +0.0f ? Float.floatToIntBits(' + javaName + ') : 0;'
-                    : 'res = 31 * res + (' + javaName + ' != +0.0f ? Float.floatToIntBits(' + javaName + ') : 0);');
-            }
-            else if (javaType === 'double') {
-                res.line((tempVar ? 'ig_hash_temp' : 'long ig_hash_temp') + ' = Double.doubleToLongBits(' + javaName + ');');
-
-                res.needEmptyLine = true;
-
-                res.line(first
-                        ? 'int res = (int)(ig_hash_temp ^ (ig_hash_temp >>> 32));'
-                        : 'res = 31 * res + (int)(ig_hash_temp ^ (ig_hash_temp >>> 32));');
-
-                tempVar = true;
-            }
-        }
-        else {
-            res.line(first
-                ? 'int res = ' + javaName + ' != null ? ' + javaName + '.hashCode() : 0;'
-                : 'res = 31 * res + (' + javaName + ' != null ? ' + javaName + '.hashCode() : 0);');
-        }
-
-        first = false;
-    });
-
-    res.needEmptyLine = true;
-    res.line('return res;');
-    res.endBlock('}');
-    res.needEmptyLine = true;
-
-    // Generate toString() method.
-    res.line('/** {@inheritDoc} */');
-    res.startBlock('@Override public String toString() {');
-
-    res.startBlock('return \"' + type + ' [" + ');
-
-    _.forEach(allFields, function(field, idx) {
-        res.line('\"' + field.javaFieldName + '=\" + ' + field.javaFieldName + (idx < allFields.length - 1 ? ' + ", " + ' : ' +'));
-    });
-
-    res.endBlock('"]";');
-    res.endBlock('}');
-
-    res.endBlock('}');
-
-    return 'package ' + pkg + ';' + '\n\n' + res.generateImports() + '\n' + res.generateStaticImports() + '\n' + res.asString();
-};
-
-/**
- * Generate source code for type by its domain models.
- *
- * @param caches List of caches to generate POJOs for.
- * @param useConstructor If 'true' then generate constructors.
- * @param includeKeyFields If 'true' then include key fields into value POJO.
- */
-$generatorJava.pojos = function(caches, useConstructor, includeKeyFields) {
-    const pojos = [];
-
-    _.forEach(caches, (cache) => {
-        _.forEach(cache.domains, (domain) => {
-            // Process only  domains with 'generatePojo' flag and skip already generated classes.
-            if (domain.generatePojo && !_.find(pojos, {valueType: domain.valueType}) &&
-                // Skip domain models without value fields.
-                $generatorCommon.isDefinedAndNotEmpty(domain.valueFields)) {
-                const pojo = {};
-
-                // Key class generation only if key is not build in java class.
-                if (!_.isNil(domain.keyFields) && domain.keyFields.length > 0) {
-                    pojo.keyType = domain.keyType;
-                    pojo.keyClass = $generatorJava.javaClassCode(domain, true,
-                        domain.keyType.substring(0, domain.keyType.lastIndexOf('.')), useConstructor, includeKeyFields);
-                }
-
-                pojo.valueType = domain.valueType;
-                pojo.valueClass = $generatorJava.javaClassCode(domain, false,
-                    domain.valueType.substring(0, domain.valueType.lastIndexOf('.')), useConstructor, includeKeyFields);
-
-                pojos.push(pojo);
-            }
-        });
-    });
-
-    return pojos;
-};
-
-/**
- * @param type Full type name.
- * @returns Field java type name.
- */
-$generatorJava.javaTypeName = function(type) {
-    const ix = $generatorJava.javaBuiltInClasses.indexOf(type);
-
-    const resType = ix >= 0 ? $generatorJava.javaBuiltInFullNameClasses[ix] : type;
-
-    return resType.indexOf('java.lang.') >= 0 ? resType.substring(10) : resType;
-};
-
-/**
- * Java code generator for cluster's SSL configuration.
- *
- * @param cluster Cluster to get SSL configuration.
- * @param res Optional configuration presentation builder object.
- * @returns Configuration presentation builder object
- */
-$generatorJava.clusterSsl = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (cluster.sslEnabled && !_.isNil(cluster.sslContextFactory)) {
-
-        cluster.sslContextFactory.keyStorePassword = $generatorCommon.isDefinedAndNotEmpty(cluster.sslContextFactory.keyStoreFilePath) ?
-            'props.getProperty("ssl.key.storage.password").toCharArray()' : null;
-
-        cluster.sslContextFactory.trustStorePassword = $generatorCommon.isDefinedAndNotEmpty(cluster.sslContextFactory.trustStoreFilePath) ?
-            'props.getProperty("ssl.trust.storage.password").toCharArray()' : null;
-
-        const propsDesc = $generatorCommon.isDefinedAndNotEmpty(cluster.sslContextFactory.trustManagers) ?
-            $generatorCommon.SSL_CONFIGURATION_TRUST_MANAGER_FACTORY.fields :
-            $generatorCommon.SSL_CONFIGURATION_TRUST_FILE_FACTORY.fields;
-
-        $generatorJava.beanProperty(res, 'cfg', cluster.sslContextFactory, 'sslContextFactory', 'sslContextFactory',
-            'org.apache.ignite.ssl.SslContextFactory', propsDesc, true);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-/**
- * Java code generator for cluster's IGFS configurations.
- *
- * @param igfss List of configured IGFS.
- * @param varName Name of IGFS configuration variable.
- * @param res Optional configuration presentation builder object.
- * @returns Configuration presentation builder object.
- */
-$generatorJava.igfss = function(igfss, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.isDefinedAndNotEmpty(igfss)) {
-        res.emptyLineIfNeeded();
-
-        const arrayName = 'fileSystems';
-        const igfsInst = 'igfs';
-
-        res.line(res.importClass('org.apache.ignite.configuration.FileSystemConfiguration') + '[] ' + arrayName + ' = new FileSystemConfiguration[' + igfss.length + '];');
-
-        _.forEach(igfss, function(igfs, ix) {
-            $generatorJava.declareVariable(res, igfsInst, 'org.apache.ignite.configuration.FileSystemConfiguration');
-
-            $generatorJava.igfsGeneral(igfs, igfsInst, res);
-            $generatorJava.igfsIPC(igfs, igfsInst, res);
-            $generatorJava.igfsFragmentizer(igfs, igfsInst, res);
-            $generatorJava.igfsDualMode(igfs, igfsInst, res);
-            $generatorJava.igfsSecondFS(igfs, igfsInst, res);
-            $generatorJava.igfsMisc(igfs, igfsInst, res);
-
-            res.line(arrayName + '[' + ix + '] = ' + igfsInst + ';');
-
-            res.needEmptyLine = true;
-        });
-
-        res.line(varName + '.' + 'setFileSystemConfiguration(' + arrayName + ');');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-/**
- * Java code generator for IGFS IPC configuration.
- *
- * @param igfs Configured IGFS.
- * @param varName Name of IGFS configuration variable.
- * @param res Optional configuration presentation builder object.
- * @returns Configuration presentation builder object.
- */
-$generatorJava.igfsIPC = function(igfs, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('igfs', igfs);
-
-    if (igfs.ipcEndpointEnabled) {
-        const desc = $generatorCommon.IGFS_IPC_CONFIGURATION;
-
-        $generatorJava.beanProperty(res, varName, igfs.ipcEndpointConfiguration, 'ipcEndpointConfiguration', 'ipcEndpointCfg',
-            desc.className, desc.fields, true);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-/**
- * Java code generator for IGFS fragmentizer configuration.
- *
- * @param igfs Configured IGFS.
- * @param varName Name of IGFS configuration variable.
- * @param res Optional configuration presentation builder object.
- * @returns Configuration presentation builder object.
- */
-$generatorJava.igfsFragmentizer = function(igfs, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('igfs', igfs);
-
-    if (igfs.fragmentizerEnabled) {
-        $generatorJava.property(res, varName, igfs, 'fragmentizerConcurrentFiles', null, null, 0);
-        $generatorJava.property(res, varName, igfs, 'fragmentizerThrottlingBlockLength', null, null, 16777216);
-        $generatorJava.property(res, varName, igfs, 'fragmentizerThrottlingDelay', null, null, 200);
-
-        res.needEmptyLine = true;
-    }
-    else
-        $generatorJava.property(res, varName, igfs, 'fragmentizerEnabled');
-
-    return res;
-};
-
-/**
- * Java code generator for IGFS dual mode configuration.
- *
- * @param igfs Configured IGFS.
- * @param varName Name of IGFS configuration variable.
- * @param res Optional configuration presentation builder object.
- * @returns Configuration presentation builder object.
- */
-$generatorJava.igfsDualMode = function(igfs, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('igfs', igfs);
-
-    $generatorJava.property(res, varName, igfs, 'dualModeMaxPendingPutsSize', null, null, 0);
-
-    if ($generatorCommon.isDefinedAndNotEmpty(igfs.dualModePutExecutorService))
-        res.line(varName + '.' + $generatorJava.setterName('dualModePutExecutorService') + '(new ' + res.importClass(igfs.dualModePutExecutorService) + '());');
-
-    $generatorJava.property(res, varName, igfs, 'dualModePutExecutorServiceShutdown', null, null, false);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-$generatorJava.igfsSecondFS = function(igfs, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('igfs', igfs);
-
-    if (igfs.secondaryFileSystemEnabled) {
-        const secondFs = igfs.secondaryFileSystem || {};
-
-        const nameDefined = $generatorCommon.isDefinedAndNotEmpty(secondFs.userName);
-        const cfgDefined = $generatorCommon.isDefinedAndNotEmpty(secondFs.cfgPath);
-
-        res.line(varName + '.setSecondaryFileSystem(new ' +
-            res.importClass('org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem') + '(' +
-                $generatorJava.constructorArg(secondFs, 'uri', null) +
-                (cfgDefined || nameDefined ? $generatorJava.constructorArg(secondFs, 'cfgPath', null, true) : '') +
-                $generatorJava.constructorArg(secondFs, 'userName', null, true, true) +
-            '));');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-/**
- * Java code generator for IGFS general configuration.
- *
- * @param igfs Configured IGFS.
- * @param varName Name of IGFS configuration variable.
- * @param res Optional configuration presentation builder object.
- * @returns Configuration presentation builder object.
- */
-$generatorJava.igfsGeneral = function(igfs, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('igfs', igfs);
-
-    if ($generatorCommon.isDefinedAndNotEmpty(igfs.name)) {
-        igfs.dataCacheName = $generatorCommon.igfsDataCache(igfs).name;
-        igfs.metaCacheName = $generatorCommon.igfsMetaCache(igfs).name;
-
-        $generatorJava.property(res, varName, igfs, 'name');
-        $generatorJava.property(res, varName, igfs, 'dataCacheName');
-        $generatorJava.property(res, varName, igfs, 'metaCacheName');
-        $generatorJava.enumProperty(res, varName, igfs, 'defaultMode', 'org.apache.ignite.igfs.IgfsMode', null, 'DUAL_ASYNC');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-/**
- * Java code generator for IGFS misc configuration.
- *
- * @param igfs Configured IGFS.
- * @param varName Name of IGFS configuration variable.
- * @param res Optional configuration presentation builder object.
- * @returns Configuration presentation builder object.
- */
-$generatorJava.igfsMisc = function(igfs, varName, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!varName)
-        varName = $generatorJava.nextVariableName('igfs', igfs);
-
-    $generatorJava.property(res, varName, igfs, 'blockSize', null, null, 65536);
-    $generatorJava.property(res, varName, igfs, 'streamBufferSize', null, null, 65536);
-    $generatorJava.property(res, varName, igfs, 'maxSpaceSize', null, null, 0);
-    $generatorJava.property(res, varName, igfs, 'maximumTaskRangeLength', null, null, 0);
-    $generatorJava.property(res, varName, igfs, 'managementPort', null, null, 11400);
-    $generatorJava.property(res, varName, igfs, 'perNodeBatchSize', null, null, 100);
-    $generatorJava.property(res, varName, igfs, 'perNodeParallelBatchCount', null, null, 8);
-    $generatorJava.property(res, varName, igfs, 'prefetchBlocks', null, null, 0);
-    $generatorJava.property(res, varName, igfs, 'sequentialReadsBeforePrefetch', null, null, 0);
-    $generatorJava.property(res, varName, igfs, 'trashPurgeTimeout', null, null, 1000);
-    $generatorJava.property(res, varName, igfs, 'colocateMetadata', null, null, true);
-    $generatorJava.property(res, varName, igfs, 'relaxedConsistency', null, null, true);
-
-    if (igfs.pathModes && igfs.pathModes.length > 0) {
-        res.needEmptyLine = true;
-
-        $generatorJava.declareVariable(res, 'pathModes', 'java.util.Map', 'java.util.HashMap', 'String', 'org.apache.ignite.igfs.IgfsMode');
-
-        _.forEach(igfs.pathModes, function(pair) {
-            res.line('pathModes.put("' + pair.path + '", IgfsMode.' + pair.mode + ');');
-        });
-
-        res.needEmptyLine = true;
-
-        res.line(varName + '.setPathModes(pathModes);');
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-$generatorJava.clusterConfiguration = function(cluster, clientNearCfg, res) {
-    $generatorJava.clusterGeneral(cluster, clientNearCfg, res);
-
-    $generatorJava.clusterAtomics(cluster.atomicConfiguration, res);
-
-    $generatorJava.clusterBinary(cluster.binaryConfiguration, res);
-
-    $generatorJava.clusterCacheKeyConfiguration(cluster.cacheKeyConfiguration, res);
-
-    $generatorJava.clusterCollision(cluster.collision, res);
-
-    $generatorJava.clusterCommunication(cluster, res);
-
-    $generatorJava.clusterConnector(cluster.connector, res);
-
-    $generatorJava.clusterDeployment(cluster, res);
-
-    $generatorJava.clusterEvents(cluster, res);
-
-    $generatorJava.clusterFailover(cluster, res);
-
-    $generatorJava.clusterLogger(cluster.logger, res);
-
-    $generatorJava.clusterODBC(cluster.odbc, res);
-
-    $generatorJava.clusterMarshaller(cluster, res);
-
-    $generatorJava.clusterMetrics(cluster, res);
-
-    $generatorJava.clusterSwap(cluster, res);
-
-    $generatorJava.clusterTime(cluster, res);
-
-    $generatorJava.clusterPools(cluster, res);
-
-    $generatorJava.clusterTransactions(cluster.transactionConfiguration, res);
-
-    const isSrvCfg = _.isNil(clientNearCfg);
-
-    if (isSrvCfg)
-        $generatorJava.clusterCacheUse(cluster.caches, cluster.igfss, res);
-
-    $generatorJava.clusterSsl(cluster, res);
-
-    if (isSrvCfg)
-        $generatorJava.igfss(cluster.igfss, 'cfg', res);
-
-    $generatorJava.clusterUserAttributes(cluster, res);
-
-    return res;
-};
-
-// Generate loading of secret properties file.
-$generatorJava.tryLoadSecretProperties = function(cluster, res) {
-    if ($generatorCommon.secretPropertiesNeeded(cluster)) {
-        res.importClass('org.apache.ignite.configuration.IgniteConfiguration');
-
-        $generatorJava.declareVariableCustom(res, 'props', 'java.util.Properties', 'new Properties()', 'private static final');
-
-        res.startBlock('static {');
-        res.startBlock('try (' + res.importClass('java.io.InputStream') + ' in = IgniteConfiguration.class.getClassLoader().getResourceAsStream("secret.properties")) {');
-        res.line('props.load(in);');
-        res.endBlock('}');
-        res.startBlock('catch (Exception ignored) {');
-        res.line('// No-op.');
-        res.endBlock('}');
-        res.endBlock('}');
-
-        res.needEmptyLine = true;
-    }
-};
-
-/**
- * Function to generate java code for cluster configuration.
- *
- * @param cluster Cluster to process.
- * @param pkg Package name.
- * @param javaClass Class name for generate factory class otherwise generate code snippet.
- * @param clientNearCfg Optional near cache configuration for client node.
- */
-$generatorJava.cluster = function(cluster, pkg, javaClass, clientNearCfg) {
-    const res = $generatorCommon.builder();
-
-    const isSrvCfg = _.isNil(clientNearCfg);
-
-    if (cluster) {
-        const resCfg = $generatorJava.clusterConfiguration(cluster, clientNearCfg, $generatorCommon.builder());
-
-        res.mergeProps(resCfg);
-
-        res.line('/**');
-        res.line(' * ' + $generatorCommon.mainComment('configuration'));
-        res.line(' */');
-        res.startBlock('public class ' + javaClass + ' {');
-
-        $generatorJava.tryLoadSecretProperties(cluster, res);
-
-        $generatorJava.clusterDataSources(cluster, res);
-
-        res.line('/**');
-        res.line(' * Configure grid.');
-        res.line(' *');
-        res.line(' * @return Ignite configuration.');
-        res.line(' * @throws Exception If failed to construct Ignite configuration instance.');
-        res.line(' */');
-        res.startBlock('public static IgniteConfiguration createConfiguration() throws Exception {');
-
-        res.mergeLines(resCfg);
-
-        res.needEmptyLine = true;
-
-        res.line('return cfg;');
-        res.endBlock('}');
-
-        res.needEmptyLine = true;
-
-        $generatorJava.clusterDomains(cluster.caches, res);
-
-        $generatorJava.clusterCaches(cluster.caches, cluster.igfss, isSrvCfg, res);
-
-        // TODO IGNITE-2269 Remove specified methods after implamentation of extended constructors.
-        $generatorJava.binaryTypeConfigurations(cluster.binaryConfiguration, res);
-
-        res.needEmptyLine = true;
-
-        if (clientNearCfg) {
-            res.line('/**');
-            res.line(' * Configure client near cache configuration.');
-            res.line(' *');
-            res.line(' * @return Near cache configuration.');
-            res.line(' * @throws Exception If failed to construct near cache configuration instance.');
-            res.line(' */');
-            res.startBlock('public static NearCacheConfiguration createNearCacheConfiguration() throws Exception {');
-
-            $generatorJava.resetVariables(res);
-
-            $generatorJava.declareVariable(res, 'clientNearCfg', 'org.apache.ignite.configuration.NearCacheConfiguration');
-
-            if (clientNearCfg.nearStartSize) {
-                $generatorJava.property(res, 'clientNearCfg', clientNearCfg, 'nearStartSize');
-
-                res.needEmptyLine = true;
-            }
-
-            if (clientNearCfg.nearEvictionPolicy && clientNearCfg.nearEvictionPolicy.kind)
-                $generatorJava.evictionPolicy(res, 'clientNearCfg', clientNearCfg.nearEvictionPolicy, 'nearEvictionPolicy');
-
-            res.line('return clientNearCfg;');
-            res.endBlock('}');
-
-            res.needEmptyLine = true;
-        }
-
-        res.endBlock('}');
-
-        return 'package ' + pkg + ';\n\n' + res.generateImports() + '\n' + res.generateStaticImports() + '\n' + res.asString();
-    }
-
-    return res.asString();
-};
-
-/** Generate data source class name for specified store factory.
- *
- * @param res Optional configuration presentation builder object.
- * @param storeFactory Store factory for data source class name generation.
- * @returns {*} Data source class name.
- */
-$generatorJava.dataSourceClassName = function(res, storeFactory) {
-    const dialect = storeFactory.connectVia ? (storeFactory.connectVia === 'DataSource' ? storeFactory.dialect : null) : storeFactory.dialect;
-
-    if (dialect) {
-        const dataSourceBean = storeFactory.dataSourceBean;
-
-        const dsClsName = $generatorCommon.dataSourceClassName(dialect);
-
-        const varType = res.importClass(dsClsName);
-
-        return $generatorCommon.toJavaName(varType, dataSourceBean);
-    }
-
-    return null;
-};
-
-// Descriptors for generation of demo data.
-const PREDEFINED_QUERIES = [
-    {
-        schema: 'CARS',
-        type: 'PARKING',
-        create: 'CREATE TABLE IF NOT EXISTS CARS.PARKING (\n' +
-            'ID       INTEGER     NOT NULL PRIMARY KEY,\n' +
-            'NAME     VARCHAR(50) NOT NULL,\n' +
-            'CAPACITY INTEGER NOT NULL)',
-        clearQuery: 'DELETE FROM CARS.PARKING',
-        insertCntConsts: [{name: 'DEMO_MAX_PARKING_CNT', val: 5, comment: 'How many parkings to generate.'}],
-        insertPattern: ['INSERT INTO CARS.PARKING(ID, NAME, CAPACITY) VALUES(?, ?, ?)'],
-        fillInsertParameters(res) {
-            res.line('stmt.setInt(1, id);');
-            res.line('stmt.setString(2, "Parking #" + (id + 1));');
-            res.line('stmt.setInt(3, 10 + rnd.nextInt(20));');
-        },
-        selectQuery: ['SELECT * FROM PARKING WHERE CAPACITY >= 20']
-    },
-    {
-        schema: 'CARS',
-        type: 'CAR',
-        create: 'CREATE TABLE IF NOT EXISTS CARS.CAR (\n' +
-            'ID         INTEGER NOT NULL PRIMARY KEY,\n' +
-            'PARKING_ID INTEGER NOT NULL,\n' +
-            'NAME       VARCHAR(50) NOT NULL);',
-        clearQuery: 'DELETE FROM CARS.CAR',
-        rndRequired: true,
-        insertCntConsts: [
-            {name: 'DEMO_MAX_CAR_CNT', val: 10, comment: 'How many cars to generate.'},
-            {name: 'DEMO_MAX_PARKING_CNT', val: 5, comment: 'How many parkings to generate.'}
-        ],
-        insertPattern: ['INSERT INTO CARS.CAR(ID, PARKING_ID, NAME) VALUES(?, ?, ?)'],
-        fillInsertParameters(res) {
-            res.line('stmt.setInt(1, id);');
-            res.line('stmt.setInt(2, rnd.nextInt(DEMO_MAX_PARKING_CNT));');
-            res.line('stmt.setString(3, "Car #" + (id + 1));');
-        },
-        selectQuery: ['SELECT * FROM CAR WHERE PARKINGID = 2']
-    },
-    {
-        type: 'COUNTRY',
-        create: 'CREATE TABLE IF NOT EXISTS COUNTRY (\n' +
-            'ID         INTEGER NOT NULL PRIMARY KEY,\n' +
-            'NAME       VARCHAR(50),\n' +
-            'POPULATION INTEGER NOT NULL);',
-        clearQuery: 'DELETE FROM COUNTRY',
-        insertCntConsts: [{name: 'DEMO_MAX_COUNTRY_CNT', val: 5, comment: 'How many countries to generate.'}],
-        insertPattern: ['INSERT INTO COUNTRY(ID, NAME, POPULATION) VALUES(?, ?, ?)'],
-        fillInsertParameters(res) {
-            res.line('stmt.setInt(1, id);');
-            res.line('stmt.setString(2, "Country #" + (id + 1));');
-            res.line('stmt.setInt(3, 10000000 + rnd.nextInt(100000000));');
-        },
-        selectQuery: ['SELECT * FROM COUNTRY WHERE POPULATION BETWEEN 15000000 AND 25000000']
-    },
-    {
-        type: 'DEPARTMENT',
-        create: 'CREATE TABLE IF NOT EXISTS DEPARTMENT (\n' +
-            'ID         INTEGER NOT NULL PRIMARY KEY,\n' +
-            'COUNTRY_ID INTEGER NOT NULL,\n' +
-            'NAME       VARCHAR(50) NOT NULL);',
-        clearQuery: 'DELETE FROM DEPARTMENT',
-        rndRequired: true,
-        insertCntConsts: [
-            {name: 'DEMO_MAX_DEPARTMENT_CNT', val: 5, comment: 'How many departments to generate.'},
-            {name: 'DEMO_MAX_COUNTRY_CNT', val: 5, comment: 'How many countries to generate.'}
-        ],
-        insertPattern: ['INSERT INTO DEPARTMENT(ID, COUNTRY_ID, NAME) VALUES(?, ?, ?)'],
-        fillInsertParameters(res) {
-            res.line('stmt.setInt(1, id);');
-            res.line('stmt.setInt(2, rnd.nextInt(DEMO_MAX_COUNTRY_CNT));');
-            res.line('stmt.setString(3, "Department #" + (id + 1));');
-        },
-        selectQuery: ['SELECT * FROM DEPARTMENT']
-    },
-    {
-        type: 'EMPLOYEE',
-        create: 'CREATE TABLE IF NOT EXISTS EMPLOYEE (\n' +
-            'ID            INTEGER NOT NULL PRIMARY KEY,\n' +
-            'DEPARTMENT_ID INTEGER NOT NULL,\n' +
-            'MANAGER_ID    INTEGER,\n' +
-            'FIRST_NAME    VARCHAR(50) NOT NULL,\n' +
-            'LAST_NAME     VARCHAR(50) NOT NULL,\n' +
-            'EMAIL         VARCHAR(50) NOT NULL,\n' +
-            'PHONE_NUMBER  VARCHAR(50),\n' +
-            'HIRE_DATE     DATE        NOT NULL,\n' +
-            'JOB           VARCHAR(50) NOT NULL,\n' +
-            'SALARY        DOUBLE);',
-        clearQuery: 'DELETE FROM EMPLOYEE',
-        rndRequired: true,
-        insertCntConsts: [
-            {name: 'DEMO_MAX_EMPLOYEE_CNT', val: 10, comment: 'How many employees to generate.'},
-            {name: 'DEMO_MAX_DEPARTMENT_CNT', val: 5, comment: 'How many departments to generate.'}
-        ],
-        specialGeneration(res, conVar) {
-            // $generatorJava.declareVariableCustom(res, 'stmt', 'java.sql.PreparedStatement', conVar +
-            //    '.prepareStatement("INSERT INTO EMPLOYEE(ID, DEPARTMENT_ID, FIRST_NAME, LAST_NAME, EMAIL, PHONE_NUMBER, HIRE_DATE, JOB, SALARY) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)")');
-            //
-            // res.startBlock('for (int id = 0; id < DEMO_MAX_DEPARTMENT_CNT; id ++) {');
-            // res.line('stmt.setInt(1, id);');
-            // res.line('stmt.setInt(2, id);');
-            // res.line('stmt.setString(3, "First name manager #" + (id + 1));');
-            // res.line('stmt.setString(4, "Last name manager#" + (id + 1));');
-            // res.line('stmt.setString(5, "Email manager#" + (id + 1));');
-            // res.line('stmt.setString(6, "Phone number manager#" + (id + 1));');
-            // res.line('stmt.setString(7, "2014-01-01");');
-            // res.line('stmt.setString(8, "Job manager #" + (id + 1));');
-            // res.line('stmt.setDouble(9, 1000.0 + rnd.nextInt(500));');
-            //
-            // res.needEmptyLine = true;
-            //
-            // res.line('stmt.executeUpdate();');
-            //
-            // res.endBlock('}');
-            //
-            // res.needEmptyLine = true;
-
-            $generatorJava.declareVariableCustom(res, 'stmt', 'java.sql.PreparedStatement', conVar +
-                '.prepareStatement("INSERT INTO EMPLOYEE(ID, DEPARTMENT_ID, MANAGER_ID, FIRST_NAME, LAST_NAME, EMAIL, PHONE_NUMBER, HIRE_DATE, JOB, SALARY) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")');
-
-            res.startBlock('for (int id = 0; id < DEMO_MAX_EMPLOYEE_CNT; id ++) {');
-
-            res.needEmptyLine = true;
-
-            res.line('int depId = rnd.nextInt(DEMO_MAX_DEPARTMENT_CNT);');
-
-            res.line('stmt.setInt(1, DEMO_MAX_DEPARTMENT_CNT + id);');
-            res.line('stmt.setInt(2, depId);');
-            res.line('stmt.setInt(3, depId);');
-            res.line('stmt.setString(4, "First name manager #" + (id + 1));');
-            res.line('stmt.setString(5, "Last name manager#" + (id + 1));');
-            res.line('stmt.setString(6, "Email manager#" + (id + 1));');
-            res.line('stmt.setString(7, "Phone number manager#" + (id + 1));');
-            res.line('stmt.setString(8, "2014-01-01");');
-            res.line('stmt.setString(9, "Job manager #" + (id + 1));');
-            res.line('stmt.setDouble(10, 600.0 + rnd.nextInt(300));');
-
-            res.needEmptyLine = true;
-
-            res.line('stmt.executeUpdate();');
-
-            res.endBlock('}');
-
-            res.needEmptyLine = true;
-        },
-        selectQuery: ['SELECT * FROM EMPLOYEE WHERE SALARY > 700']
-    }
-];
-
-// Generate creation and execution of prepared statement.
-function _prepareStatement(res, conVar, query, select) {
-    if (query) {
-        const lines = query.split('\n');
-
-        _.forEach(lines, function(line, ix) {
-            if (ix === 0) {
-                if (lines.length === 1)
-                    res.line(conVar + '.prepareStatement("' + line + '").execute' + (select ? 'Query' : 'Update') + '();');
-                else
-                    res.startBlock(conVar + '.prepareStatement("' + line + '" +');
-            }
-            else
-                res.line('"' + line + '"' + (ix === lines.length - 1 ? ').execute' + (select ? 'Query' : 'Update') + '();' : ' +'));
-        });
-
-        if (lines.length > 0)
-            res.needEmptyLine = true;
-
-        if (lines.length > 1)
-            res.endBlock();
-    }
-}
-
-// Generate creation and execution of cache query.
-function _multilineQuery(res, query, prefix, postfix) {
-    if (query) {
-        const lines = query.split('\n');
-
-        _.forEach(lines, function(line, ix) {
-            if (ix === 0) {
-                if (lines.length === 1)
-                    res.line(prefix + '"' + line + '"' + postfix);
-                else
-                    res.startBlock(prefix + '"' + line + '" +');
-            }
-            else
-                res.line('"' + line + '"' + (ix === lines.length - 1 ? postfix : ' +'));
-        });
-
-        if (lines.length > 1)
-            res.endBlock();
-    }
-}
-
-/**
- * Checks if cluster has demo types.
- *
- * @param cluster Cluster to check.
- * @param demo Is demo enabled.
- * @returns {boolean} True if cluster has caches with demo types.
- */
-$generatorJava.isDemoConfigured = function(cluster, demo) {
-    return demo &&
-        _.find(cluster.caches, (cache) => _.find(cache.domains, (domain) => _.find(PREDEFINED_QUERIES,
-            (desc) => domain.valueType.toUpperCase().endsWith(desc.type))));
-};
-
-$generatorJava.generateDemo = function(cluster, res, factoryCls) {
-    const cachesWithDataSource = _.filter(cluster.caches, (cache) => {
-        if (cache.cacheStoreFactory && cache.cacheStoreFactory.kind) {
-            const storeFactory = cache.cacheStoreFactory[cache.cacheStoreFactory.kind];
-
-            return storeFactory.connectVia === 'DataSource' && storeFactory.dialect ||
-                !storeFactory.connectVia && storeFactory.dialect;
-        }
-
-        return false;
-    });
-
-    // Prepare array of cache and his demo domain model list. Every domain is contained only in first cache.
-    const demoTypes = _.filter(_.map(cachesWithDataSource, (cache, idx) => {
-        return {
-            cache,
-            domains: _.filter(cache.domains, (domain) =>
-                $generatorCommon.isDefinedAndNotEmpty(domain.valueFields) &&
-                    !_.find(cachesWithDataSource, (checkCache, checkIx) => checkIx < idx && _.find(checkCache.domains, domain))
-            )
-        };
-    }), (cache) => $generatorCommon.isDefinedAndNotEmpty(cache.domains));
-
-    if ($generatorCommon.isDefinedAndNotEmpty(demoTypes)) {
-        const typeByDs = {};
-
-        // Group domain modes by data source
-        _.forEach(demoTypes, function(type) {
-            const ds = type.cache.cacheStoreFactory[type.cache.cacheStoreFactory.kind].dataSourceBean;
-
-            if (!typeByDs[ds])
-                typeByDs[ds] = [type];
-            else
-                typeByDs[ds].push(type);
-        });
-
-        let rndDefined = false;
-
-        const generatedConsts = [];
-
-        _.forEach(typeByDs, function(types) {
-            _.forEach(types, function(type) {
-                _.forEach(type.domains, function(domain) {
-                    const desc = _.find(PREDEFINED_QUERIES, (d) => domain.valueType.toUpperCase().endsWith(d.type));
-
-                    if (desc) {
-                        if (!rndDefined && desc.rndRequired) {
-                            res.line('/** Random generator for demo data. */');
-                            $generatorJava.declareVariableCustom(res, 'rnd', 'java.util.Random', 'new Random()', 'private static final');
-
-                            rndDefined = true;
-                        }
-
-                        _.forEach(desc.insertCntConsts, function(cnt) {
-                            if (!_.includes(generatedConsts, cnt.name)) {
-                                res.line('/** ' + cnt.comment + ' */');
-                                res.line('private static final int ' + cnt.name + ' = ' + cnt.val + ';');
-                                res.needEmptyLine = true;
-
-                                generatedConsts.push(cnt.name);
-                            }
-                        });
-                    }
-                });
-            });
-        });
-
-        res.needEmptyLine = true;
-
-        // Generation of fill database method
-        res.line('/** Fill data for Demo. */');
-        res.startBlock('private static void prepareDemoData() throws ' + res.importClass('java.sql.SQLException') + ' {');
-
-        _.forEach(typeByDs, function(types, ds) {
-            const conVar = ds + 'Con';
-
-            res.startBlock('try (' + res.importClass('java.sql.Connection') + ' ' + conVar + ' = ' + res.importClass(factoryCls) + '.DataSources.INSTANCE_' + ds + '.getConnection()) {');
-
-            _.forEach(types, function(type) {
-                _.forEach(type.domains, function(domain) {
-                    const desc = _.find(PREDEFINED_QUERIES, (d) => domain.valueType.toUpperCase().endsWith(d.type));
-
-                    if (desc) {
-                        res.line('// Generate ' + desc.type + '.');
-
-                        if (desc.schema)
-                            _prepareStatement(res, conVar, 'CREATE SCHEMA IF NOT EXISTS ' + desc.schema);
-
-                        _prepareStatement(res, conVar, desc.create);
-
-                        _prepareStatement(res, conVar, desc.clearQuery);
-
-                        res.needEmptyLine = true;
-
-                        if (!desc.specialGeneration) {
-                            $generatorJava.declareVariableCustom(res, 'stmt', 'java.sql.PreparedStatement', conVar + '.prepareStatement("' + desc.insertPattern + '")');
-
-                            res.startBlock('for (int id = 0; id < ' + desc.insertCntConsts[0].name + '; id ++) {');
-
-                            desc.fillInsertParameters(res);
-
-                            res.needEmptyLine = true;
-
-                            res.line('stmt.executeUpdate();');
-
-                            res.endBlock('}');
-
-                            res.needEmptyLine = true;
-                        }
-                        else
-                            desc.specialGeneration(res, conVar);
-
-                        res.line(conVar + '.commit();');
-
-                        res.needEmptyLine = true;
-                    }
-                });
-            });
-
-            res.endBlock('}');
-
-            res.needEmptyLine = true;
-        });
-
-        res.endBlock('}');
-
-        res.needEmptyLine = true;
-
-        res.line('/** Print result table to console */');
-        res.startBlock('private static void printResult(' + res.importClass('java.util.List') + '<' + res.importClass('javax.cache.Cache') + '.Entry<Object, Object>> rows) {');
-        res.startBlock('for (Cache.Entry<Object, Object> row: rows) {');
-        res.line('System.out.println(row);');
-        res.endBlock('}');
-        res.endBlock('}');
-        res.needEmptyLine = true;
-
-        // Generation of execute queries method.
-        res.line('/** Run demo. */');
-        res.startBlock('private static void runDemo(Ignite ignite) throws SQLException {');
-
-        const getType = function(fullType) {
-            return fullType.substr(fullType.lastIndexOf('.') + 1);
-        };
-
-        const cacheLoaded = [];
-        let rowVariableDeclared = false;
-
-        _.forEach(typeByDs, function(types, ds) {
-            const conVar = ds + 'Con';
-
-            res.startBlock('try (Connection ' + conVar + ' = ' + factoryCls + '.DataSources.INSTANCE_' + ds + '.getConnection()) {');
-
-            _.forEach(types, function(type) {
-                _.forEach(type.domains, function(domain) {
-                    const desc = _.find(PREDEFINED_QUERIES, (d) => domain.valueType.toUpperCase().endsWith(d.type));
-
-                    if (desc) {
-                        _.forEach(desc.selectQuery, function(query) {
-                            const cacheName = type.cache.name;
-
-                            if (!_.includes(cacheLoaded, cacheName)) {
-                                res.line('ignite.cache("' + cacheName + '").loadCache(null);');
-
-                                cacheLoaded.push(cacheName);
-                            }
-
-                            _multilineQuery(res, query, (rowVariableDeclared ? 'rows' : 'List<Cache.Entry<Object, Object>> rows') +
-                                ' = ignite.cache("' + cacheName + '").query(new ' + res.importClass('org.apache.ignite.cache.query.SqlQuery') +
-                                '<>("' + getType(domain.valueType) + '", ', ')).getAll();');
-
-                            res.line('printResult(rows);');
-
-                            rowVariableDeclared = true;
-
-                            res.needEmptyLine = true;
-                        });
-                    }
-                });
-            });
-
-            res.endBlock('}');
-
-            res.needEmptyLine = true;
-        });
-
-        res.endBlock('}');
-
-        res.needEmptyLine = true;
-    }
-
-    return demoTypes;
-};
-
-/**
- * Function to generate java class for node startup with cluster configuration.
- *
- * @param cluster Cluster to process.
- * @param pkg Class package name.
- * @param cls Class name.
- * @param cfg Config.
- * @param factoryCls Optional fully qualified class name of configuration factory.
- * @param clientNearCfg Optional near cache configuration for client node.
- */
-$generatorJava.nodeStartup = function(cluster, pkg, cls, cfg, factoryCls, clientNearCfg) {
-    const demo = cls === 'DemoStartup';
-
-    const res = $generatorCommon.builder();
-
-    res.line('/**');
-    res.line(' * ' + $generatorCommon.mainComment('node start up'));
-
-    if (demo) {
-        res.line(' *');
-        res.line(' * To start demo configure data sources in secret.properties file.');
-        res.line(' * For H2 database it should be like following:');
-        res.line(' * dsH2.jdbc.url=jdbc:h2:tcp://localhost/mem:DemoDB;DB_CLOSE_DELAY=-1');
-        res.line(' * dsH2.jdbc.username=sa');
-        res.line(' * dsH2.jdbc.password=');
-    }
-
-    res.line(' */');
-    res.startBlock('public class ' + cls + ' {');
-
-    const demoTypes = demo ? $generatorJava.generateDemo(cluster, res, factoryCls) : null;
-
-    res.line('/**');
-    res.line(' * Start up node with specified configuration.');
-    res.line(' *');
-    res.line(' * @param args Command line arguments, none required.');
-    res.line(' * @throws Exception If failed.');
-    res.line(' */');
-
-    res.startBlock('public static void main(String[] args) throws Exception {');
-
-    if (demo) {
-        res.startBlock('try {');
-        res.line('// Start H2 database server.');
-        res.line(res.importClass('org.h2.tools.Server') + '.createTcpServer("-tcpDaemon").start();');
-        res.endBlock('}');
-        res.startBlock('catch (' + res.importClass('java.sql.SQLException') + ' ignore) {');
-        res.line('// No-op.');
-        res.endBlock('}');
-
-        res.needEmptyLine = true;
-    }
-
-    if (factoryCls)
-        res.importClass(factoryCls);
-
-    if (clientNearCfg || $generatorCommon.isDefinedAndNotEmpty(demoTypes)) {
-        res.line(res.importClass('org.apache.ignite.Ignite') + ' ignite = ' +
-            res.importClass('org.apache.ignite.Ignition') + '.start(' + cfg + ');');
-    }
-    else
-        res.line(res.importClass('org.apache.ignite.Ignition') + '.start(' + cfg + ');');
-
-    if (clientNearCfg) {
-        res.needEmptyLine = true;
-
-        if ($generatorCommon.isDefinedAndNotEmpty(cluster.caches)) {
-            res.line('// Demo of near cache creation on client node.');
-
-            const names = [];
-
-            _.forEach(cluster.caches, function(cache) {
-                $generatorJava.nextVariableName('cache', cache, names);
-
-                res.line('ignite.getOrCreateCache(' + res.importClass(factoryCls) + '.' +
-                    $generatorJava.nextVariableName('cache', cache, names[names.length - 1]) + '(), ' +
-                    res.importClass(factoryCls) + '.createNearCacheConfiguration());');
-            });
-
-            res.needEmptyLine = true;
-        }
-    }
-
-    if (demo) {
-        res.needEmptyLine = true;
-
-        res.line('prepareDemoData();');
-
-        res.needEmptyLine = true;
-
-        res.line('runDemo(ignite);');
-    }
-
-    res.endBlock('}');
-
-    res.endBlock('}');
-
-    return 'package ' + pkg + ';\n\n' + res.generateImports() + '\n' + res.generateStaticImports() + '\n' + res.asString();
-};
-
-/**
- * Function to generate java class for load caches.
- *
- * @param caches Caches to load.
- * @param pkg Class package name.
- * @param cls Class name.
- * @param cfg Config.
- */
-$generatorJava.loadCaches = function(caches, pkg, cls, cfg) {
-    const res = $generatorCommon.builder();
-
-    res.line('/**');
-    res.line(' * ' + $generatorCommon.mainComment('utility'));
-    res.line(' */');
-    res.startBlock('public class ' + cls + ' {');
-
-    res.line('/**');
-    res.line(' * <p>');
-    res.line(' * Utility to load caches from database.');
-    res.line(' * <p>');
-    res.line(' * How to use:');
-    res.line(' * <ul>');
-    res.line(' *    <li>Start cluster.</li>');
-    res.line(' *    <li>Start this utility and wait while load complete.</li>');
-    res.line(' * </ul>');
-    res.line(' *');
-    res.line(' * @param args Command line arguments, none required.');
-    res.line(' * @throws Exception If failed.');
-    res.line(' */');
-
-    res.startBlock('public static void main(String[] args) throws Exception {');
-
-    res.startBlock('try (' + res.importClass('org.apache.ignite.Ignite') + ' ignite = ' +
-        res.importClass('org.apache.ignite.Ignition') + '.start(' + cfg + ')) {');
-
-    res.line('System.out.println(">>> Loading caches...");');
-
-    res.needEmptyLine = true;
-
-    _.forEach(caches, (cache) => {
-        res.line('System.out.println(">>> Loading cache: ' + cache.name + '");');
-        res.line('ignite.cache("' + cache.name + '").loadCache(null);');
-
-        res.needEmptyLine = true;
-    });
-
-    res.needEmptyLine = true;
-
-    res.line('System.out.println(">>> All caches loaded!");');
-
-    res.endBlock('}');
-
-    res.endBlock('}');
-
-    res.endBlock('}');
-
-    return 'package ' + pkg + ';\n\n' + res.generateImports() + '\n' + res.generateStaticImports() + '\n' + res.asString();
-};
-
-
-export default $generatorJava;
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/generator-spring.js b/modules/web-console/frontend/app/modules/configuration/generator/generator-spring.js
deleted file mode 100644
index f70c66f5c91e1..0000000000000
--- a/modules/web-console/frontend/app/modules/configuration/generator/generator-spring.js
+++ /dev/null
@@ -1,2111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// XML generation entry point.
-const $generatorSpring = {};
-
-// Do XML escape.
-$generatorSpring.escape = function(s) {
-    if (typeof (s) !== 'string')
-        return s;
-
-    return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
-};
-
-// Add constructor argument
-$generatorSpring.constructorArg = function(res, ix, obj, propName, dflt, opt) {
-    const v = (obj ? obj[propName] : null) || dflt;
-
-    if ($generatorCommon.isDefinedAndNotEmpty(v))
-        res.line('<constructor-arg ' + (ix >= 0 ? 'index="' + ix + '" ' : '') + 'value="' + v + '"/>');
-    else if (!opt) {
-        res.startBlock('<constructor-arg ' + (ix >= 0 ? 'index="' + ix + '"' : '') + '>');
-        res.line('<null/>');
-        res.endBlock('</constructor-arg>');
-    }
-};
-
-// Add XML element.
-$generatorSpring.element = function(res, tag, attr1, val1, attr2, val2) {
-    let elem = '<' + tag;
-
-    if (attr1)
-        elem += ' ' + attr1 + '="' + val1 + '"';
-
-    if (attr2)
-        elem += ' ' + attr2 + '="' + val2 + '"';
-
-    elem += '/>';
-
-    res.emptyLineIfNeeded();
-    res.line(elem);
-};
-
-// Add property.
-$generatorSpring.property = function(res, obj, propName, setterName, dflt) {
-    if (!_.isNil(obj)) {
-        const val = obj[propName];
-
-        if ($generatorCommon.isDefinedAndNotEmpty(val)) {
-            const missDflt = _.isNil(dflt);
-
-            // Add to result if no default provided or value not equals to default.
-            if (missDflt || (!missDflt && val !== dflt)) {
-                $generatorSpring.element(res, 'property', 'name', setterName ? setterName : propName, 'value', $generatorSpring.escape(val));
-
-                return true;
-            }
-        }
-    }
-
-    return false;
-};
-
-// Add property for class name.
-$generatorSpring.classNameProperty = function(res, obj, propName) {
-    const val = obj[propName];
-
-    if (!_.isNil(val))
-        $generatorSpring.element(res, 'property', 'name', propName, 'value', $generatorCommon.JavaTypes.fullClassName(val));
-};
-
-// Add list property.
-$generatorSpring.listProperty = function(res, obj, propName, listType, rowFactory) {
-    const val = obj[propName];
-
-    if (val && val.length > 0) {
-        res.emptyLineIfNeeded();
-
-        if (!listType)
-            listType = 'list';
-
-        if (!rowFactory)
-            rowFactory = (v) => '<value>' + $generatorSpring.escape(v) + '</value>';
-
-        res.startBlock('<property name="' + propName + '">');
-        res.startBlock('<' + listType + '>');
-
-        _.forEach(val, (v) => res.line(rowFactory(v)));
-
-        res.endBlock('</' + listType + '>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-};
-
-// Add array property
-$generatorSpring.arrayProperty = function(res, obj, propName, descr, rowFactory) {
-    const val = obj[propName];
-
-    if (val && val.length > 0) {
-        res.emptyLineIfNeeded();
-
-        if (!rowFactory)
-            rowFactory = (v) => '<bean class="' + v + '"/>';
-
-        res.startBlock('<property name="' + propName + '">');
-        res.startBlock('<list>');
-
-        _.forEach(val, (v) => res.append(rowFactory(v)));
-
-        res.endBlock('</list>');
-        res.endBlock('</property>');
-    }
-};
-
-/**
- * Add bean property with internal content.
- *
- * @param res Optional configuration presentation builder object.
- * @param bean Bean object for code generation.
- * @param beanPropName Name of property to set generated bean as value.
- * @param desc Bean metadata object.
- * @param createBeanAlthoughNoProps Always generate bean even it has no properties defined.
- */
-$generatorSpring.beanProperty = function(res, bean, beanPropName, desc, createBeanAlthoughNoProps) {
-    const props = desc.fields;
-
-    if (bean && $generatorCommon.hasProperty(bean, props)) {
-        if (!createBeanAlthoughNoProps)
-            res.startSafeBlock();
-
-        res.emptyLineIfNeeded();
-        res.startBlock('<property name="' + beanPropName + '">');
-
-        if (createBeanAlthoughNoProps)
-            res.startSafeBlock();
-
-        res.startBlock('<bean class="' + desc.className + '">');
-
-        let hasData = false;
-
-        _.forIn(props, function(descr, propName) {
-            if (props.hasOwnProperty(propName)) {
-                if (descr) {
-                    switch (descr.type) {
-                        case 'list':
-                            $generatorSpring.listProperty(res, bean, propName, descr.setterName);
-
-                            break;
-
-                        case 'array':
-                            $generatorSpring.arrayProperty(res, bean, propName, descr);
-
-                            break;
-
-                        case 'propertiesAsList':
-                            const val = bean[propName];
-
-                            if (val && val.length > 0) {
-                                res.startBlock('<property name="' + propName + '">');
-                                res.startBlock('<props>');
-
-                                _.forEach(val, function(nameAndValue) {
-                                    const eqIndex = nameAndValue.indexOf('=');
-                                    if (eqIndex >= 0) {
-                                        res.line('<prop key="' + $generatorSpring.escape(nameAndValue.substring(0, eqIndex)) + '">' +
-                                            $generatorSpring.escape(nameAndValue.substr(eqIndex + 1)) + '</prop>');
-                                    }
-                                });
-
-                                res.endBlock('</props>');
-                                res.endBlock('</property>');
-
-                                hasData = true;
-                            }
-
-                            break;
-
-                        case 'bean':
-                            if ($generatorCommon.isDefinedAndNotEmpty(bean[propName])) {
-                                res.startBlock('<property name="' + propName + '">');
-                                res.line('<bean class="' + bean[propName] + '"/>');
-                                res.endBlock('</property>');
-
-                                hasData = true;
-                            }
-
-                            break;
-
-                        default:
-                            if ($generatorSpring.property(res, bean, propName, descr.setterName, descr.dflt))
-                                hasData = true;
-                    }
-                }
-                else
-                    if ($generatorSpring.property(res, bean, propName))
-                        hasData = true;
-            }
-        });
-
-        res.endBlock('</bean>');
-
-        if (createBeanAlthoughNoProps && !hasData) {
-            res.rollbackSafeBlock();
-
-            res.line('<bean class="' + desc.className + '"/>');
-        }
-
-        res.endBlock('</property>');
-
-        if (!createBeanAlthoughNoProps && !hasData)
-            res.rollbackSafeBlock();
-    }
-    else if (createBeanAlthoughNoProps) {
-        res.emptyLineIfNeeded();
-        res.startBlock('<property name="' + beanPropName + '">');
-        res.line('<bean class="' + desc.className + '"/>');
-        res.endBlock('</property>');
-    }
-};
-
-/**
- * Add bean property without internal content.
- *
- * @param res Optional configuration presentation builder object.
- * @param obj Object to take bean class name.
- * @param propName Property name.
- */
-$generatorSpring.simpleBeanProperty = function(res, obj, propName) {
-    if (!_.isNil(obj)) {
-        const val = obj[propName];
-
-        if ($generatorCommon.isDefinedAndNotEmpty(val)) {
-            res.startBlock('<property name="' + propName + '">');
-            res.line('<bean class="' + val + '"/>');
-            res.endBlock('</property>');
-        }
-    }
-
-    return false;
-};
-
-// Generate eviction policy.
-$generatorSpring.evictionPolicy = function(res, evtPlc, propName) {
-    if (evtPlc && evtPlc.kind) {
-        $generatorSpring.beanProperty(res, evtPlc[evtPlc.kind.toUpperCase()], propName,
-            $generatorCommon.EVICTION_POLICIES[evtPlc.kind], true);
-    }
-};
-
-// Generate discovery.
-$generatorSpring.clusterGeneral = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, cluster, 'name', 'gridName');
-    $generatorSpring.property(res, cluster, 'localHost');
-
-    if (cluster.discovery) {
-        res.startBlock('<property name="discoverySpi">');
-        res.startBlock('<bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">');
-        res.startBlock('<property name="ipFinder">');
-
-        const d = cluster.discovery;
-
-        switch (d.kind) {
-            case 'Multicast':
-                res.startBlock('<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">');
-
-                if (d.Multicast) {
-                    $generatorSpring.property(res, d.Multicast, 'multicastGroup');
-                    $generatorSpring.property(res, d.Multicast, 'multicastPort');
-                    $generatorSpring.property(res, d.Multicast, 'responseWaitTime');
-                    $generatorSpring.property(res, d.Multicast, 'addressRequestAttempts');
-                    $generatorSpring.property(res, d.Multicast, 'localAddress');
-                    $generatorSpring.listProperty(res, d.Multicast, 'addresses');
-                }
-
-                res.endBlock('</bean>');
-
-                break;
-
-            case 'Vm':
-                res.startBlock('<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">');
-
-                if (d.Vm)
-                    $generatorSpring.listProperty(res, d.Vm, 'addresses');
-
-                res.endBlock('</bean>');
-
-                break;
-
-            case 'S3':
-                res.startBlock('<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.s3.TcpDiscoveryS3IpFinder">');
-
-                if (d.S3) {
-                    if (d.S3.bucketName)
-                        res.line('<property name="bucketName" value="' + $generatorSpring.escape(d.S3.bucketName) + '"/>');
-                }
-
-                res.endBlock('</bean>');
-
-                break;
-
-            case 'Cloud':
-                res.startBlock('<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.cloud.TcpDiscoveryCloudIpFinder">');
-
-                if (d.Cloud) {
-                    $generatorSpring.property(res, d.Cloud, 'credential');
-                    $generatorSpring.property(res, d.Cloud, 'credentialPath');
-                    $generatorSpring.property(res, d.Cloud, 'identity');
-                    $generatorSpring.property(res, d.Cloud, 'provider');
-                    $generatorSpring.listProperty(res, d.Cloud, 'regions');
-                    $generatorSpring.listProperty(res, d.Cloud, 'zones');
-                }
-
-                res.endBlock('</bean>');
-
-                break;
-
-            case 'GoogleStorage':
-                res.startBlock('<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.gce.TcpDiscoveryGoogleStorageIpFinder">');
-
-                if (d.GoogleStorage) {
-                    $generatorSpring.property(res, d.GoogleStorage, 'projectName');
-                    $generatorSpring.property(res, d.GoogleStorage, 'bucketName');
-                    $generatorSpring.property(res, d.GoogleStorage, 'serviceAccountP12FilePath');
-                    $generatorSpring.property(res, d.GoogleStorage, 'serviceAccountId');
-                }
-
-                res.endBlock('</bean>');
-
-                break;
-
-            case 'Jdbc':
-                res.startBlock('<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.jdbc.TcpDiscoveryJdbcIpFinder">');
-
-                if (d.Jdbc) {
-                    const datasource = d.Jdbc;
-
-                    res.line('<property name="initSchema" value="' + (!_.isNil(datasource.initSchema) && datasource.initSchema) + '"/>');
-
-                    if (datasource.dataSourceBean && datasource.dialect) {
-                        res.line('<property name="dataSource" ref="' + datasource.dataSourceBean + '"/>');
-
-                        if (!_.find(res.datasources, { dataSourceBean: datasource.dataSourceBean })) {
-                            res.datasources.push({
-                                dataSourceBean: datasource.dataSourceBean,
-                                dialect: datasource.dialect
-                            });
-                        }
-                    }
-                }
-
-                res.endBlock('</bean>');
-
-                break;
-
-            case 'SharedFs':
-                res.startBlock('<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.sharedfs.TcpDiscoverySharedFsIpFinder">');
-
-                if (d.SharedFs)
-                    $generatorSpring.property(res, d.SharedFs, 'path');
-
-                res.endBlock('</bean>');
-
-                break;
-
-            case 'ZooKeeper':
-                res.startBlock('<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.zk.TcpDiscoveryZookeeperIpFinder">');
-
-                if (d.ZooKeeper) {
-                    if ($generatorCommon.isDefinedAndNotEmpty(d.ZooKeeper.curator)) {
-                        res.startBlock('<property name="curator">');
-                        res.line('<bean class="' + d.ZooKeeper.curator + '"/>');
-                        res.endBlock('</property>');
-                    }
-
-                    $generatorSpring.property(res, d.ZooKeeper, 'zkConnectionString');
-
-                    if (d.ZooKeeper.retryPolicy && d.ZooKeeper.retryPolicy.kind) {
-                        const kind = d.ZooKeeper.retryPolicy.kind;
-                        const retryPolicy = d.ZooKeeper.retryPolicy[kind];
-                        const customClassDefined = retryPolicy && $generatorCommon.isDefinedAndNotEmpty(retryPolicy.className);
-
-                        if (kind !== 'Custom' || customClassDefined)
-                            res.startBlock('<property name="retryPolicy">');
-
-                        switch (kind) {
-                            case 'ExponentialBackoff':
-                                res.startBlock('<bean class="org.apache.curator.retry.ExponentialBackoffRetry">');
-                                $generatorSpring.constructorArg(res, 0, retryPolicy, 'baseSleepTimeMs', 1000);
-                                $generatorSpring.constructorArg(res, 1, retryPolicy, 'maxRetries', 10);
-                                $generatorSpring.constructorArg(res, 2, retryPolicy, 'maxSleepMs', null, true);
-                                res.endBlock('</bean>');
-
-                                break;
-
-                            case 'BoundedExponentialBackoff':
-                                res.startBlock('<bean class="org.apache.curator.retry.BoundedExponentialBackoffRetry">');
-                                $generatorSpring.constructorArg(res, 0, retryPolicy, 'baseSleepTimeMs', 1000);
-                                $generatorSpring.constructorArg(res, 1, retryPolicy, 'maxSleepTimeMs', 2147483647);
-                                $generatorSpring.constructorArg(res, 2, retryPolicy, 'maxRetries', 10);
-                                res.endBlock('</bean>');
-
-                                break;
-
-                            case 'UntilElapsed':
-                                res.startBlock('<bean class="org.apache.curator.retry.RetryUntilElapsed">');
-                                $generatorSpring.constructorArg(res, 0, retryPolicy, 'maxElapsedTimeMs', 60000);
-                                $generatorSpring.constructorArg(res, 1, retryPolicy, 'sleepMsBetweenRetries', 1000);
-                                res.endBlock('</bean>');
-
-                                break;
-
-                            case 'NTimes':
-                                res.startBlock('<bean class="org.apache.curator.retry.RetryNTimes">');
-                                $generatorSpring.constructorArg(res, 0, retryPolicy, 'n', 10);
-                                $generatorSpring.constructorArg(res, 1, retryPolicy, 'sleepMsBetweenRetries', 1000);
-                                res.endBlock('</bean>');
-
-                                break;
-
-                            case 'OneTime':
-                                res.startBlock('<bean class="org.apache.curator.retry.RetryOneTime">');
-                                $generatorSpring.constructorArg(res, 0, retryPolicy, 'sleepMsBetweenRetry', 1000);
-                                res.endBlock('</bean>');
-
-                                break;
-
-                            case 'Forever':
-                                res.startBlock('<bean class="org.apache.curator.retry.RetryForever">');
-                                $generatorSpring.constructorArg(res, 0, retryPolicy, 'retryIntervalMs', 1000);
-                                res.endBlock('</bean>');
-
-                                break;
-
-                            case 'Custom':
-                                if (customClassDefined)
-                                    res.line('<bean class="' + retryPolicy.className + '"/>');
-
-                                break;
-
-                            default:
-                        }
-
-                        if (kind !== 'Custom' || customClassDefined)
-                            res.endBlock('</property>');
-                    }
-
-                    $generatorSpring.property(res, d.ZooKeeper, 'basePath', null, '/services');
-                    $generatorSpring.property(res, d.ZooKeeper, 'serviceName', null, 'ignite');
-                    $generatorSpring.property(res, d.ZooKeeper, 'allowDuplicateRegistrations', null, false);
-                }
-
-                res.endBlock('</bean>');
-
-                break;
-
-            default:
-                res.line('Unknown discovery kind: ' + d.kind);
-        }
-
-        res.endBlock('</property>');
-
-        $generatorSpring.clusterDiscovery(d, res);
-
-        res.endBlock('</bean>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate atomics group.
-$generatorSpring.clusterAtomics = function(atomics, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.hasAtLeastOneProperty(atomics, ['cacheMode', 'atomicSequenceReserveSize', 'backups'])) {
-        res.startSafeBlock();
-
-        res.emptyLineIfNeeded();
-
-        res.startBlock('<property name="atomicConfiguration">');
-        res.startBlock('<bean class="org.apache.ignite.configuration.AtomicConfiguration">');
-
-        const cacheMode = atomics.cacheMode ? atomics.cacheMode : 'PARTITIONED';
-
-        let hasData = cacheMode !== 'PARTITIONED';
-
-        $generatorSpring.property(res, atomics, 'cacheMode', null, 'PARTITIONED');
-
-        hasData = $generatorSpring.property(res, atomics, 'atomicSequenceReserveSize', null, 1000) || hasData;
-
-        if (cacheMode === 'PARTITIONED')
-            hasData = $generatorSpring.property(res, atomics, 'backups', null, 0) || hasData;
-
-        res.endBlock('</bean>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-
-        if (!hasData)
-            res.rollbackSafeBlock();
-    }
-
-    return res;
-};
-
-// Generate binary group.
-$generatorSpring.clusterBinary = function(binary, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.binaryIsDefined(binary)) {
-        res.startBlock('<property name="binaryConfiguration">');
-        res.startBlock('<bean class="org.apache.ignite.configuration.BinaryConfiguration">');
-
-        $generatorSpring.simpleBeanProperty(res, binary, 'idMapper');
-        $generatorSpring.simpleBeanProperty(res, binary, 'nameMapper');
-        $generatorSpring.simpleBeanProperty(res, binary, 'serializer');
-
-        if ($generatorCommon.isDefinedAndNotEmpty(binary.typeConfigurations)) {
-            res.startBlock('<property name="typeConfigurations">');
-            res.startBlock('<list>');
-
-            _.forEach(binary.typeConfigurations, function(type) {
-                res.startBlock('<bean class="org.apache.ignite.binary.BinaryTypeConfiguration">');
-
-                $generatorSpring.property(res, type, 'typeName');
-                $generatorSpring.simpleBeanProperty(res, type, 'idMapper');
-                $generatorSpring.simpleBeanProperty(res, type, 'nameMapper');
-                $generatorSpring.simpleBeanProperty(res, type, 'serializer');
-                $generatorSpring.property(res, type, 'enum', null, false);
-
-                res.endBlock('</bean>');
-            });
-
-            res.endBlock('</list>');
-            res.endBlock('</property>');
-        }
-
-        $generatorSpring.property(res, binary, 'compactFooter', null, true);
-
-        res.endBlock('</bean>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate cache key configurations.
-$generatorSpring.clusterCacheKeyConfiguration = function(keyCfgs, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    keyCfgs = _.filter(keyCfgs, (cfg) => cfg.typeName && cfg.affinityKeyFieldName);
-
-    if (_.isEmpty(keyCfgs))
-        return res;
-
-    res.startBlock('<property name="cacheKeyConfiguration">');
-    res.startBlock('<array>');
-
-    _.forEach(keyCfgs, (cfg) => {
-        res.startBlock('<bean class="org.apache.ignite.cache.CacheKeyConfiguration">');
-
-        $generatorSpring.constructorArg(res, -1, cfg, 'typeName');
-        $generatorSpring.constructorArg(res, -1, cfg, 'affinityKeyFieldName');
-
-        res.endBlock('</bean>');
-    });
-
-    res.endBlock('</array>');
-    res.endBlock('</property>');
-
-    return res;
-};
-
-// Generate collision group.
-$generatorSpring.clusterCollision = function(collision, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (collision && collision.kind && collision.kind !== 'Noop') {
-        const spi = collision[collision.kind];
-
-        if (collision.kind !== 'Custom' || (spi && $generatorCommon.isDefinedAndNotEmpty(spi.class))) {
-            res.startBlock('<property name="collisionSpi">');
-
-            switch (collision.kind) {
-                case 'JobStealing':
-                    res.startBlock('<bean class="org.apache.ignite.spi.collision.jobstealing.JobStealingCollisionSpi">');
-                    $generatorSpring.property(res, spi, 'activeJobsThreshold', null, 95);
-                    $generatorSpring.property(res, spi, 'waitJobsThreshold', null, 0);
-                    $generatorSpring.property(res, spi, 'messageExpireTime', null, 1000);
-                    $generatorSpring.property(res, spi, 'maximumStealingAttempts', null, 5);
-                    $generatorSpring.property(res, spi, 'stealingEnabled', null, true);
-
-                    if ($generatorCommon.isDefinedAndNotEmpty(spi.externalCollisionListener)) {
-                        res.needEmptyLine = true;
-
-                        res.startBlock('<property name="externalCollisionListener">');
-                        res.line('<bean class="' + spi.externalCollisionListener + ' "/>');
-                        res.endBlock('</property>');
-                    }
-
-                    if ($generatorCommon.isDefinedAndNotEmpty(spi.stealingAttributes)) {
-                        res.needEmptyLine = true;
-
-                        res.startBlock('<property name="stealingAttributes">');
-                        res.startBlock('<map>');
-
-                        _.forEach(spi.stealingAttributes, function(attr) {
-                            $generatorSpring.element(res, 'entry', 'key', attr.name, 'value', attr.value);
-                        });
-
-                        res.endBlock('</map>');
-                        res.endBlock('</property>');
-                    }
-
-                    res.endBlock('</bean>');
-
-                    break;
-
-                case 'FifoQueue':
-                    res.startBlock('<bean class="org.apache.ignite.spi.collision.fifoqueue.FifoQueueCollisionSpi">');
-                    $generatorSpring.property(res, spi, 'parallelJobsNumber');
-                    $generatorSpring.property(res, spi, 'waitingJobsNumber');
-                    res.endBlock('</bean>');
-
-                    break;
-
-                case 'PriorityQueue':
-                    res.startBlock('<bean class="org.apache.ignite.spi.collision.priorityqueue.PriorityQueueCollisionSpi">');
-                    $generatorSpring.property(res, spi, 'parallelJobsNumber');
-                    $generatorSpring.property(res, spi, 'waitingJobsNumber');
-                    $generatorSpring.property(res, spi, 'priorityAttributeKey', null, 'grid.task.priority');
-                    $generatorSpring.property(res, spi, 'jobPriorityAttributeKey', null, 'grid.job.priority');
-                    $generatorSpring.property(res, spi, 'defaultPriority', null, 0);
-                    $generatorSpring.property(res, spi, 'starvationIncrement', null, 1);
-                    $generatorSpring.property(res, spi, 'starvationPreventionEnabled', null, true);
-                    res.endBlock('</bean>');
-
-                    break;
-
-                case 'Custom':
-                    res.line('<bean class="' + spi.class + '"/>');
-
-                    break;
-
-                default:
-            }
-
-            res.endBlock('</property>');
-        }
-    }
-
-    return res;
-};
-
-// Generate communication group.
-$generatorSpring.clusterCommunication = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.beanProperty(res, cluster.communication, 'communicationSpi', $generatorCommon.COMMUNICATION_CONFIGURATION);
-
-    $generatorSpring.property(res, cluster, 'networkTimeout', null, 5000);
-    $generatorSpring.property(res, cluster, 'networkSendRetryDelay', null, 1000);
-    $generatorSpring.property(res, cluster, 'networkSendRetryCount', null, 3);
-    $generatorSpring.property(res, cluster, 'segmentCheckFrequency');
-    $generatorSpring.property(res, cluster, 'waitForSegmentOnStart', null, false);
-    $generatorSpring.property(res, cluster, 'discoveryStartupDelay', null, 60000);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-/**
- * XML generator for cluster's REST access configuration.
- *
- * @param connector Cluster REST connector configuration.
- * @param res Optional configuration presentation builder object.
- * @returns Configuration presentation builder object
- */
-$generatorSpring.clusterConnector = function(connector, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (!_.isNil(connector) && connector.enabled) {
-        const cfg = _.cloneDeep($generatorCommon.CONNECTOR_CONFIGURATION);
-
-        if (connector.sslEnabled) {
-            cfg.fields.sslClientAuth = {dflt: false};
-            cfg.fields.sslFactory = {type: 'bean'};
-        }
-
-        $generatorSpring.beanProperty(res, connector, 'connectorConfiguration', cfg, true);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate deployment group.
-$generatorSpring.clusterDeployment = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorSpring.property(res, cluster, 'deploymentMode', null, 'SHARED'))
-        res.needEmptyLine = true;
-
-    const p2pEnabled = cluster.peerClassLoadingEnabled;
-
-    if (!_.isNil(p2pEnabled)) {
-        $generatorSpring.property(res, cluster, 'peerClassLoadingEnabled', null, false);
-
-        if (p2pEnabled) {
-            $generatorSpring.property(res, cluster, 'peerClassLoadingMissedResourcesCacheSize', null, 100);
-            $generatorSpring.property(res, cluster, 'peerClassLoadingThreadPoolSize', null, 2);
-            $generatorSpring.listProperty(res, cluster, 'peerClassLoadingLocalClassPathExclude');
-        }
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate discovery group.
-$generatorSpring.clusterDiscovery = function(disco, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (disco) {
-        $generatorSpring.property(res, disco, 'localAddress');
-        $generatorSpring.property(res, disco, 'localPort', null, 47500);
-        $generatorSpring.property(res, disco, 'localPortRange', null, 100);
-        if ($generatorCommon.isDefinedAndNotEmpty(disco.addressResolver))
-            $generatorSpring.beanProperty(res, disco, 'addressResolver', {className: disco.addressResolver}, true);
-        $generatorSpring.property(res, disco, 'socketTimeout', null, 5000);
-        $generatorSpring.property(res, disco, 'ackTimeout', null, 5000);
-        $generatorSpring.property(res, disco, 'maxAckTimeout', null, 600000);
-        $generatorSpring.property(res, disco, 'networkTimeout', null, 5000);
-        $generatorSpring.property(res, disco, 'joinTimeout', null, 0);
-        $generatorSpring.property(res, disco, 'threadPriority', null, 10);
-        $generatorSpring.property(res, disco, 'heartbeatFrequency', null, 2000);
-        $generatorSpring.property(res, disco, 'maxMissedHeartbeats', null, 1);
-        $generatorSpring.property(res, disco, 'maxMissedClientHeartbeats', null, 5);
-        $generatorSpring.property(res, disco, 'topHistorySize', null, 1000);
-        if ($generatorCommon.isDefinedAndNotEmpty(disco.listener))
-            $generatorSpring.beanProperty(res, disco, 'listener', {className: disco.listener}, true);
-        if ($generatorCommon.isDefinedAndNotEmpty(disco.dataExchange))
-            $generatorSpring.beanProperty(res, disco, 'dataExchange', {className: disco.dataExchange}, true);
-        if ($generatorCommon.isDefinedAndNotEmpty(disco.metricsProvider))
-            $generatorSpring.beanProperty(res, disco, 'metricsProvider', {className: disco.metricsProvider}, true);
-        $generatorSpring.property(res, disco, 'reconnectCount', null, 10);
-        $generatorSpring.property(res, disco, 'statisticsPrintFrequency', null, 0);
-        $generatorSpring.property(res, disco, 'ipFinderCleanFrequency', null, 60000);
-        if ($generatorCommon.isDefinedAndNotEmpty(disco.authenticator))
-            $generatorSpring.beanProperty(res, disco, 'authenticator', {className: disco.authenticator}, true);
-        $generatorSpring.property(res, disco, 'forceServerMode', null, false);
-        $generatorSpring.property(res, disco, 'clientReconnectDisabled', null, false);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate events group.
-$generatorSpring.clusterEvents = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (cluster.includeEventTypes && cluster.includeEventTypes.length > 0) {
-        res.emptyLineIfNeeded();
-
-        res.startBlock('<property name="includeEventTypes">');
-
-        const evtGrps = angular.element(document.getElementById('app')).injector().get('igniteEventGroups');
-
-        if (cluster.includeEventTypes.length === 1) {
-            const evtGrp = _.find(evtGrps, {value: cluster.includeEventTypes[0]});
-
-            if (evtGrp)
-                res.line('<util:constant static-field="' + evtGrp.class + '.' + evtGrp.value + '"/>');
-        }
-        else {
-            res.startBlock('<list>');
-
-            _.forEach(cluster.includeEventTypes, (item, ix) => {
-                ix > 0 && res.line();
-
-                const evtGrp = _.find(evtGrps, {value: item});
-
-                if (evtGrp) {
-                    res.line('<!-- EventType.' + item + ' -->');
-
-                    _.forEach(evtGrp.events, (event) => res.line('<util:constant static-field="' + evtGrp.class + '.' + event + '"/>'));
-                }
-            });
-
-            res.endBlock('</list>');
-        }
-
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate failover group.
-$generatorSpring.clusterFailover = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.isDefinedAndNotEmpty(cluster.failoverSpi) && _.findIndex(cluster.failoverSpi, function(spi) {
-        return $generatorCommon.isDefinedAndNotEmpty(spi.kind) && (spi.kind !== 'Custom' || $generatorCommon.isDefinedAndNotEmpty(_.get(spi, spi.kind + '.class')));
-    }) >= 0) {
-        res.startBlock('<property name="failoverSpi">');
-        res.startBlock('<list>');
-
-        _.forEach(cluster.failoverSpi, function(spi) {
-            if (spi.kind && (spi.kind !== 'Custom' || $generatorCommon.isDefinedAndNotEmpty(_.get(spi, spi.kind + '.class')))) {
-                const maxAttempts = _.get(spi, spi.kind + '.maximumFailoverAttempts');
-
-                if ((spi.kind === 'JobStealing' || spi.kind === 'Always') && $generatorCommon.isDefinedAndNotEmpty(maxAttempts) && maxAttempts !== 5) {
-                    res.startBlock('<bean class="' + $generatorCommon.failoverSpiClass(spi) + '">');
-
-                    $generatorSpring.property(res, spi[spi.kind], 'maximumFailoverAttempts', null, 5);
-
-                    res.endBlock('</bean>');
-                }
-                else
-                    res.line('<bean class="' + $generatorCommon.failoverSpiClass(spi) + '"/>');
-
-                res.needEmptyLine = true;
-            }
-        });
-
-        res.needEmptyLine = true;
-
-        res.endBlock('</list>');
-        res.endBlock('</property>');
-    }
-
-    return res;
-};
-
-// Generate marshaller group.
-$generatorSpring.clusterLogger = function(logger, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.loggerConfigured(logger)) {
-        res.startBlock('<property name="gridLogger">');
-
-        const log = logger[logger.kind];
-
-        switch (logger.kind) {
-            case 'Log4j2':
-                res.startBlock('<bean class="org.apache.ignite.logger.log4j2.Log4J2Logger">');
-                res.line('<constructor-arg value="' + $generatorSpring.escape(log.path) + '"/>');
-                $generatorSpring.property(res, log, 'level');
-                res.endBlock('</bean>');
-
-                break;
-
-            case 'Null':
-                res.line('<bean class="org.apache.ignite.logger.NullLogger"/>');
-
-                break;
-
-            case 'Java':
-                res.line('<bean class="org.apache.ignite.logger.java.JavaLogger"/>');
-
-                break;
-
-            case 'JCL':
-                res.line('<bean class="org.apache.ignite.logger.jcl.JclLogger"/>');
-
-                break;
-
-            case 'SLF4J':
-                res.line('<bean class="org.apache.ignite.logger.slf4j.Slf4jLogger"/>');
-
-                break;
-
-            case 'Log4j':
-                if (log.mode === 'Default' && !$generatorCommon.isDefinedAndNotEmpty(log.level))
-                    res.line('<bean class="org.apache.ignite.logger.log4j.Log4JLogger"/>');
-                else {
-                    res.startBlock('<bean class="org.apache.ignite.logger.log4j.Log4JLogger">');
-
-                    if (log.mode === 'Path')
-                        res.line('<constructor-arg value="' + $generatorSpring.escape(log.path) + '"/>');
-
-                    $generatorSpring.property(res, log, 'level');
-                    res.endBlock('</bean>');
-                }
-
-                break;
-
-            case 'Custom':
-                res.line('<bean class="' + log.class + '"/>');
-
-                break;
-
-            default:
-        }
-
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate marshaller group.
-$generatorSpring.clusterMarshaller = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    const marshaller = cluster.marshaller;
-
-    if (marshaller && marshaller.kind)
-        $generatorSpring.beanProperty(res, marshaller[marshaller.kind], 'marshaller', $generatorCommon.MARSHALLERS[marshaller.kind], true);
-
-    res.softEmptyLine();
-
-    $generatorSpring.property(res, cluster, 'marshalLocalJobs', null, false);
-    $generatorSpring.property(res, cluster, 'marshallerCacheKeepAliveTime', null, 10000);
-    $generatorSpring.property(res, cluster, 'marshallerCacheThreadPoolSize', 'marshallerCachePoolSize');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate metrics group.
-$generatorSpring.clusterMetrics = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, cluster, 'metricsExpireTime');
-    $generatorSpring.property(res, cluster, 'metricsHistorySize', null, 10000);
-    $generatorSpring.property(res, cluster, 'metricsLogFrequency', null, 60000);
-    $generatorSpring.property(res, cluster, 'metricsUpdateFrequency', null, 2000);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate swap group.
-$generatorSpring.clusterSwap = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (cluster.swapSpaceSpi && cluster.swapSpaceSpi.kind === 'FileSwapSpaceSpi') {
-        $generatorSpring.beanProperty(res, cluster.swapSpaceSpi.FileSwapSpaceSpi, 'swapSpaceSpi',
-            $generatorCommon.SWAP_SPACE_SPI, true);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate time group.
-$generatorSpring.clusterTime = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, cluster, 'clockSyncSamples', null, 8);
-    $generatorSpring.property(res, cluster, 'clockSyncFrequency', null, 120000);
-    $generatorSpring.property(res, cluster, 'timeServerPortBase', null, 31100);
-    $generatorSpring.property(res, cluster, 'timeServerPortRange', null, 100);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate OBC configuration group.
-$generatorSpring.clusterODBC = function(odbc, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (odbc && odbc.odbcEnabled)
-        $generatorSpring.beanProperty(res, odbc, 'odbcConfiguration', $generatorCommon.ODBC_CONFIGURATION, true);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate thread pools group.
-$generatorSpring.clusterPools = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, cluster, 'publicThreadPoolSize');
-    $generatorSpring.property(res, cluster, 'systemThreadPoolSize');
-    $generatorSpring.property(res, cluster, 'managementThreadPoolSize');
-    $generatorSpring.property(res, cluster, 'igfsThreadPoolSize');
-    $generatorSpring.property(res, cluster, 'rebalanceThreadPoolSize');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate transactions group.
-$generatorSpring.clusterTransactions = function(transactionConfiguration, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.beanProperty(res, transactionConfiguration, 'transactionConfiguration', $generatorCommon.TRANSACTION_CONFIGURATION, false);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate user attributes group.
-$generatorSpring.clusterUserAttributes = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.isDefinedAndNotEmpty(cluster.attributes)) {
-        res.startBlock('<property name="userAttributes">');
-        res.startBlock('<map>');
-
-        _.forEach(cluster.attributes, function(attr) {
-            $generatorSpring.element(res, 'entry', 'key', attr.name, 'value', attr.value);
-        });
-
-        res.endBlock('</map>');
-        res.endBlock('</property>');
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-/**
- * XML generator for cluster's SSL configuration.
- *
- * @param cluster Cluster to get SSL configuration.
- * @param res Optional configuration presentation builder object.
- * @returns Configuration presentation builder object
- */
-$generatorSpring.clusterSsl = function(cluster, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (cluster.sslEnabled && !_.isNil(cluster.sslContextFactory)) {
-        let sslFactory;
-
-        if (_.isEmpty(cluster.sslContextFactory.keyStoreFilePath) && _.isEmpty(cluster.sslContextFactory.trustStoreFilePath))
-            sslFactory = cluster.sslContextFactory;
-        else {
-            sslFactory = _.clone(cluster.sslContextFactory);
-
-            sslFactory.keyStorePassword = _.isEmpty(cluster.sslContextFactory.keyStoreFilePath) ? null : '${ssl.key.storage.password}';
-            sslFactory.trustStorePassword = _.isEmpty(cluster.sslContextFactory.trustStoreFilePath) ? null : '${ssl.trust.storage.password}';
-        }
-
-        const propsDesc = $generatorCommon.isDefinedAndNotEmpty(cluster.sslContextFactory.trustManagers) ?
-            $generatorCommon.SSL_CONFIGURATION_TRUST_MANAGER_FACTORY :
-            $generatorCommon.SSL_CONFIGURATION_TRUST_FILE_FACTORY;
-
-        $generatorSpring.beanProperty(res, sslFactory, 'sslContextFactory', propsDesc, true);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate cache general group.
-$generatorSpring.cacheGeneral = function(cache, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, cache, 'name');
-
-    $generatorSpring.property(res, cache, 'cacheMode');
-    $generatorSpring.property(res, cache, 'atomicityMode');
-
-    if (cache.cacheMode === 'PARTITIONED' && $generatorSpring.property(res, cache, 'backups'))
-        $generatorSpring.property(res, cache, 'readFromBackup');
-
-    $generatorSpring.property(res, cache, 'copyOnRead');
-
-    if (cache.cacheMode === 'PARTITIONED' && cache.atomicityMode === 'TRANSACTIONAL')
-        $generatorSpring.property(res, cache, 'invalidate');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache memory group.
-$generatorSpring.cacheMemory = function(cache, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, cache, 'memoryMode', null, 'ONHEAP_TIERED');
-
-    if (cache.memoryMode !== 'OFFHEAP_VALUES')
-        $generatorSpring.property(res, cache, 'offHeapMaxMemory', null, -1);
-
-    res.softEmptyLine();
-
-    $generatorSpring.evictionPolicy(res, cache.evictionPolicy, 'evictionPolicy');
-
-    res.softEmptyLine();
-
-    $generatorSpring.property(res, cache, 'startSize', null, 1500000);
-    $generatorSpring.property(res, cache, 'swapEnabled', null, false);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache query & indexing group.
-$generatorSpring.cacheQuery = function(cache, domains, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, cache, 'sqlSchema');
-    $generatorSpring.property(res, cache, 'sqlOnheapRowCacheSize', null, 10240);
-    $generatorSpring.property(res, cache, 'longQueryWarningTimeout', null, 3000);
-
-    const indexedTypes = _.filter(domains, (domain) => domain.queryMetadata === 'Annotations');
-
-    if (indexedTypes.length > 0) {
-        res.softEmptyLine();
-
-        res.startBlock('<property name="indexedTypes">');
-        res.startBlock('<list>');
-
-        _.forEach(indexedTypes, function(domain) {
-            res.line('<value>' + $generatorCommon.JavaTypes.fullClassName(domain.keyType) + '</value>');
-            res.line('<value>' + $generatorCommon.JavaTypes.fullClassName(domain.valueType) + '</value>');
-        });
-
-        res.endBlock('</list>');
-        res.endBlock('</property>');
-    }
-
-    res.softEmptyLine();
-
-    $generatorSpring.listProperty(res, cache, 'sqlFunctionClasses');
-
-    res.softEmptyLine();
-
-    $generatorSpring.property(res, cache, 'snapshotableIndex', null, false);
-    $generatorSpring.property(res, cache, 'sqlEscapeAll', null, false);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache store group.
-$generatorSpring.cacheStore = function(cache, domains, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (cache.cacheStoreFactory && cache.cacheStoreFactory.kind) {
-        const factoryKind = cache.cacheStoreFactory.kind;
-
-        const storeFactory = cache.cacheStoreFactory[factoryKind];
-
-        if (storeFactory) {
-            if (factoryKind === 'CacheJdbcPojoStoreFactory') {
-                res.startBlock('<property name="cacheStoreFactory">');
-                res.startBlock('<bean class="org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory">');
-
-                $generatorSpring.property(res, storeFactory, 'dataSourceBean');
-
-                res.startBlock('<property name="dialect">');
-                res.line('<bean class="' + $generatorCommon.jdbcDialectClassName(storeFactory.dialect) + '"/>');
-                res.endBlock('</property>');
-
-                if (storeFactory.sqlEscapeAll)
-                    $generatorSpring.property(res, storeFactory, 'sqlEscapeAll');
-
-                const domainConfigs = _.filter(domains, function(domain) {
-                    return $generatorCommon.isDefinedAndNotEmpty(domain.databaseTable);
-                });
-
-                if ($generatorCommon.isDefinedAndNotEmpty(domainConfigs)) {
-                    res.startBlock('<property name="types">');
-                    res.startBlock('<list>');
-
-                    _.forEach(domainConfigs, function(domain) {
-                        res.startBlock('<bean class="org.apache.ignite.cache.store.jdbc.JdbcType">');
-
-                        $generatorSpring.property(res, cache, 'name', 'cacheName');
-
-                        $generatorSpring.classNameProperty(res, domain, 'keyType');
-                        $generatorSpring.property(res, domain, 'valueType');
-
-                        $generatorSpring.domainStore(domain, res);
-
-                        res.endBlock('</bean>');
-                    });
-
-                    res.endBlock('</list>');
-                    res.endBlock('</property>');
-                }
-
-                res.endBlock('</bean>');
-                res.endBlock('</property>');
-            }
-            else if (factoryKind === 'CacheJdbcBlobStoreFactory') {
-                res.startBlock('<property name="cacheStoreFactory">');
-                res.startBlock('<bean class="org.apache.ignite.cache.store.jdbc.CacheJdbcBlobStoreFactory">');
-
-                if (storeFactory.connectVia === 'DataSource')
-                    $generatorSpring.property(res, storeFactory, 'dataSourceBean');
-                else {
-                    $generatorSpring.property(res, storeFactory, 'connectionUrl');
-                    $generatorSpring.property(res, storeFactory, 'user');
-                    res.line('<property name="password" value="${ds.' + storeFactory.user + '.password}"/>');
-                }
-
-                $generatorSpring.property(res, storeFactory, 'initSchema');
-                $generatorSpring.property(res, storeFactory, 'createTableQuery');
-                $generatorSpring.property(res, storeFactory, 'loadQuery');
-                $generatorSpring.property(res, storeFactory, 'insertQuery');
-                $generatorSpring.property(res, storeFactory, 'updateQuery');
-                $generatorSpring.property(res, storeFactory, 'deleteQuery');
-
-                res.endBlock('</bean>');
-                res.endBlock('</property>');
-            }
-            else
-                $generatorSpring.beanProperty(res, storeFactory, 'cacheStoreFactory', $generatorCommon.STORE_FACTORIES[factoryKind], true);
-
-            if (storeFactory.dataSourceBean && (storeFactory.connectVia ? (storeFactory.connectVia === 'DataSource' ? storeFactory.dialect : null) : storeFactory.dialect)) {
-                if (!_.find(res.datasources, { dataSourceBean: storeFactory.dataSourceBean})) {
-                    res.datasources.push({
-                        dataSourceBean: storeFactory.dataSourceBean,
-                        dialect: storeFactory.dialect
-                    });
-                }
-            }
-        }
-    }
-
-    res.softEmptyLine();
-
-    $generatorSpring.property(res, cache, 'storeKeepBinary', null, false);
-    $generatorSpring.property(res, cache, 'loadPreviousValue', null, false);
-    $generatorSpring.property(res, cache, 'readThrough', null, false);
-    $generatorSpring.property(res, cache, 'writeThrough', null, false);
-
-    res.softEmptyLine();
-
-    if (cache.writeBehindEnabled) {
-        $generatorSpring.property(res, cache, 'writeBehindEnabled', null, false);
-        $generatorSpring.property(res, cache, 'writeBehindBatchSize', null, 512);
-        $generatorSpring.property(res, cache, 'writeBehindFlushSize', null, 10240);
-        $generatorSpring.property(res, cache, 'writeBehindFlushFrequency', null, 5000);
-        $generatorSpring.property(res, cache, 'writeBehindFlushThreadCount', null, 1);
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache node filter group.
-$generatorSpring.cacheNodeFilter = function(cache, igfss, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    const kind = _.get(cache, 'nodeFilter.kind');
-
-    if (_.isNil(kind) || _.isNil(cache.nodeFilter[kind]))
-        return res;
-
-    switch (kind) {
-        case 'IGFS':
-            const foundIgfs = _.find(igfss, (igfs) => igfs._id === cache.nodeFilter.IGFS.igfs);
-
-            if (foundIgfs) {
-                res.startBlock('<property name="nodeFilter">');
-                res.startBlock('<bean class="org.apache.ignite.internal.processors.igfs.IgfsNodePredicate">');
-                res.line('<constructor-arg value="' + foundIgfs.name + '"/>');
-                res.endBlock('</bean>');
-                res.endBlock('</property>');
-            }
-
-            break;
-
-        case 'OnNodes':
-            const nodes = cache.nodeFilter.OnNodes.nodeIds;
-
-            if ($generatorCommon.isDefinedAndNotEmpty(nodes)) {
-                res.startBlock('<property name="nodeFilter">');
-                res.startBlock('<bean class="org.apache.ignite.internal.util.lang.GridNodePredicate">');
-                res.startBlock('<constructor-arg>');
-                res.startBlock('<list>');
-
-                _.forEach(nodes, (nodeId) => {
-                    res.startBlock('<bean class="java.util.UUID" factory-method="fromString">');
-                    res.line('<constructor-arg value="' + nodeId + '"/>');
-                    res.endBlock('</bean>');
-                });
-
-                res.endBlock('</list>');
-                res.endBlock('</constructor-arg>');
-                res.endBlock('</bean>');
-                res.endBlock('</property>');
-            }
-
-            break;
-
-        case 'Custom':
-            res.startBlock('<property name="nodeFilter">');
-            res.line('<bean class="' + cache.nodeFilter.Custom.className + '"/>');
-            res.endBlock('</property>');
-
-            break;
-
-        default: break;
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache concurrency group.
-$generatorSpring.cacheConcurrency = function(cache, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, cache, 'maxConcurrentAsyncOperations', null, 500);
-    $generatorSpring.property(res, cache, 'defaultLockTimeout', null, 0);
-    $generatorSpring.property(res, cache, 'atomicWriteOrderMode');
-    $generatorSpring.property(res, cache, 'writeSynchronizationMode', null, 'PRIMARY_SYNC');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache rebalance group.
-$generatorSpring.cacheRebalance = function(cache, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (cache.cacheMode !== 'LOCAL') {
-        $generatorSpring.property(res, cache, 'rebalanceMode', null, 'ASYNC');
-        $generatorSpring.property(res, cache, 'rebalanceThreadPoolSize', null, 1);
-        $generatorSpring.property(res, cache, 'rebalanceBatchSize', null, 524288);
-        $generatorSpring.property(res, cache, 'rebalanceBatchesPrefetchCount', null, 2);
-        $generatorSpring.property(res, cache, 'rebalanceOrder', null, 0);
-        $generatorSpring.property(res, cache, 'rebalanceDelay', null, 0);
-        $generatorSpring.property(res, cache, 'rebalanceTimeout', null, 10000);
-        $generatorSpring.property(res, cache, 'rebalanceThrottle', null, 0);
-    }
-
-    res.softEmptyLine();
-
-    if (cache.igfsAffinnityGroupSize) {
-        res.startBlock('<property name="affinityMapper">');
-        res.startBlock('<bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper">');
-        $generatorSpring.constructorArg(res, -1, cache, 'igfsAffinnityGroupSize');
-        res.endBlock('</bean>');
-        res.endBlock('</property>');
-    }
-
-    return res;
-};
-
-// Generate cache server near cache group.
-$generatorSpring.cacheServerNearCache = function(cache, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (cache.cacheMode === 'PARTITIONED' && cache.nearCacheEnabled) {
-        res.emptyLineIfNeeded();
-
-        res.startBlock('<property name="nearConfiguration">');
-        res.startBlock('<bean class="org.apache.ignite.configuration.NearCacheConfiguration">');
-
-        if (cache.nearConfiguration) {
-            if (cache.nearConfiguration.nearStartSize)
-                $generatorSpring.property(res, cache.nearConfiguration, 'nearStartSize', null, 375000);
-
-            $generatorSpring.evictionPolicy(res, cache.nearConfiguration.nearEvictionPolicy, 'nearEvictionPolicy');
-        }
-
-        res.endBlock('</bean>');
-        res.endBlock('</property>');
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate cache statistics group.
-$generatorSpring.cacheStatistics = function(cache, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, cache, 'statisticsEnabled', null, false);
-    $generatorSpring.property(res, cache, 'managementEnabled', null, false);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate domain model query fields.
-$generatorSpring.domainModelQueryFields = function(res, domain) {
-    const fields = domain.fields;
-
-    if (fields && fields.length > 0) {
-        res.emptyLineIfNeeded();
-
-        res.startBlock('<property name="fields">');
-        res.startBlock('<map>');
-
-        _.forEach(fields, function(field) {
-            $generatorSpring.element(res, 'entry', 'key', field.name, 'value', $generatorCommon.JavaTypes.fullClassName(field.className));
-        });
-
-        res.endBlock('</map>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-};
-
-// Generate domain model query fields.
-$generatorSpring.domainModelQueryAliases = function(res, domain) {
-    const aliases = domain.aliases;
-
-    if (aliases && aliases.length > 0) {
-        res.emptyLineIfNeeded();
-
-        res.startBlock('<property name="aliases">');
-        res.startBlock('<map>');
-
-        _.forEach(aliases, function(alias) {
-            $generatorSpring.element(res, 'entry', 'key', alias.field, 'value', alias.alias);
-        });
-
-        res.endBlock('</map>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-};
-
-// Generate domain model indexes.
-$generatorSpring.domainModelQueryIndexes = function(res, domain) {
-    const indexes = domain.indexes;
-
-    if (indexes && indexes.length > 0) {
-        res.emptyLineIfNeeded();
-
-        res.startBlock('<property name="indexes">');
-        res.startBlock('<list>');
-
-        _.forEach(indexes, function(index) {
-            res.startBlock('<bean class="org.apache.ignite.cache.QueryIndex">');
-
-            $generatorSpring.property(res, index, 'name');
-            $generatorSpring.property(res, index, 'indexType');
-
-            const fields = index.fields;
-
-            if (fields && fields.length > 0) {
-                res.startBlock('<property name="fields">');
-                res.startBlock('<map>');
-
-                _.forEach(fields, function(field) {
-                    $generatorSpring.element(res, 'entry', 'key', field.name, 'value', field.direction);
-                });
-
-                res.endBlock('</map>');
-                res.endBlock('</property>');
-            }
-
-            res.endBlock('</bean>');
-        });
-
-        res.endBlock('</list>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-};
-
-// Generate domain model db fields.
-$generatorSpring.domainModelDatabaseFields = function(res, domain, fieldProp) {
-    const fields = domain[fieldProp];
-
-    if (fields && fields.length > 0) {
-        res.emptyLineIfNeeded();
-
-        res.startBlock('<property name="' + fieldProp + '">');
-
-        res.startBlock('<list>');
-
-        _.forEach(fields, function(field) {
-            res.startBlock('<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">');
-
-            $generatorSpring.property(res, field, 'databaseFieldName');
-
-            res.startBlock('<property name="databaseFieldType">');
-            res.line('<util:constant static-field="java.sql.Types.' + field.databaseFieldType + '"/>');
-            res.endBlock('</property>');
-
-            $generatorSpring.property(res, field, 'javaFieldName');
-
-            $generatorSpring.classNameProperty(res, field, 'javaFieldType');
-
-            res.endBlock('</bean>');
-        });
-
-        res.endBlock('</list>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-};
-
-// Generate domain model general group.
-$generatorSpring.domainModelGeneral = function(domain, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    switch ($generatorCommon.domainQueryMetadata(domain)) {
-        case 'Annotations':
-            if ($generatorCommon.isDefinedAndNotEmpty(domain.keyType) || $generatorCommon.isDefinedAndNotEmpty(domain.valueType)) {
-                res.startBlock('<property name="indexedTypes">');
-                res.startBlock('<list>');
-
-                if ($generatorCommon.isDefinedAndNotEmpty(domain.keyType))
-                    res.line('<value>' + $generatorCommon.JavaTypes.fullClassName(domain.keyType) + '</value>');
-                else
-                    res.line('<value>???</value>');
-
-                if ($generatorCommon.isDefinedAndNotEmpty(domain.valueType))
-                    res.line('<value>' + $generatorCommon.JavaTypes.fullClassName(domain.valueType) + '</value>');
-                else
-                    res.line('<value>>???</value>');
-
-                res.endBlock('</list>');
-                res.endBlock('</property>');
-            }
-
-            break;
-
-        case 'Configuration':
-            $generatorSpring.classNameProperty(res, domain, 'keyType');
-            $generatorSpring.property(res, domain, 'valueType');
-
-            break;
-
-        default:
-    }
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate domain model for query group.
-$generatorSpring.domainModelQuery = function(domain, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.domainQueryMetadata(domain) === 'Configuration') {
-        $generatorSpring.domainModelQueryFields(res, domain);
-        $generatorSpring.domainModelQueryAliases(res, domain);
-        $generatorSpring.domainModelQueryIndexes(res, domain);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate domain model for store group.
-$generatorSpring.domainStore = function(domain, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, domain, 'databaseSchema');
-    $generatorSpring.property(res, domain, 'databaseTable');
-
-    res.softEmptyLine();
-
-    $generatorSpring.domainModelDatabaseFields(res, domain, 'keyFields');
-    $generatorSpring.domainModelDatabaseFields(res, domain, 'valueFields');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-$generatorSpring.cacheQueryMetadata = function(domain, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    res.startBlock('<bean class="org.apache.ignite.cache.QueryEntity">');
-
-    $generatorSpring.classNameProperty(res, domain, 'keyType');
-    $generatorSpring.property(res, domain, 'valueType');
-
-    $generatorSpring.domainModelQuery(domain, res);
-
-    res.endBlock('</bean>');
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-// Generate domain models configs.
-$generatorSpring.cacheDomains = function(domains, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    const domainConfigs = _.filter(domains, function(domain) {
-        return $generatorCommon.domainQueryMetadata(domain) === 'Configuration' &&
-            $generatorCommon.isDefinedAndNotEmpty(domain.fields);
-    });
-
-    if ($generatorCommon.isDefinedAndNotEmpty(domainConfigs)) {
-        res.emptyLineIfNeeded();
-
-        res.startBlock('<property name="queryEntities">');
-        res.startBlock('<list>');
-
-        _.forEach(domainConfigs, function(domain) {
-            $generatorSpring.cacheQueryMetadata(domain, res);
-        });
-
-        res.endBlock('</list>');
-        res.endBlock('</property>');
-    }
-
-    return res;
-};
-
-// Generate cache configs.
-$generatorSpring.cache = function(cache, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    res.startBlock('<bean class="org.apache.ignite.configuration.CacheConfiguration">');
-
-    $generatorSpring.cacheConfiguration(cache, res);
-
-    res.endBlock('</bean>');
-
-    return res;
-};
-
-// Generate cache configs.
-$generatorSpring.cacheConfiguration = function(cache, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.cacheGeneral(cache, res);
-    $generatorSpring.cacheMemory(cache, res);
-    $generatorSpring.cacheQuery(cache, cache.domains, res);
-    $generatorSpring.cacheStore(cache, cache.domains, res);
-
-    const igfs = _.get(cache, 'nodeFilter.IGFS.instance');
-
-    $generatorSpring.cacheNodeFilter(cache, igfs ? [igfs] : [], res);
-    $generatorSpring.cacheConcurrency(cache, res);
-    $generatorSpring.cacheRebalance(cache, res);
-    $generatorSpring.cacheServerNearCache(cache, res);
-    $generatorSpring.cacheStatistics(cache, res);
-    $generatorSpring.cacheDomains(cache.domains, res);
-
-    return res;
-};
-
-// Generate caches configs.
-$generatorSpring.clusterCaches = function(caches, igfss, isSrvCfg, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.isDefinedAndNotEmpty(caches) || (isSrvCfg && $generatorCommon.isDefinedAndNotEmpty(igfss))) {
-        res.emptyLineIfNeeded();
-
-        res.startBlock('<property name="cacheConfiguration">');
-        res.startBlock('<list>');
-
-        _.forEach(caches, function(cache) {
-            $generatorSpring.cache(cache, res);
-
-            res.needEmptyLine = true;
-        });
-
-        if (isSrvCfg) {
-            _.forEach(igfss, (igfs) => {
-                $generatorSpring.cache($generatorCommon.igfsDataCache(igfs), res);
-
-                res.needEmptyLine = true;
-
-                $generatorSpring.cache($generatorCommon.igfsMetaCache(igfs), res);
-
-                res.needEmptyLine = true;
-            });
-        }
-
-        res.endBlock('</list>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate IGFSs configs.
-$generatorSpring.igfss = function(igfss, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.isDefinedAndNotEmpty(igfss)) {
-        res.emptyLineIfNeeded();
-
-        res.startBlock('<property name="fileSystemConfiguration">');
-        res.startBlock('<list>');
-
-        _.forEach(igfss, function(igfs) {
-            res.startBlock('<bean class="org.apache.ignite.configuration.FileSystemConfiguration">');
-
-            $generatorSpring.igfsGeneral(igfs, res);
-            $generatorSpring.igfsIPC(igfs, res);
-            $generatorSpring.igfsFragmentizer(igfs, res);
-            $generatorSpring.igfsDualMode(igfs, res);
-            $generatorSpring.igfsSecondFS(igfs, res);
-            $generatorSpring.igfsMisc(igfs, res);
-
-            res.endBlock('</bean>');
-
-            res.needEmptyLine = true;
-        });
-
-        res.endBlock('</list>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate IGFS IPC configuration.
-$generatorSpring.igfsIPC = function(igfs, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (igfs.ipcEndpointEnabled) {
-        $generatorSpring.beanProperty(res, igfs.ipcEndpointConfiguration, 'ipcEndpointConfiguration', $generatorCommon.IGFS_IPC_CONFIGURATION, true);
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate IGFS fragmentizer configuration.
-$generatorSpring.igfsFragmentizer = function(igfs, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (igfs.fragmentizerEnabled) {
-        $generatorSpring.property(res, igfs, 'fragmentizerConcurrentFiles', null, 0);
-        $generatorSpring.property(res, igfs, 'fragmentizerThrottlingBlockLength', null, 16777216);
-        $generatorSpring.property(res, igfs, 'fragmentizerThrottlingDelay', null, 200);
-
-        res.needEmptyLine = true;
-    }
-    else
-        $generatorSpring.property(res, igfs, 'fragmentizerEnabled');
-
-    return res;
-};
-
-// Generate IGFS dual mode configuration.
-$generatorSpring.igfsDualMode = function(igfs, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, igfs, 'dualModeMaxPendingPutsSize', null, 0);
-
-    if ($generatorCommon.isDefinedAndNotEmpty(igfs.dualModePutExecutorService)) {
-        res.startBlock('<property name="dualModePutExecutorService">');
-        res.line('<bean class="' + igfs.dualModePutExecutorService + '"/>');
-        res.endBlock('</property>');
-    }
-
-    $generatorSpring.property(res, igfs, 'dualModePutExecutorServiceShutdown', null, false);
-
-    res.needEmptyLine = true;
-
-    return res;
-};
-
-$generatorSpring.igfsSecondFS = function(igfs, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (igfs.secondaryFileSystemEnabled) {
-        const secondFs = igfs.secondaryFileSystem || {};
-
-        res.startBlock('<property name="secondaryFileSystem">');
-
-        res.startBlock('<bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem">');
-
-        const nameDefined = $generatorCommon.isDefinedAndNotEmpty(secondFs.userName);
-        const cfgDefined = $generatorCommon.isDefinedAndNotEmpty(secondFs.cfgPath);
-
-        $generatorSpring.constructorArg(res, 0, secondFs, 'uri');
-
-        if (cfgDefined || nameDefined)
-            $generatorSpring.constructorArg(res, 1, secondFs, 'cfgPath');
-
-        $generatorSpring.constructorArg(res, 2, secondFs, 'userName', null, true);
-
-        res.endBlock('</bean>');
-        res.endBlock('</property>');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate IGFS general configuration.
-$generatorSpring.igfsGeneral = function(igfs, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if ($generatorCommon.isDefinedAndNotEmpty(igfs.name)) {
-        igfs.dataCacheName = $generatorCommon.igfsDataCache(igfs).name;
-        igfs.metaCacheName = $generatorCommon.igfsMetaCache(igfs).name;
-
-        $generatorSpring.property(res, igfs, 'name');
-        $generatorSpring.property(res, igfs, 'dataCacheName');
-        $generatorSpring.property(res, igfs, 'metaCacheName');
-        $generatorSpring.property(res, igfs, 'defaultMode', null, 'DUAL_ASYNC');
-
-        res.needEmptyLine = true;
-    }
-
-    return res;
-};
-
-// Generate IGFS misc configuration.
-$generatorSpring.igfsMisc = function(igfs, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    $generatorSpring.property(res, igfs, 'blockSize', null, 65536);
-    $generatorSpring.property(res, igfs, 'streamBufferSize', null, 65536);
-    $generatorSpring.property(res, igfs, 'maxSpaceSize', null, 0);
-    $generatorSpring.property(res, igfs, 'maximumTaskRangeLength', null, 0);
-    $generatorSpring.property(res, igfs, 'managementPort', null, 11400);
-    $generatorSpring.property(res, igfs, 'perNodeBatchSize', null, 100);
-    $generatorSpring.property(res, igfs, 'perNodeParallelBatchCount', null, 8);
-    $generatorSpring.property(res, igfs, 'prefetchBlocks', null, 0);
-    $generatorSpring.property(res, igfs, 'sequentialReadsBeforePrefetch', null, 0);
-    $generatorSpring.property(res, igfs, 'trashPurgeTimeout', null, 1000);
-    $generatorSpring.property(res, igfs, 'colocateMetadata', null, true);
-    $generatorSpring.property(res, igfs, 'relaxedConsistency', null, true);
-
-    res.softEmptyLine();
-
-    if (igfs.pathModes && igfs.pathModes.length > 0) {
-        res.startBlock('<property name="pathModes">');
-        res.startBlock('<map>');
-
-        _.forEach(igfs.pathModes, function(pair) {
-            res.line('<entry key="' + pair.path + '" value="' + pair.mode + '"/>');
-        });
-
-        res.endBlock('</map>');
-        res.endBlock('</property>');
-    }
-
-    return res;
-};
-
-// Generate DataSource beans.
-$generatorSpring.generateDataSources = function(datasources, res) {
-    if (!res)
-        res = $generatorCommon.builder();
-
-    if (datasources.length > 0) {
-        res.line('<!-- Data source beans will be initialized from external properties file. -->');
-
-        _.forEach(datasources, (datasource) => $generatorSpring.generateDataSource(datasource, res));
-
-        res.needEmptyLine = true;
-
-        res.emptyLineIfNeeded();
-    }
-
-    return res;
-};
-
-$generatorSpring.generateDataSource = function(datasource, res) {
-    const beanId = datasource.dataSourceBean;
-
-    res.startBlock('<bean id="' + beanId + '" class="' + $generatorCommon.DATA_SOURCES[datasource.dialect] + '">');
-
-    switch (datasource.dialect) {
-        case 'Generic':
-            res.line('<property name="jdbcUrl" value="${' + beanId + '.jdbc.url}"/>');
-
-            break;
-
-        case 'DB2':
-            res.line('<property name="serverName" value="${' + beanId + '.jdbc.server_name}"/>');
-            res.line('<property name="portNumber" value="${' + beanId + '.jdbc.port_number}"/>');
-            res.line('<property name="databaseName" value="${' + beanId + '.jdbc.database_name}"/>');
-            res.line('<property name="driverType" value="${' + beanId + '.jdbc.driver_type}"/>');
-
-            break;
-
-        case 'PostgreSQL':
-            res.line('<property name="url" value="${' + beanId + '.jdbc.url}"/>');
-
-            break;
-
-        default:
-            res.line('<property name="URL" value="${' + beanId + '.jdbc.url}"/>');
-    }
-
-    res.line('<property name="user" value="${' + beanId + '.jdbc.username}"/>');
-    res.line('<property name="password" value="${' + beanId + '.jdbc.password}"/>');
-
-    res.endBlock('</bean>');
-
-    res.needEmptyLine = true;
-
-    res.emptyLineIfNeeded();
-};
-
-$generatorSpring.clusterConfiguration = function(cluster, clientNearCfg, res) {
-    const isSrvCfg = _.isNil(clientNearCfg);
-
-    if (!isSrvCfg) {
-        res.line('<property name="clientMode" value="true"/>');
-
-        res.needEmptyLine = true;
-    }
-
-    $generatorSpring.clusterGeneral(cluster, res);
-
-    $generatorSpring.clusterAtomics(cluster.atomicConfiguration, res);
-
-    $generatorSpring.clusterBinary(cluster.binaryConfiguration, res);
-
-    $generatorSpring.clusterCacheKeyConfiguration(cluster.cacheKeyConfiguration, res);
-
-    $generatorSpring.clusterCollision(cluster.collision, res);
-
-    $generatorSpring.clusterCommunication(cluster, res);
-
-    $generatorSpring.clusterConnector(cluster.connector, res);
-
-    $generatorSpring.clusterDeployment(cluster, res);
-
-    $generatorSpring.clusterEvents(cluster, res);
-
-    $generatorSpring.clusterFailover(cluster, res);
-
-    $generatorSpring.clusterLogger(cluster.logger, res);
-
-    $generatorSpring.clusterODBC(cluster.odbc, res);
-
-    $generatorSpring.clusterMarshaller(cluster, res);
-
-    $generatorSpring.clusterMetrics(cluster, res);
-
-    $generatorSpring.clusterSwap(cluster, res);
-
-    $generatorSpring.clusterTime(cluster, res);
-
-    $generatorSpring.clusterPools(cluster, res);
-
-    $generatorSpring.clusterTransactions(cluster.transactionConfiguration, res);
-
-    $generatorSpring.clusterCaches(cluster.caches, cluster.igfss, isSrvCfg, res);
-
-    $generatorSpring.clusterSsl(cluster, res);
-
-    if (isSrvCfg)
-        $generatorSpring.igfss(cluster.igfss, res);
-
-    $generatorSpring.clusterUserAttributes(cluster, res);
-
-    return res;
-};
-
-$generatorSpring.cluster = function(cluster, clientNearCfg) {
-    if (cluster) {
-        const res = $generatorCommon.builder(1);
-
-        if (clientNearCfg) {
-            res.startBlock('<bean id="nearCacheBean" class="org.apache.ignite.configuration.NearCacheConfiguration">');
-
-            if (clientNearCfg.nearStartSize)
-                $generatorSpring.property(res, clientNearCfg, 'nearStartSize');
-
-            if (clientNearCfg.nearEvictionPolicy && clientNearCfg.nearEvictionPolicy.kind)
-                $generatorSpring.evictionPolicy(res, clientNearCfg.nearEvictionPolicy, 'nearEvictionPolicy');
-
-            res.endBlock('</bean>');
-
-            res.needEmptyLine = true;
-
-            res.emptyLineIfNeeded();
-        }
-
-        // Generate Ignite Configuration.
-        res.startBlock('<bean class="org.apache.ignite.configuration.IgniteConfiguration">');
-
-        $generatorSpring.clusterConfiguration(cluster, clientNearCfg, res);
-
-        res.endBlock('</bean>');
-
-        // Build final XML:
-        // 1. Add header.
-        let xml = '<?xml version="1.0" encoding="UTF-8"?>\n\n';
-
-        xml += '<!-- ' + $generatorCommon.mainComment('configuration') + ' -->\n\n';
-        xml += '<beans xmlns="http://www.springframework.org/schema/beans"\n';
-        xml += '       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n';
-        xml += '       xmlns:util="http://www.springframework.org/schema/util"\n';
-        xml += '       xsi:schemaLocation="http://www.springframework.org/schema/beans\n';
-        xml += '                           http://www.springframework.org/schema/beans/spring-beans.xsd\n';
-        xml += '                           http://www.springframework.org/schema/util\n';
-        xml += '                           http://www.springframework.org/schema/util/spring-util.xsd">\n';
-
-        // 2. Add external property file
-        if ($generatorCommon.secretPropertiesNeeded(cluster)) {
-            xml += '    <!-- Load external properties file. -->\n';
-            xml += '    <bean id="placeholderConfig" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">\n';
-            xml += '        <property name="location" value="classpath:secret.properties"/>\n';
-            xml += '    </bean>\n\n';
-        }
-
-        // 3. Add data sources.
-        xml += $generatorSpring.generateDataSources(res.datasources, $generatorCommon.builder(1)).asString();
-
-        // 3. Add main content.
-        xml += res.asString();
-
-        // 4. Add footer.
-        xml += '\n</beans>';
-
-        return xml;
-    }
-
-    return '';
-};
-
-export default $generatorSpring;
diff --git a/modules/web-console/frontend/app/modules/sql/Notebook.data.js b/modules/web-console/frontend/app/modules/sql/Notebook.data.js
index f66faba07cd8f..3f98bedc5ca0b 100644
--- a/modules/web-console/frontend/app/modules/sql/Notebook.data.js
+++ b/modules/web-console/frontend/app/modules/sql/Notebook.data.js
@@ -21,7 +21,8 @@ const DEMO_NOTEBOOK = {
         {
             name: 'Query with refresh rate',
             cacheName: 'CarCache',
-            pageSize: 50,
+            pageSize: 100,
+            limit: 0,
             query: [
                 'SELECT count(*)',
                 'FROM "CarCache".Car'
@@ -37,7 +38,8 @@ const DEMO_NOTEBOOK = {
         {
             name: 'Simple query',
             cacheName: 'CarCache',
-            pageSize: 50,
+            pageSize: 100,
+            limit: 0,
             query: 'SELECT * FROM "CarCache".Car',
             result: 'table',
             timeLineSpan: '1',
@@ -49,8 +51,9 @@ const DEMO_NOTEBOOK = {
         },
         {
             name: 'Query with aggregates',
-            cacheName: 'CarCache',
-            pageSize: 50,
+            cacheName: 'ParkingCache',
+            pageSize: 100,
+            limit: 0,
             query: [
                 'SELECT p.name, count(*) AS cnt',
                 'FROM "ParkingCache".Parking p',
diff --git a/modules/web-console/frontend/app/modules/sql/Notebook.service.js b/modules/web-console/frontend/app/modules/sql/Notebook.service.js
index 12730be686611..f32d26f45c4f5 100644
--- a/modules/web-console/frontend/app/modules/sql/Notebook.service.js
+++ b/modules/web-console/frontend/app/modules/sql/Notebook.service.js
@@ -60,7 +60,7 @@ export default class Notebook {
     }
 
     remove(notebook) {
-        return this.confirmModal.confirm(`Are you sure you want to remove: "${notebook.name}"?`)
+        return this.confirmModal.confirm(`Are you sure you want to remove notebook: "${notebook.name}"?`)
             .then(() => this.NotebookData.findIndex(notebook))
             .then((idx) => {
                 this.NotebookData.remove(notebook)
diff --git a/modules/web-console/frontend/app/modules/sql/scan-filter-input.jade b/modules/web-console/frontend/app/modules/sql/scan-filter-input.jade
deleted file mode 100644
index 0396727f95d89..0000000000000
--- a/modules/web-console/frontend/app/modules/sql/scan-filter-input.jade
+++ /dev/null
@@ -1,39 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-include /app/helpers/jade/mixins.jade
-
-.modal(tabindex='-1' role='dialog')
-    .modal-dialog
-        .modal-content
-            .modal-header
-                button.close(ng-click='$hide()') &times;
-                h4.modal-title Scan filter
-            form.form-horizontal.modal-body.row(name='ui.inputForm' novalidate)
-                .settings-row
-                    .col-sm-2
-                        label.required.labelFormField Filter:&nbsp;
-                    .col-sm-10
-                        .input-tip
-                            +ignite-form-field-input('"filter"', 'ui.filter', false, 'true', 'Enter filter')(
-                                data-ignite-form-field-input-autofocus='true'
-                                ignite-on-enter='form.$valid && ok()'
-                            )
-                .settings-row
-                    +checkbox('Case sensitive search', 'ui.caseSensitive', '"caseSensitive"', 'Select this checkbox for case sensitive search')
-            .modal-footer
-                button.btn.btn-default(id='btn-cancel' ng-click='$hide()') Cancel
-                button.btn.btn-primary(id='btn-scan' ng-disabled='ui.inputForm.$invalid' ng-click='ok()') Scan
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 4e6e3720bfbef..0d0b171df5595 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -50,6 +50,9 @@ class Paragraph {
         const self = this;
 
         self.id = 'paragraph-' + paragraphId++;
+        self.qryType = paragraph.qryType || 'query';
+        self.maxPages = 0;
+        self.filter = '';
 
         _.assign(this, paragraph);
 
@@ -77,27 +80,28 @@ class Paragraph {
             enableColumnMenus: false,
             flatEntityAccess: true,
             fastWatch: true,
+            categories: [],
             rebuildColumns() {
                 if (_.isNil(this.api))
                     return;
 
-                this.categories = [];
+                this.categories.length = 0;
+
                 this.columnDefs = _.reduce(self.meta, (cols, col, idx) => {
-                    if (self.columnFilter(col)) {
-                        cols.push({
-                            displayName: col.fieldName,
-                            headerTooltip: _fullColName(col),
-                            field: idx.toString(),
-                            minWidth: 50,
-                            cellClass: 'cell-left'
-                        });
+                    cols.push({
+                        displayName: col.fieldName,
+                        headerTooltip: _fullColName(col),
+                        field: idx.toString(),
+                        minWidth: 50,
+                        cellClass: 'cell-left',
+                        visible: self.columnFilter(col)
+                    });
 
-                        this.categories.push({
-                            name: col.fieldName,
-                            visible: true,
-                            selectable: true
-                        });
-                    }
+                    this.categories.push({
+                        name: col.fieldName,
+                        visible: self.columnFilter(col),
+                        selectable: true
+                    });
 
                     return cols;
                 }, []);
@@ -182,8 +186,8 @@ class Paragraph {
 }
 
 // Controller for SQL notebook screen.
-export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteScanFilterInput', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion',
-    function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, ScanFilterInput, Nodes, uiGridExporterConstants, Version) {
+export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion',
+    function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version) {
         let stopTopology = null;
 
         const _tryStopRefresh = function(paragraph) {
@@ -206,6 +210,15 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
         $scope.caches = [];
 
         $scope.pageSizes = [50, 100, 200, 400, 800, 1000];
+        $scope.maxPages = [
+            {label: 'Unlimited', value: 0},
+            {label: '1', value: 1},
+            {label: '5', value: 5},
+            {label: '10', value: 10},
+            {label: '20', value: 20},
+            {label: '50', value: 50},
+            {label: '100', value: 100}
+        ];
 
         $scope.timeLineSpans = ['1', '5', '10', '15', '30'];
 
@@ -213,7 +226,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
         $scope.modes = LegacyUtils.mkOptions(['PARTITIONED', 'REPLICATED', 'LOCAL']);
 
-        $scope.loadingText = $root.IgniteDemoMode ? 'Demo grid is starting. Please wait...' : 'Loading notebook screen...';
+        $scope.loadingText = $root.IgniteDemoMode ? 'Demo grid is starting. Please wait...' : 'Loading query notebook screen...';
 
         $scope.timeUnit = [
             {value: 1000, label: 'seconds', short: 's'},
@@ -768,11 +781,10 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
             if (idx >= 0) {
                 if (!_.includes($scope.notebook.expandedParagraphs, idx))
-                    $scope.notebook.expandedParagraphs.push(idx);
+                    $scope.notebook.expandedParagraphs = $scope.notebook.expandedParagraphs.concat([idx]);
 
-                setTimeout(function() {
-                    $scope.notebook.paragraphs[idx].ace.focus();
-                });
+                if ($scope.notebook.paragraphs[idx].ace)
+                    setTimeout(() => $scope.notebook.paragraphs[idx].ace.focus());
             }
 
             $location.hash(id);
@@ -816,7 +828,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                             let item = _.find(cachesAcc, {name: cache.name});
 
                             if (_.isNil(item)) {
-                                cache.label = maskCacheName(cache.name);
+                                cache.label = maskCacheName(cache.name, true);
+                                cache.value = cache.name;
 
                                 cache.nodes = [];
 
@@ -839,7 +852,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                         return;
 
                     // Reset to first cache in case of stopped selected.
-                    const cacheNames = _.map($scope.caches, (cache) => cache.name);
+                    const cacheNames = _.map($scope.caches, (cache) => cache.value);
 
                     _.forEach($scope.notebook.paragraphs, (paragraph) => {
                         if (!_.includes(cacheNames, paragraph.cacheName))
@@ -885,7 +898,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                     (paragraph) => new Paragraph($animate, $timeout, paragraph));
 
                 if (_.isEmpty($scope.notebook.paragraphs))
-                    $scope.addParagraph();
+                    $scope.addQuery();
                 else
                     $scope.rebuildScrollParagraphs();
             })
@@ -936,32 +949,37 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 paragraph.edit = false;
         };
 
-        $scope.addParagraph = function() {
+        $scope.addParagraph = (paragraph, sz) => {
+            if ($scope.caches && $scope.caches.length > 0)
+                paragraph.cacheName = _.head($scope.caches).value;
+
+            $scope.notebook.paragraphs.push(paragraph);
+
+            $scope.notebook.expandedParagraphs.push(sz);
+
+            $scope.rebuildScrollParagraphs();
+
+            $location.hash(paragraph.id);
+        };
+
+        $scope.addQuery = function() {
             const sz = $scope.notebook.paragraphs.length;
 
             const paragraph = new Paragraph($animate, $timeout, {
                 name: 'Query' + (sz === 0 ? '' : sz),
                 query: '',
-                pageSize: $scope.pageSizes[0],
+                pageSize: $scope.pageSizes[1],
                 timeLineSpan: $scope.timeLineSpans[0],
                 result: 'none',
                 rate: {
                     value: 1,
                     unit: 60000,
                     installed: false
-                }
+                },
+                qryType: 'query'
             });
 
-            if ($scope.caches && $scope.caches.length > 0)
-                paragraph.cacheName = $scope.caches[0].name;
-
-            $scope.notebook.paragraphs.push(paragraph);
-
-            $scope.notebook.expandedParagraphs.push(sz);
-
-            $scope.rebuildScrollParagraphs();
-
-            $location.hash(paragraph.id);
+            $scope.addParagraph(paragraph, sz);
 
             $timeout(() => {
                 $anchorScroll();
@@ -970,6 +988,26 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
             });
         };
 
+        $scope.addScan = function() {
+            const sz = $scope.notebook.paragraphs.length;
+
+            const paragraph = new Paragraph($animate, $timeout, {
+                name: 'Scan' + (sz === 0 ? '' : sz),
+                query: '',
+                pageSize: $scope.pageSizes[1],
+                timeLineSpan: $scope.timeLineSpans[0],
+                result: 'none',
+                rate: {
+                    value: 1,
+                    unit: 60000,
+                    installed: false
+                },
+                qryType: 'scan'
+            });
+
+            $scope.addParagraph(paragraph, sz);
+        };
+
         function _saveChartSettings(paragraph) {
             if (!_.isEmpty(paragraph.charts)) {
                 const chart = paragraph.charts[0].api.getScope().chart;
@@ -1010,7 +1048,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
         };
 
         $scope.removeParagraph = function(paragraph) {
-            Confirm.confirm('Are you sure you want to remove: "' + paragraph.name + '"?')
+            Confirm.confirm('Are you sure you want to remove query: "' + paragraph.name + '"?')
                 .then(function() {
                     $scope.stopRefresh(paragraph);
 
@@ -1315,8 +1353,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
             return false;
         };
 
-        $scope.execute = (paragraph, nonCollocatedJoins = false) => {
-            const local = !!paragraph.localQry;
+        $scope.execute = (paragraph, local = false) => {
+            const nonCollocatedJoins = !!paragraph.nonCollocatedJoins;
 
             $scope.actionAvailable(paragraph, true) && _chooseNode(paragraph.cacheName, local)
                 .then((nid) => {
@@ -1330,16 +1368,16 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                     return _closeOldQuery(paragraph)
                         .then(() => {
                             const args = paragraph.queryArgs = {
+                                type: 'QUERY',
                                 cacheName: paragraph.cacheName,
-                                pageSize: paragraph.pageSize,
                                 query: paragraph.query,
-                                firstPageOnly: paragraph.firstPageOnly,
+                                pageSize: paragraph.pageSize,
+                                maxPages: paragraph.maxPages,
                                 nonCollocatedJoins,
-                                type: 'QUERY',
                                 localNid: local ? nid : null
                             };
 
-                            const qry = args.firstPageOnly ? addLimit(args.query, args.pageSize) : paragraph.query;
+                            const qry = args.maxPages ? addLimit(args.query, args.pageSize * args.maxPages) : paragraph.query;
 
                             return agentMonitor.query(nid, args.cacheName, qry, nonCollocatedJoins, local, args.pageSize);
                         })
@@ -1386,10 +1424,10 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 .then(() => _chooseNode(paragraph.cacheName, false))
                 .then((nid) => {
                     const args = paragraph.queryArgs = {
+                        type: 'EXPLAIN',
                         cacheName: paragraph.cacheName,
-                        pageSize: paragraph.pageSize,
                         query: 'EXPLAIN ' + paragraph.query,
-                        type: 'EXPLAIN'
+                        pageSize: paragraph.pageSize
                     };
 
                     return agentMonitor.query(nid, args.cacheName, args.query, false, false, args.pageSize);
@@ -1403,8 +1441,10 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 .then(() => paragraph.ace.focus());
         };
 
-        $scope.scan = (paragraph, query = null) => {
-            const local = !!paragraph.localQry;
+        $scope.scan = (paragraph, local = false) => {
+            const {filter, caseSensitive} = paragraph;
+            const prefix = caseSensitive ? SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE : SCAN_CACHE_WITH_FILTER;
+            const query = `${prefix}${filter}`;
 
             $scope.actionAvailable(paragraph, false) && _chooseNode(paragraph.cacheName, local)
                 .then((nid) => {
@@ -1418,45 +1458,22 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                     _closeOldQuery(paragraph)
                         .then(() => {
                             const args = paragraph.queryArgs = {
+                                type: 'SCAN',
                                 cacheName: paragraph.cacheName,
-                                pageSize: paragraph.pageSize,
-                                firstPageOnly: paragraph.firstPageOnly,
                                 query,
-                                type: 'SCAN',
+                                filter,
+                                pageSize: paragraph.pageSize,
                                 localNid: local ? nid : null
                             };
 
                             return agentMonitor.query(nid, args.cacheName, query, false, local, args.pageSize);
                         })
-                        .then((res) => {
-                            if (paragraph.firstPageOnly) {
-                                res.hasMore = false;
-
-                                _processQueryResult(paragraph, true, res);
-
-                                _closeOldQuery(paragraph);
-                            }
-                            else
-                                _processQueryResult(paragraph, true, res);
-                        })
+                        .then((res) => _processQueryResult(paragraph, true, res))
                         .catch((err) => {
                             paragraph.errMsg = err.message;
 
                             _showLoading(paragraph, false);
-                        })
-                        .then(() => paragraph.ace.focus());
-                });
-        };
-
-        $scope.scanWithFilter = (paragraph) => {
-            if (!$scope.actionAvailable(paragraph, false))
-                return;
-
-            ScanFilterInput.open()
-                .then(({filter, caseSensitive}) => {
-                    const prefix = caseSensitive ? SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE : SCAN_CACHE_WITH_FILTER;
-
-                    $scope.scan(paragraph, `${prefix}${filter}`);
+                        });
                 });
         };
 
@@ -1511,25 +1528,23 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
                     _showLoading(paragraph, false);
                 })
-                .then(() => paragraph.ace.focus());
+                .then(() => paragraph.ace && paragraph.ace.focus());
         };
 
-        const _export = (fileName, columnFilter, meta, rows) => {
+        const _export = (fileName, columnDefs, meta, rows) => {
             let csvContent = '';
 
             const cols = [];
             const excludedCols = [];
 
-            if (meta) {
-                _.forEach(meta, (col, idx) => {
-                    if (columnFilter(col))
-                        cols.push(_fullColName(col));
-                    else
-                        excludedCols.push(idx);
-                });
+            _.forEach(meta, (col, idx) => {
+                if (columnDefs[idx].visible)
+                    cols.push(_fullColName(col));
+                else
+                    excludedCols.push(idx);
+            });
 
-                csvContent += cols.join(';') + '\n';
-            }
+            csvContent += cols.join(';') + '\n';
 
             _.forEach(rows, (row) => {
                 cols.length = 0;
@@ -1543,8 +1558,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                     });
                 }
                 else {
-                    _.forEach(meta, (col) => {
-                        if (columnFilter(col)) {
+                    _.forEach(columnDefs, (col) => {
+                        if (col.visible) {
                             const elem = row[col.fieldName];
 
                             cols.push(_.isUndefined(elem) ? '' : JSON.stringify(elem));
@@ -1559,7 +1574,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
         };
 
         $scope.exportCsv = function(paragraph) {
-            _export(paragraph.name + '.csv', paragraph.columnFilter, paragraph.meta, paragraph.rows);
+            _export(paragraph.name + '.csv', paragraph.gridOptions.columnDefs, paragraph.meta, paragraph.rows);
 
             // paragraph.gridOptions.api.exporter.csvExport(uiGridExporterConstants.ALL, uiGridExporterConstants.VISIBLE);
         };
@@ -1573,17 +1588,17 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
             return Promise.resolve(args.localNid || _chooseNode(args.cacheName, false))
                 .then((nid) => agentMonitor.queryGetAll(nid, args.cacheName, args.query, !!args.nonCollocatedJoins, !!args.localNid))
-                .then((res) => _export(paragraph.name + '-all.csv', paragraph.columnFilter, res.columns, res.rows))
+                .then((res) => _export(paragraph.name + '-all.csv', paragraph.gridOptions.columnDefs, res.columns, res.rows))
                 .catch(Messages.showError)
-                .then(() => paragraph.ace.focus());
+                .then(() => paragraph.ace && paragraph.ace.focus());
         };
 
         // $scope.exportPdfAll = function(paragraph) {
         //    $http.post('/api/v1/agent/query/getAll', {query: paragraph.query, cacheName: paragraph.cacheName})
-        //        .success(function(item) {
-        //            _export(paragraph.name + '-all.csv', item.meta, item.rows);
+        //    .then(({data}) {
+        //        _export(paragraph.name + '-all.csv', data.meta, data.rows);
         //    })
-        //    .error(Messages.showError);
+        //    .catch(Messages.showError);
         // };
 
         $scope.rateAsString = function(paragraph) {
@@ -1652,9 +1667,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
         $scope.dblclickMetadata = function(paragraph, node) {
             paragraph.ace.insert(node.name);
 
-            setTimeout(function() {
-                paragraph.ace.focus();
-            }, 1);
+            setTimeout(() => paragraph.ace.focus(), 1);
         };
 
         $scope.importMetadata = function() {
diff --git a/modules/web-console/frontend/app/modules/sql/sql.module.js b/modules/web-console/frontend/app/modules/sql/sql.module.js
index d615d28e108c2..a1ffde9646f99 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.module.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.module.js
@@ -19,7 +19,6 @@ import angular from 'angular';
 
 import NotebookData from './Notebook.data';
 import Notebook from './Notebook.service';
-import ScanFilterInput from './scan-filter-input.service';
 import notebook from './notebook.controller';
 import sql from './sql.controller';
 
@@ -55,6 +54,5 @@ angular.module('ignite-console.sql', [
     )
     .service('IgniteNotebookData', NotebookData)
     .service('IgniteNotebook', Notebook)
-    .service('IgniteScanFilterInput', ScanFilterInput)
     .controller('notebookController', notebook)
     .controller('sqlController', sql);
diff --git a/modules/web-console/frontend/app/modules/states/configuration.state.js b/modules/web-console/frontend/app/modules/states/configuration.state.js
index 888c804b37af4..61dca13f9633b 100644
--- a/modules/web-console/frontend/app/modules/states/configuration.state.js
+++ b/modules/web-console/frontend/app/modules/states/configuration.state.js
@@ -24,12 +24,14 @@ import previewPanel from './configuration/preview-panel.directive.js';
 import ConfigurationSummaryCtrl from './configuration/summary/summary.controller';
 import ConfigurationResource from './configuration/Configuration.resource';
 import summaryTabs from './configuration/summary/summary-tabs.directive';
+import IgniteSummaryZipper from './configuration/summary/summary-zipper.service';
 
 angular.module('ignite-console.states.configuration', ['ui.router'])
     .directive(...previewPanel)
     // Summary screen
     .directive(...summaryTabs)
     // Services.
+    .service('IgniteSummaryZipper', IgniteSummaryZipper)
     .service('IgniteConfigurationResource', ConfigurationResource)
     // Configure state provider.
     .config(['$stateProvider', 'AclRouteProvider', ($stateProvider, AclRoute) => {
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.jade
index b34aba032f6ff..bcac5adcaf995 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.jade
@@ -54,6 +54,6 @@ include /app/helpers/jade/mixins.jade
                         -var required = nodeFilterKind + ' === "Custom"'
 
                         +java-class('Class name:', customNodeFilter + '.className', '"customNodeFilter"',
-                            'true', required, 'Class name of custom node filter implementation')
+                            'true', required, 'Class name of custom node filter implementation', required)
             .col-sm-6
                 +preview-xml-java(model, 'cacheNodeFilter', 'igfss')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/query.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/query.jade
index 5062ce1ac9445..cfbaf12c57a8a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/query.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/query.jade
@@ -52,6 +52,9 @@ include /app/helpers/jade/mixins.jade
                 .settings-row
                     +number('Long query timeout:', model + '.longQueryWarningTimeout', '"longQueryWarningTimeout"', 'true', '3000', '0',
                         'Timeout in milliseconds after which long query warning will be printed')
+                .settings-row
+                    +number('History size:', model + '.queryDetailMetricsSize', '"queryDetailMetricsSize"', 'true', '0', '0',
+                        'Size of queries detail metrics that will be stored in memory for monitoring purposes')
                 .settings-row
                     -var form = 'querySqlFunctionClasses';
                     -var sqlFunctionClasses = model + '.sqlFunctionClasses';
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/store.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/store.jade
index 1cf80b8a193f4..ea350f2394ef7 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/store.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/store.jade
@@ -102,9 +102,9 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                                         'Parallel load cache minimum threshold.<br/>\
                                         If <b>0</b> then load sequentially.')
                                 .details-row
-                                    +java-class('Hasher', pojoStoreFactory + '.hasher', '"pojoHasher"', 'true', 'false', 'Hash calculator')
+                                    +java-class('Hasher', pojoStoreFactory + '.hasher', '"pojoHasher"', 'true', 'false', 'Hash calculator', required)
                                 .details-row
-                                    +java-class('Transformer', pojoStoreFactory + '.transformer', '"pojoTransformer"', 'true', 'false', 'Types transformer')
+                                    +java-class('Transformer', pojoStoreFactory + '.transformer', '"pojoTransformer"', 'true', 'false', 'Types transformer', required)
                                 .details-row
                                     +checkbox('Escape table and filed names', pojoStoreFactory + '.sqlEscapeAll', '"sqlEscapeAll"',
                                         'If enabled than all schema, table and field names will be escaped with double quotes (for example: "tableName"."fieldName").<br/>\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.jade
index 5cc996d5ff311..259909eac9ed9 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.jade
@@ -19,6 +19,7 @@ include /app/helpers/jade/mixins.jade
 -var form = 'checkpoint'
 -var model = 'backupItem.checkpointSpi'
 -var CustomCheckpoint = 'model.kind === "Custom"'
+-var CacheCheckpoint = 'model.kind === "Cache"'
 
 .panel.panel-default(ng-form=form novalidate)
     .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
@@ -44,7 +45,7 @@ include /app/helpers/jade/mixins.jade
                         .group-content(ng-show='#{model} && #{model}.length > 0' ng-repeat='model in #{model} track by $index')
                             hr(ng-if='$index != 0')
                             .settings-row
-                                +dropdown-required('Checkpoint SPI:', 'model.kind', '"checkpointKind" + $index', 'true', 'true', 'Choose checkpoint configuration variant', '[\
+                                +dropdown-required-autofocus('Checkpoint SPI:', 'model.kind', '"checkpointKind" + $index', 'true', 'true', 'Choose checkpoint configuration variant', '[\
                                         {value: "FS", label: "File System"},\
                                         {value: "Cache", label: "Cache"},\
                                         {value: "S3", label: "Amazon S3"},\
@@ -64,13 +65,13 @@ include /app/helpers/jade/mixins.jade
                             div(ng-show='model.kind === "FS"')
                                 include ./checkpoint/fs.jade
 
-                            div(ng-show='model.kind === "Cache"')
+                            div(ng-show=CacheCheckpoint)
                                 .settings-row
-                                    +dropdown-required-empty('Cache:', 'model.Cache.cache', '"checkpointCacheCache"+ $index', 'true', 'true',
+                                    +dropdown-required-empty('Cache:', 'model.Cache.cache', '"checkpointCacheCache"+ $index', 'true', CacheCheckpoint,
                                         'Choose cache', 'No caches configured for current cluster', 'clusterCaches', 'Cache to use for storing checkpoints')
                                 .settings-row
                                     +java-class('Listener:', 'model.Cache.checkpointListener', '"checkpointCacheListener" + $index', 'true', 'false',
-                                        'Checkpoint listener implementation class name')
+                                        'Checkpoint listener implementation class name', CacheCheckpoint)
 
                             div(ng-show='model.kind === "S3"')
                                 include ./checkpoint/s3.jade
@@ -80,6 +81,6 @@ include /app/helpers/jade/mixins.jade
 
                             .settings-row(ng-show=CustomCheckpoint)
                                 +java-class('Class name:', 'model.Custom.className', '"checkpointCustomClassName" + $index', 'true', CustomCheckpoint,
-                                'Custom CheckpointSpi implementation class')
+                                'Custom CheckpointSpi implementation class', CustomCheckpoint)
             .col-sm-6
                 +preview-xml-java('backupItem', 'clusterCheckpoint', 'caches')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.jade
index efb6ad0699260..6ec4535dbdcf5 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.jade
@@ -36,13 +36,13 @@ include /app/helpers/jade/mixins.jade
             -var valid = form + '[' + name + '].$valid'
             -var save = dirPaths + '[$index] = ' + model
 
-            div(ng-repeat='model in #{dirPaths} track by $index' ng-init='obj = {}')
+            div(ng-repeat='item in #{dirPaths} track by $index' ng-init='obj = {}')
                 label.col-xs-12.col-sm-12.col-md-12
                     .indexField
                         | {{ $index+1 }})
-                    +table-remove-conditional-button(dirPaths, 'true', 'Remove path')
+                    +table-remove-conditional-button(dirPaths, 'true', 'Remove path', 'item')
                     span(ng-hide='field.edit')
-                        a.labelFormField(ng-click='(field.edit = true) && (#{model} = model)') {{ model }}
+                        a.labelFormField(ng-click='(field.edit = true) && (#{model} = item)') {{ item }}
                     span(ng-if='field.edit')
                         +table-text-field(name, model, dirPaths, valid, save, 'Input directory path', false)
                             +table-save-button(valid, save, false)
@@ -63,4 +63,4 @@ include /app/helpers/jade/mixins.jade
 
 .settings-row
     +java-class('Listener:', 'model.FS.checkpointListener', '"checkpointFsListener" + $index', 'true', 'false',
-        'Checkpoint listener implementation class name')
+        'Checkpoint listener implementation class name', 'model.kind === "FS"')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.jade
index 874799c17c0b3..5a13337f564f1 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.jade
@@ -16,15 +16,17 @@
 
 include /app/helpers/jade/mixins.jade
 
+-var jdbcCheckpoint = 'model.kind === "JDBC"'
+
 .settings-row
-    +text('Data source bean name:', 'model.JDBC.dataSourceBean', '"checkpointJdbcDataSourceBean" + $index', 'model.kind === "JDBC"', 'Input bean name',
+    +text('Data source bean name:', 'model.JDBC.dataSourceBean', '"checkpointJdbcDataSourceBean" + $index', jdbcCheckpoint, 'Input bean name',
     'Name of the data source bean in Spring context')
 .settings-row
-    +dialect('Dialect:', 'model.JDBC.dialect', '"checkpointJdbcDialect" + $index', 'model.kind === "JDBC"',
+    +dialect('Dialect:', 'model.JDBC.dialect', '"checkpointJdbcDialect" + $index', jdbcCheckpoint,
     'Dialect of SQL implemented by a particular RDBMS:', 'Generic JDBC dialect', 'Choose JDBC dialect')
 .settings-row
     +java-class('Listener:', 'model.JDBC.checkpointListener', '"checkpointJdbcListener" + $index', 'true', 'false',
-        'Checkpoint listener implementation class name')
+        'Checkpoint listener implementation class name', jdbcCheckpoint)
 +showHideLink('jdbcExpanded', 'settings')
     .details-row
         +text('User:', 'model.JDBC.user', '"checkpointJdbcUser" + $index', 'false', 'Input user name', 'Checkpoint jdbc user name')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.jade
index da28da780a155..6531897130051 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.jade
@@ -18,16 +18,17 @@ include /app/helpers/jade/mixins.jade
 
 -var credentialsModel = 'model.S3.awsCredentials'
 -var clientCfgModel = 'model.S3.clientConfiguration'
--var checkpointS3Path = 'model.S3.awsCredentials.kind === "Properties"'
--var checkpointS3Custom = 'model.S3.awsCredentials.kind === "Custom"'
+-var checkpointS3 = 'model.kind === "S3"'
+-var checkpointS3Path = checkpointS3 + ' && model.S3.awsCredentials.kind === "Properties"'
+-var checkpointS3Custom = checkpointS3 + ' && model.S3.awsCredentials.kind === "Custom"'
 
 -var clientRetryModel = clientCfgModel + '.retryPolicy'
--var checkpointS3DefaultMaxRetry = clientRetryModel + '.kind === "DefaultMaxRetries"'
--var checkpointS3DynamoDbMaxRetry = clientRetryModel + '.kind === "DynamoDBMaxRetries"'
--var checkpointS3CustomRetry = clientRetryModel + '.kind === "Custom"'
+-var checkpointS3DefaultMaxRetry = checkpointS3 + ' && ' + clientRetryModel + '.kind === "DefaultMaxRetries"'
+-var checkpointS3DynamoDbMaxRetry = checkpointS3 + ' && ' + clientRetryModel + '.kind === "DynamoDBMaxRetries"'
+-var checkpointS3CustomRetry = checkpointS3 + ' && ' + clientRetryModel + '.kind === "Custom"'
 
 .settings-row
-    +dropdown-required('AWS credentials:', 'model.S3.awsCredentials.kind', '"checkpointS3AwsCredentials"', 'true', 'model.kind === "S3"', 'Custom', '[\
+    +dropdown-required('AWS credentials:', 'model.S3.awsCredentials.kind', '"checkpointS3AwsCredentials"', 'true', checkpointS3, 'Custom', '[\
         {value: "Basic", label: "Basic"},\
         {value: "Properties", label: "Properties"},\
         {value: "Anonymous", label: "Anonymous"},\
@@ -51,12 +52,12 @@ include /app/helpers/jade/mixins.jade
 .panel-details(ng-show=checkpointS3Custom)
     .details-row
         +java-class('Class name:', credentialsModel + '.Custom.className', '"checkpointS3CustomClassName" + $index', 'true', checkpointS3Custom,
-        'Custom AWS credentials provider implementation class')
+        'Custom AWS credentials provider implementation class', checkpointS3Custom)
 .settings-row
     +text('Bucket name suffix:', 'model.S3.bucketNameSuffix', '"checkpointS3BucketNameSuffix"', 'false', 'default-bucket', 'Bucket name suffix')
 .settings-row
     +java-class('Listener:', 'model.S3.checkpointListener', '"checkpointS3Listener" + $index', 'true', 'false',
-        'Checkpoint listener implementation class name')
+        'Checkpoint listener implementation class name', checkpointS3)
 +showHideLink('s3Expanded', 'client configuration')
     .details-row
         +dropdown('Protocol:', clientCfgModel + '.protocol', '"checkpointS3Protocol"', 'true', 'HTTPS', '[\
@@ -121,10 +122,10 @@ include /app/helpers/jade/mixins.jade
     .panel-details(ng-show=checkpointS3CustomRetry)
         .details-row
             +java-class('Retry condition:', clientRetryModel + '.Custom.retryCondition', '"checkpointS3CustomRetryPolicy" + $index', 'true', checkpointS3CustomRetry,
-            'Retry condition on whether a specific request and exception should be retried')
+            'Retry condition on whether a specific request and exception should be retried', checkpointS3CustomRetry)
         .details-row
             +java-class('Backoff strategy:', clientRetryModel + '.Custom.backoffStrategy', '"checkpointS3CustomBackoffStrategy" + $index', 'true', checkpointS3CustomRetry,
-            'Back-off strategy for controlling how long the next retry should wait')
+            'Back-off strategy for controlling how long the next retry should wait', checkpointS3CustomRetry)
         .details-row
             +number-required('Maximum retry attempts:', clientRetryModel + '.Custom.maxErrorRetry', '"checkpointS3CustomMaxErrorRetry"', 'true', checkpointS3CustomRetry, '-1', '1',
             'Maximum number of retry attempts for failed requests')
@@ -159,13 +160,13 @@ include /app/helpers/jade/mixins.jade
         'Maximum amount of time that an idle connection may sit in the connection pool and still be eligible for reuse')
     .details-row
         +java-class('DNS resolver:', clientCfgModel + '.dnsResolver', '"checkpointS3DnsResolver" + $index', 'true', 'false',
-        'DNS Resolver that should be used to for resolving AWS IP addresses')
+        'DNS Resolver that should be used to for resolving AWS IP addresses', checkpointS3)
     .details-row
         +number('Response metadata cache size:', clientCfgModel + '.responseMetadataCacheSize', '"checkpointS3ResponseMetadataCacheSize"', 'true', '50', '0',
         'Response metadata cache size')
     .details-row
         +java-class('SecureRandom class name:', clientCfgModel + '.secureRandom', '"checkpointS3SecureRandom" + $index', 'true', 'false',
-        'SecureRandom to be used by the SDK class name')
+        'SecureRandom to be used by the SDK class name', checkpointS3)
     .details-row
         +checkbox('Use reaper', clientCfgModel + '.useReaper', '"checkpointS3UseReaper"', 'Checks if the IdleConnectionReaper is to be started')
     .details-row
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.jade
index 31a6be7e237a2..8e77ac42922c6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.jade
@@ -21,4 +21,4 @@ include /app/helpers/jade/mixins.jade
 
 div
     .details-row
-        +java-class('Class:', model + '.class', '"collisionCustom"', 'true', required, 'CollisionSpi implementation class')
+        +java-class('Class:', model + '.class', '"collisionCustom"', 'true', required, 'CollisionSpi implementation class', required)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.jade
index d4e537aff6d7f..dbe0478344492 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.jade
@@ -37,7 +37,7 @@ div
             'Node should attempt to steal jobs from other nodes')
     .details-row
         +java-class('External listener:', model + '.externalCollisionListener', '"jsExternalCollisionListener"', 'true', 'false',
-            'Listener to be set for notification of external collision events')
+            'Listener to be set for notification of external collision events', 'backupItem.collision.kind === "JobStealing"')
     .details-row
         +ignite-form-group
             ignite-form-field-label
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.jade
index 4cfd9f52aef7e..aa99b491c0f1c 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.jade
@@ -18,8 +18,14 @@ include /app/helpers/jade/mixins.jade
 
 -var form = 'deployment'
 -var model = 'backupItem'
+-var modelDeployment = 'backupItem.deploymentSpi'
 -var exclude = model + '.peerClassLoadingLocalClassPathExclude'
 -var enabled = 'backupItem.peerClassLoadingEnabled'
+-var uriListModel = modelDeployment + '.URI.uriList'
+-var scannerModel = modelDeployment + '.URI.scanners'
+-var uriDeployment = modelDeployment + '.kind === "URI"'
+-var localDeployment = modelDeployment + '.kind === "Local"'
+-var customDeployment = modelDeployment + '.kind === "Custom"'
 
 .panel.panel-default(ng-form=form novalidate)
     .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
@@ -57,7 +63,7 @@ include /app/helpers/jade/mixins.jade
                 .settings-row
                     +number('Pool size:', model + '.peerClassLoadingThreadPoolSize', '"peerClassLoadingThreadPoolSize"', enabled, '2', '1', 'Thread pool size to use for peer class loading')
                 .settings-row
-                    +ignite-form-group(ng-model=exclude ng-form=form)
+                    +ignite-form-group
                         -var uniqueTip = 'Such package already exists'
 
                         ignite-form-field-label
@@ -81,7 +87,7 @@ include /app/helpers/jade/mixins.jade
                                             | {{ $index+1 }})
                                         +table-remove-button(exclude, 'Remove package name')
                                         span(ng-hide='field.edit')
-                                            a.labelFormField(ng-click='#{enabled} && (field.edit = true) && (#{model} = model)') {{ model }}
+                                            a.labelFormField(ng-click='(field.edit = true) && (#{model} = model)') {{ model }}
                                         span(ng-if='field.edit')
                                             +table-java-package-field(name, model, exclude, valid, save, false)
                                                 +table-save-button(valid, save, false)
@@ -107,8 +113,125 @@ include /app/helpers/jade/mixins.jade
                                         +table-save-button(valid, save, true)
                                         +unique-feedback(name, uniqueTip)
 
-
                         .group-content-empty(ng-if='!(#{exclude}.length) && !group.add.length')
                             | Not defined
+                .settings-row
+                    +dropdown('Deployment variant:', modelDeployment + '.kind', '"deploymentKind"', 'true', 'Default',
+                        '[\
+                            {value: "URI", label: "URI"},\
+                            {value: "Local", label: "Local"}, \
+                            {value: "Custom", label: "Custom"},\
+                            {value: undefined, label: "Default"}\
+                        ]',
+                        'Grid deployment SPI is in charge of deploying tasks and classes from different sources:\
+                        <ul>\
+                            <li>URI - Deploy tasks from different sources like file system folders, email and HTTP</li>\
+                            <li>Local - Only within VM deployment on local node</li>\
+                            <li>Custom - Custom implementation of DeploymentSpi</li>\
+                            <li>Default - Default configuration of LocalDeploymentSpi will be used</li>\
+                        </ul>')
+                .panel-details(ng-show=uriDeployment)
+                    .details-row
+                        +ignite-form-group()
+                            -var uniqueTip = 'Such URI already configured'
+
+                            ignite-form-field-label
+                                | URI list
+                            ignite-form-group-tooltip
+                                | List of URI which point to GAR file and which should be scanned by SPI for the new tasks
+                            ignite-form-group-add(ng-click='(group.add = [{}])')
+                                | Add URI.
+
+                            .group-content(ng-if=uriListModel + '.length')
+                                -var model = 'obj.model';
+                                -var name = '"edit" + $index'
+                                -var valid = form + '[' + name + '].$valid'
+                                -var save = uriListModel + '[$index] = ' + model
+
+                                div(ng-repeat='model in #{uriListModel} track by $index' ng-init='obj = {}')
+                                    label.col-xs-12.col-sm-12.col-md-12
+                                        .indexField
+                                            | {{ $index+1 }})
+                                        +table-remove-button(uriListModel, 'Remove URI')
+                                        span(ng-hide='field.edit')
+                                            a.labelFormField(ng-click='(field.edit = true) && (#{model} = model)') {{ model }}
+                                        span(ng-if='field.edit')
+                                            +table-url-field(name, model, uriListModel, valid, save, false)
+                                                +table-save-button(valid, save, false)
+                                                +unique-feedback(name, uniqueTip)
+
+                            .group-content(ng-repeat='field in group.add')
+                                -var model = 'new';
+                                -var name = '"new"'
+                                -var valid = form + '[' + name + '].$valid'
+                                -var save = uriListModel + '.push(' + model + ')'
+
+                                div(type='internal' name='URI')
+                                    label.col-xs-12.col-sm-12.col-md-12
+                                        +table-url-field(name, model, uriListModel, valid, save, true)
+                                            +table-save-button(valid, save, true)
+                                            +unique-feedback(name, uniqueTip)
+
+                            .group-content-empty(ng-if='!(#{uriListModel}.length) && !group.add.length')
+                                | Not defined
+                    .details-row
+                        +text('Temporary directory path:', modelDeployment + '.URI.temporaryDirectoryPath', '"DeploymentURITemporaryDirectoryPath"', 'false', 'Temporary directory path',
+                        'Absolute path to temporary directory which will be used by deployment SPI to keep all deployed classes in')
+                    .details-row
+                        +ignite-form-group()
+                            -var uniqueTip = 'Such scanner already configured'
+
+                            ignite-form-field-label
+                                | Scanner list
+                            ignite-form-group-tooltip
+                                | List of URI deployment scanners
+                            ignite-form-group-add(ng-click='(group.add = [{}])')
+                                | Add scanner
+
+                            .group-content(ng-if=scannerModel + '.length')
+                                -var model = 'obj.model';
+                                -var name = '"edit" + $index'
+                                -var valid = form + '[' + name + '].$valid'
+                                -var save = scannerModel + '[$index] = ' + model
+
+                                div(ng-repeat='model in #{scannerModel} track by $index' ng-init='obj = {}')
+                                    label.col-xs-12.col-sm-12.col-md-12
+                                        .indexField
+                                            | {{ $index+1 }})
+                                        +table-remove-button(scannerModel, 'Remove scanner')
+                                        span(ng-hide='field.edit')
+                                            a.labelFormField(ng-click='(field.edit = true) && (#{model} = model)') {{ model }}
+                                        span(ng-if='field.edit')
+                                            +table-java-class-field('Scanner:', name, model, scannerModel, valid, save, false)
+                                                +table-save-button(valid, save, false)
+                                                +unique-feedback(name, uniqueTip)
+
+                            .group-content(ng-repeat='field in group.add')
+                                -var model = 'new';
+                                -var name = '"new"'
+                                -var valid = form + '[' + name + '].$valid'
+                                -var save = scannerModel + '.push(' + model + ')'
+
+                                div(type='internal' name='Scanner')
+                                    label.col-xs-12.col-sm-12.col-md-12
+                                        // (lbl, name, model, items, valid, save, newItem)
+                                        +table-java-class-field('Scanner:', name, model, scannerModel, valid, save, true)
+                                            +table-save-button(valid, save, true)
+                                            +unique-feedback(name, uniqueTip)
+
+                            .group-content-empty(ng-if='!(#{scannerModel}.length) && !group.add.length')
+                                | Not defined
+                    .details-row
+                        +java-class('Listener:', modelDeployment + '.URI.listener', '"DeploymentURIListener"', 'true', 'false', 'Deployment event listener', uriDeployment)
+                    .details-row
+                        +checkbox('Check MD5', modelDeployment + '.URI.checkMd5', '"DeploymentURICheckMd5"', 'Exclude files with same md5s from deployment')
+                    .details-row
+                        +checkbox('Encode URI', modelDeployment + '.URI.encodeUri', '"DeploymentURIEncodeUri"', 'URI must be encoded before usage')
+                .panel-details(ng-show=localDeployment)
+                    .details-row
+                        +java-class('Listener:', modelDeployment + '.Local.listener', '"DeploymentLocalListener"', 'true', 'false', 'Deployment event listener', localDeployment)
+                .panel-details(ng-show=customDeployment)
+                    .details-row
+                        +java-class('Class:', modelDeployment + '.Custom.className', '"DeploymentCustom"', 'true', customDeployment, 'DeploymentSpi implementation class', customDeployment)
             .col-sm-6
                 +preview-xml-java(model, 'clusterDeployment')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/events.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/events.jade
index 3f2d6cb74ff12..643ea9703d230 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/events.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/events.jade
@@ -59,10 +59,10 @@ include /app/helpers/jade/mixins.jade
                     .settings-row
                         +java-class('Filter:', modelEventStorage + '.Memory.filter', '"EventStorageFilter"', 'true', 'false',
                         'Filter for events to be recorded<br/>\
-                        Should be implementation of o.a.i.lang.IgnitePredicate&lt;o.a.i.events.Event&gt;')
+                        Should be implementation of o.a.i.lang.IgnitePredicate&lt;o.a.i.events.Event&gt;', eventStorageMemory)
 
                 .settings-row(ng-show=eventStorageCustom)
-                    +java-class('Class:', modelEventStorage + '.Custom.className', '"EventStorageCustom"', 'true', eventStorageCustom, 'Event storage implementation class name')
+                    +java-class('Class:', modelEventStorage + '.Custom.className', '"EventStorageCustom"', 'true', eventStorageCustom, 'Event storage implementation class name', eventStorageCustom)
 
             .col-sm-6
                 +preview-xml-java(model, 'clusterEvents')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.jade
index aaed8e9a3b07e..16656598c2f99 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.jade
@@ -45,7 +45,7 @@ include /app/helpers/jade/mixins.jade
                         .group-content(ng-show='#{failoverSpi} && #{failoverSpi}.length > 0' ng-repeat='model in #{failoverSpi} track by $index')
                             hr(ng-if='$index != 0')
                             .settings-row
-                                +dropdown-required('Failover SPI:', 'model.kind', '"failoverKind" + $index', 'true', 'true', 'Choose Failover SPI', '[\
+                                +dropdown-required-autofocus('Failover SPI:', 'model.kind', '"failoverKind" + $index', 'true', 'true', 'Choose Failover SPI', '[\
                                         {value: "JobStealing", label: "Job stealing"},\
                                         {value: "Never", label: "Never"},\
                                         {value: "Always", label: "Always"},\
@@ -68,6 +68,6 @@ include /app/helpers/jade/mixins.jade
                                     'Maximum number of attempts to execute a failed job on another node')
                             .settings-row(ng-show=failoverCustom)
                                 +java-class('SPI implementation', 'model.Custom.class', '"failoverSpiClass" + $index', 'true', failoverCustom,
-                                    'Custom FailoverSpi implementation class name.')
+                                    'Custom FailoverSpi implementation class name.', failoverCustom)
             .col-sm-6
                 +preview-xml-java(model, 'clusterFailover')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.jade
index 2e567ed509a67..48b1776db2c35 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.jade
@@ -27,7 +27,7 @@ div
         +java-class('Curator:', model + '.curator', '"curator"', 'true', 'false',
             'The Curator framework in use<br/>\
             By default generates curator of org.apache.curator. framework.imps.CuratorFrameworkImpl\
-            class with configured connect string, retry policy, and default session and connection timeouts')
+            class with configured connect string, retry policy, and default session and connection timeouts', required)
     .details-row
         +text('Connect string:', model + '.zkConnectionString', '"' + discoveryKind + 'ConnectionString"', required, 'host:port[chroot][,host:port[chroot]]',
             'When "IGNITE_ZK_CONNECTION_STRING" system property is not configured this property will be used')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.jade
index 5a03de84fe98d..5db89f5162e55 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.jade
@@ -21,4 +21,4 @@ include /app/helpers/jade/mixins.jade
 -var required = 'backupItem.discovery.kind === "ZooKeeper" && backupItem.discovery.ZooKeeper.retryPolicy.kind === "Custom"'
 
 .details-row
-    +java-class('Class name:', retry + '.className', '"customClassName"', 'true', required, 'Custom retry policy implementation class name')
+    +java-class('Class name:', retry + '.className', '"customClassName"', 'true', required, 'Custom retry policy implementation class name', required)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.jade
index 7fd78bf266771..9fa9fc9300cf5 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.jade
@@ -46,7 +46,7 @@ include /app/helpers/jade/mixins.jade
                         .group-content(ng-show='#{loadBalancingSpi} && #{loadBalancingSpi}.length > 0' ng-repeat='model in #{loadBalancingSpi} track by $index')
                             hr(ng-if='$index != 0')
                             .settings-row
-                                +dropdown-required('Load balancing:', 'model.kind', '"loadBalancingKind" + $index', 'true', 'true', 'Choose load balancing SPI', '[\
+                                +dropdown-required-autofocus('Load balancing:', 'model.kind', '"loadBalancingKind" + $index', 'true', 'true', 'Choose load balancing SPI', '[\
                                         {value: "RoundRobin", label: "Round-robin"},\
                                         {value: "Adaptive", label: "Adaptive"},\
                                         {value: "WeightedRandom", label: "Random"},\
@@ -78,27 +78,30 @@ include /app/helpers/jade/mixins.jade
                                         <li>Default - Default load probing implementation</li>\
                                     </ul>')
                             .settings-row(ng-show='model.kind === "Adaptive" && model.Adaptive.loadProbe.kind')
-                                .panel-details
-                                    .details-row(ng-show='model.Adaptive.loadProbe.kind === "Job"')
+                                .panel-details(ng-show='model.Adaptive.loadProbe.kind === "Job"')
+                                    .details-row
                                         +checkbox('Use average', 'model.Adaptive.loadProbe.Job.useAverage', '"loadBalancingAdaptiveJobUseAverage" + $index', 'Use average CPU load vs. current')
-                                    .details-row(ng-show='model.Adaptive.loadProbe.kind === "CPU"')
+                                .panel-details(ng-show='model.Adaptive.loadProbe.kind === "CPU"')
+                                    .details-row
                                         +checkbox('Use average', 'model.Adaptive.loadProbe.CPU.useAverage', '"loadBalancingAdaptiveCPUUseAverage" + $index', 'Use average CPU load vs. current')
-                                    .details-row(ng-show='model.Adaptive.loadProbe.kind === "CPU"')
+                                    .details-row
                                         +checkbox('Use processors', 'model.Adaptive.loadProbe.CPU.useProcessors', '"loadBalancingAdaptiveCPUUseProcessors" + $index', "divide each node's CPU load by the number of processors on that node")
-                                    .details-row(ng-show='model.Adaptive.loadProbe.kind === "CPU"')
+                                    .details-row
                                         +number-min-max-step('Processor coefficient:', 'model.Adaptive.loadProbe.CPU.processorCoefficient',
                                             '"loadBalancingAdaptiveCPUProcessorCoefficient" + $index', 'true', '1', '0.001', '1', '0.05', 'Coefficient of every CPU')
-                                    .details-row(ng-show='model.Adaptive.loadProbe.kind === "ProcessingTime"')
+                                .panel-details(ng-show='model.Adaptive.loadProbe.kind === "ProcessingTime"')
+                                    .details-row
                                         +checkbox('Use average', 'model.Adaptive.loadProbe.ProcessingTime.useAverage', '"loadBalancingAdaptiveJobUseAverage" + $index', 'Use average execution time vs. current')
-                                    .details-row(ng-show=loadProbeCustom)
+                                .panel-details(ng-show=loadProbeCustom)
+                                    .details-row
                                         +java-class('Load brobe implementation:', 'model.Adaptive.loadProbe.Custom.className', '"loadBalancingAdaptiveJobUseClass" + $index', 'true', loadProbeCustom,
-                                            'Custom load balancing SPI implementation class name.')
+                                            'Custom load balancing SPI implementation class name.', loadProbeCustom)
                             .settings-row(ng-show='model.kind === "WeightedRandom"')
                                 +number('Node weight:', 'model.WeightedRandom.nodeWeight', '"loadBalancingWRNodeWeight" + $index', 'true', 10, '1', 'Weight of node')
                             .settings-row(ng-show='model.kind === "WeightedRandom"')
                                 +checkbox('Use weights', 'model.WeightedRandom.useWeights', '"loadBalancingWRUseWeights" + $index', 'Node weights should be checked when doing random load balancing')
                             .settings-row(ng-show=loadBalancingCustom)
                                 +java-class('Load balancing SPI implementation:', 'model.Custom.className', '"loadBalancingClass" + $index', 'true', loadBalancingCustom,
-                                    'Custom load balancing SPI implementation class name.')
+                                    'Custom load balancing SPI implementation class name.', loadBalancingCustom)
             .col-sm-6
                 +preview-xml-java(model, 'clusterLoadBalancing')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.jade
index 385d647568317..87d2b7d7e56b9 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.jade
@@ -22,4 +22,4 @@ include /app/helpers/jade/mixins.jade
 
 div
     .details-row
-        +java-class('Class:', model + '.class', '"customLogger"', 'true', required, 'Logger implementation class name')
+        +java-class('Class:', model + '.class', '"customLogger"', 'true', required, 'Logger implementation class name', required)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.jade
index 85ec073bf61fd..fbd979c954c05 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.jade
@@ -72,7 +72,7 @@ include /app/helpers/jade/mixins.jade
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .indexField
                                             | {{ $index+1 }})
-                                        +table-remove-conditional-button(trust, enabled, 'Remove trust manager')
+                                        +table-remove-conditional-button(trust, enabled, 'Remove trust manager', 'model')
                                         span(ng-hide='field.edit')
                                             a.labelFormField(ng-click='#{enabled} && (field.edit = true) && (#{model} = model)') {{ model }}
                                         span(ng-if='field.edit')
diff --git a/modules/web-console/frontend/app/modules/sql/scan-filter-input.service.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary-zipper.service.js
similarity index 51%
rename from modules/web-console/frontend/app/modules/sql/scan-filter-input.service.js
rename to modules/web-console/frontend/app/modules/states/configuration/summary/summary-zipper.service.js
index 18ba3baa2107f..08cfa71c43292 100644
--- a/modules/web-console/frontend/app/modules/sql/scan-filter-input.service.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary-zipper.service.js
@@ -15,37 +15,23 @@
  * limitations under the License.
  */
 
-export default class ScanFilter {
-    static $inject = ['$rootScope', '$q', '$modal'];
+import Worker from 'worker?inline=true!./summary.worker';
 
-    constructor($root, $q, $modal) {
-        this.deferred = null;
-        this.$q = $q;
+export default ['$q', function($q) {
+    return function({ cluster, data }) {
+        const defer = $q.defer();
+        const worker = new Worker();
 
-        const scope = $root.$new();
+        worker.postMessage({ cluster, data });
 
-        scope.ui = {};
-
-        scope.ok = () => {
-            this.deferred.resolve({filter: scope.ui.filter, caseSensitive: !!scope.ui.caseSensitive});
-
-            this.modal.hide();
+        worker.onmessage = (e) => {
+            defer.resolve(e.data);
         };
 
-        scope.$hide = () => {
-            this.modal.hide();
-
-            this.deferred.reject();
+        worker.onerror = (err) => {
+            defer.reject(err);
         };
 
-        this.modal = $modal({templateUrl: '/scan-filter-input.html', scope, placement: 'center', show: false});
-    }
-
-    open() {
-        this.deferred = this.$q.defer();
-
-        this.modal.$promise.then(this.modal.show);
-
-        return this.deferred.promise;
-    }
-}
+        return defer.promise;
+    };
+}];
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
index d739c4349a99b..cfc6df9c89538 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
@@ -16,15 +16,19 @@
  */
 
 import _ from 'lodash';
-import JSZip from 'jszip';
 import saver from 'file-saver';
 
+const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
+
 export default [
-    '$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteLoading', '$filter', 'IgniteConfigurationResource', 'JavaTypes', 'IgniteVersion', 'IgniteConfigurationGenerator', 'SpringTransformer', 'JavaTransformer', 'GeneratorDocker', 'GeneratorPom', 'IgnitePropertiesGenerator', 'IgniteReadmeGenerator', 'IgniteFormUtils',
-    function($root, $scope, $http, LegacyUtils, Messages, Loading, $filter, Resource, JavaTypes, Version, generator, spring, java, docker, pom, propsGenerator, readme, FormUtils) {
+    '$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteLoading', '$filter', 'IgniteConfigurationResource', 'JavaTypes', 'IgniteVersion', 'IgniteConfigurationGenerator', 'SpringTransformer', 'JavaTransformer', 'IgniteDockerGenerator', 'IgniteMavenGenerator', 'IgnitePropertiesGenerator', 'IgniteReadmeGenerator', 'IgniteFormUtils', 'IgniteSummaryZipper',
+    function($root, $scope, $http, LegacyUtils, Messages, Loading, $filter, Resource, JavaTypes, Version, generator, spring, java, docker, pom, propsGenerator, readme, FormUtils, SummaryZipper) {
         const ctrl = this;
 
-        $scope.ui = { ready: false };
+        $scope.ui = {
+            isSafari: !!(/constructor/i.test(window.HTMLElement) || window.safari),
+            ready: false
+        };
 
         Loading.start('summaryPage');
 
@@ -223,10 +227,6 @@ export default [
             return false;
         }
 
-        function escapeFileName(name) {
-            return name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
-        }
-
         $scope.selectItem = (cluster) => {
             delete ctrl.cluster;
 
@@ -297,84 +297,19 @@ export default [
 
         // TODO IGNITE-2114: implemented as independent logic for download.
         $scope.downloadConfiguration = function() {
-            const cluster = $scope.cluster;
-
-            const zip = new JSZip();
-
-            if (!ctrl.data)
-                ctrl.data = {};
-
-            if (!ctrl.data.docker)
-                ctrl.data.docker = docker.generate(cluster, 'latest');
-
-            zip.file('Dockerfile', ctrl.data.docker);
-            zip.file('.dockerignore', docker.ignoreFile());
-
-            const cfg = generator.igniteConfiguration(cluster, false);
-            const clientCfg = generator.igniteConfiguration(cluster, true);
-            const clientNearCaches = _.filter(cluster.caches, (cache) => _.get(cache, 'clientNearConfiguration.enabled'));
-
-            const secProps = propsGenerator.generate(cfg);
-
-            if (secProps)
-                zip.file('src/main/resources/secret.properties', secProps);
-
-            const srcPath = 'src/main/java';
-            const resourcesPath = 'src/main/resources';
-
-            const serverXml = `${escapeFileName(cluster.name)}-server.xml`;
-            const clientXml = `${escapeFileName(cluster.name)}-client.xml`;
-
-            const metaPath = `${resourcesPath}/META-INF`;
-
-            zip.file(`${metaPath}/${serverXml}`, spring.igniteConfiguration(cfg).asString());
-            zip.file(`${metaPath}/${clientXml}`, spring.igniteConfiguration(clientCfg, clientNearCaches).asString());
-
-            const cfgPath = `${srcPath}/config`;
-
-            zip.file(`${cfgPath}/ServerConfigurationFactory.java`, java.igniteConfiguration(cfg, 'config', 'ServerConfigurationFactory').asString());
-            zip.file(`${cfgPath}/ClientConfigurationFactory.java`, java.igniteConfiguration(clientCfg, 'config', 'ClientConfigurationFactory', clientNearCaches).asString());
-
-            if (java.isDemoConfigured(cluster, $root.IgniteDemoMode)) {
-                zip.file(`${srcPath}/demo/DemoStartup.java`, java.nodeStartup(cluster, 'demo.DemoStartup',
-                    'ServerConfigurationFactory.createConfiguration()', 'config.ServerConfigurationFactory'));
-            }
-
-            // Generate loader for caches with configured store.
-            const cachesToLoad = _.filter(cluster.caches, (cache) => _.nonNil(cache.cacheStoreFactory));
-
-            if (_.nonEmpty(cachesToLoad))
-                zip.file(`${srcPath}/load/LoadCaches.java`, java.loadCaches(cachesToLoad, 'load', 'LoadCaches', `"${clientXml}"`));
-
-            const startupPath = `${srcPath}/startup`;
-
-            zip.file(`${startupPath}/ServerNodeSpringStartup.java`, java.nodeStartup(cluster, 'startup.ServerNodeSpringStartup', `"${serverXml}"`));
-            zip.file(`${startupPath}/ClientNodeSpringStartup.java`, java.nodeStartup(cluster, 'startup.ClientNodeSpringStartup', `"${clientXml}"`));
-
-            zip.file(`${startupPath}/ServerNodeCodeStartup.java`, java.nodeStartup(cluster, 'startup.ServerNodeCodeStartup',
-                'ServerConfigurationFactory.createConfiguration()', 'config.ServerConfigurationFactory'));
-            zip.file(`${startupPath}/ClientNodeCodeStartup.java`, java.nodeStartup(cluster, 'startup.ClientNodeCodeStartup',
-                'ClientConfigurationFactory.createConfiguration()', 'config.ClientConfigurationFactory', clientNearCaches));
-
-            zip.file('pom.xml', pom.generate(cluster, Version.productVersion().ignite).asString());
-
-            zip.file('README.txt', readme.generate());
-            zip.file('jdbc-drivers/README.txt', readme.generateJDBC());
-
-            if (_.isEmpty(ctrl.data.pojos))
-                ctrl.data.pojos = java.pojos(cluster.caches);
-
-            for (const pojo of ctrl.data.pojos) {
-                if (pojo.keyClass && JavaTypes.nonBuiltInClass(pojo.keyType))
-                    zip.file(`${srcPath}/${pojo.keyType.replace(/\./g, '/')}.java`, pojo.keyClass);
+            if ($scope.isPrepareDownloading)
+                return;
 
-                zip.file(`${srcPath}/${pojo.valueType.replace(/\./g, '/')}.java`, pojo.valueClass);
-            }
+            const cluster = $scope.cluster;
 
-            $generatorOptional.optionalContent(zip, cluster);
+            $scope.isPrepareDownloading = true;
 
-            zip.generateAsync({type: 'blob', compression: 'DEFLATE', mimeType: 'application/octet-stream'})
-                .then((blob) => saver.saveAs(blob, escapeFileName(cluster.name) + '-project.zip'));
+            return new SummaryZipper({ cluster, data: ctrl.data || {}, IgniteDemoMode: $root.IgniteDemoMode })
+                .then((data) => {
+                    saver.saveAs(data, escapeFileName(cluster.name) + '-project.zip');
+                })
+                .catch((err) => Messages.showError('Failed to generate project files. ' + err.message))
+                .then(() => $scope.isPrepareDownloading = false);
         };
 
         /**
@@ -393,7 +328,7 @@ export default [
             const dialects = $scope.dialects;
 
             if (dialects.Oracle)
-                window.open('http://www.oracle.com/technetwork/apps-tech/jdbc-112010-090769.html');
+                window.open('http://www.oracle.com/technetwork/database/features/jdbc/default-2280470.html');
 
             if (dialects.DB2)
                 window.open('http://www-01.ibm.com/support/docview.wss?uid=swg21363866');
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
new file mode 100644
index 0000000000000..6b240017ab017
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import JSZip from 'jszip';
+
+import IgniteVersion from 'app/modules/configuration/Version.service';
+
+import MavenGenerator from 'app/modules/configuration/generator/Maven.service';
+import DockerGenerator from 'app/modules/configuration/generator/Docker.service';
+import ReadmeGenerator from 'app/modules/configuration/generator/Readme.service';
+import PropertiesGenerator from 'app/modules/configuration/generator/Properties.service';
+import ConfigurationGenerator from 'app/modules/configuration/generator/ConfigurationGenerator';
+
+import JavaTransformer from 'app/modules/configuration/generator/JavaTransformer.service';
+import SpringTransformer from 'app/modules/configuration/generator/SpringTransformer.service';
+
+const Version = new IgniteVersion();
+
+const maven = new MavenGenerator();
+const docker = new DockerGenerator();
+const readme = new ReadmeGenerator();
+const properties = new PropertiesGenerator();
+
+const java = new JavaTransformer[0]();
+const spring = new SpringTransformer[0]();
+
+const generator = new ConfigurationGenerator[0]();
+
+const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
+
+// eslint-disable-next-line no-undef
+onmessage = function(e) {
+    const {cluster, data, demo} = e.data;
+
+    const zip = new JSZip();
+
+    if (!data.docker)
+        data.docker = docker.generate(cluster, 'latest');
+
+    zip.file('Dockerfile', data.docker);
+    zip.file('.dockerignore', docker.ignoreFile());
+
+    const cfg = generator.igniteConfiguration(cluster, false);
+    const clientCfg = generator.igniteConfiguration(cluster, true);
+    const clientNearCaches = _.filter(cluster.caches, (cache) => _.get(cache, 'clientNearConfiguration.enabled'));
+
+    const secProps = properties.generate(cfg);
+
+    if (secProps)
+        zip.file('src/main/resources/secret.properties', secProps);
+
+    const srcPath = 'src/main/java';
+    const resourcesPath = 'src/main/resources';
+
+    const serverXml = `${escapeFileName(cluster.name)}-server.xml`;
+    const clientXml = `${escapeFileName(cluster.name)}-client.xml`;
+
+    const metaPath = `${resourcesPath}/META-INF`;
+
+    zip.file(`${metaPath}/${serverXml}`, spring.igniteConfiguration(cfg).asString());
+    zip.file(`${metaPath}/${clientXml}`, spring.igniteConfiguration(clientCfg, clientNearCaches).asString());
+
+    const cfgPath = `${srcPath}/config`;
+
+    zip.file(`${cfgPath}/ServerConfigurationFactory.java`, java.igniteConfiguration(cfg, 'config', 'ServerConfigurationFactory').asString());
+    zip.file(`${cfgPath}/ClientConfigurationFactory.java`, java.igniteConfiguration(clientCfg, 'config', 'ClientConfigurationFactory', clientNearCaches).asString());
+
+    if (java.isDemoConfigured(cluster, demo)) {
+        zip.file(`${srcPath}/demo/DemoStartup.java`, java.nodeStartup(cluster, 'demo.DemoStartup',
+            'ServerConfigurationFactory.createConfiguration()', 'config.ServerConfigurationFactory'));
+    }
+
+    // Generate loader for caches with configured store.
+    const cachesToLoad = _.filter(cluster.caches, (cache) => _.nonNil(cache.cacheStoreFactory));
+
+    if (_.nonEmpty(cachesToLoad))
+        zip.file(`${srcPath}/load/LoadCaches.java`, java.loadCaches(cachesToLoad, 'load', 'LoadCaches', `"${clientXml}"`));
+
+    const startupPath = `${srcPath}/startup`;
+
+    zip.file(`${startupPath}/ServerNodeSpringStartup.java`, java.nodeStartup(cluster, 'startup.ServerNodeSpringStartup', `"${serverXml}"`));
+    zip.file(`${startupPath}/ClientNodeSpringStartup.java`, java.nodeStartup(cluster, 'startup.ClientNodeSpringStartup', `"${clientXml}"`));
+
+    zip.file(`${startupPath}/ServerNodeCodeStartup.java`, java.nodeStartup(cluster, 'startup.ServerNodeCodeStartup',
+        'ServerConfigurationFactory.createConfiguration()', 'config.ServerConfigurationFactory'));
+    zip.file(`${startupPath}/ClientNodeCodeStartup.java`, java.nodeStartup(cluster, 'startup.ClientNodeCodeStartup',
+        'ClientConfigurationFactory.createConfiguration()', 'config.ClientConfigurationFactory', clientNearCaches));
+
+    zip.file('pom.xml', maven.generate(cluster, Version.productVersion().ignite).asString());
+
+    zip.file('README.txt', readme.generate());
+    zip.file('jdbc-drivers/README.txt', readme.generateJDBC());
+
+    if (_.isEmpty(data.pojos))
+        data.pojos = java.pojos(cluster.caches);
+
+    for (const pojo of data.pojos) {
+        if (pojo.keyClass)
+            zip.file(`${srcPath}/${pojo.keyType.replace(/\./g, '/')}.java`, pojo.keyClass);
+
+        zip.file(`${srcPath}/${pojo.valueType.replace(/\./g, '/')}.java`, pojo.valueClass);
+    }
+
+    zip.generateAsync({
+        type: 'blob',
+        compression: 'DEFLATE',
+        mimeType: 'application/octet-stream'
+    }).then((blob) => postMessage(blob));
+};
diff --git a/modules/web-console/frontend/app/modules/user/Auth.service.js b/modules/web-console/frontend/app/modules/user/Auth.service.js
index 43e2f9216071e..e0f905da8bf8b 100644
--- a/modules/web-console/frontend/app/modules/user/Auth.service.js
+++ b/modules/web-console/frontend/app/modules/user/Auth.service.js
@@ -20,12 +20,11 @@ export default ['Auth', ['$http', '$rootScope', '$state', '$window', 'IgniteErro
         return {
             forgotPassword(userInfo) {
                 $http.post('/api/v1/password/forgot', userInfo)
-                    .success(() => $state.go('password.send'))
-                    .error((err) => ErrorPopover.show('forgot_email', Messages.errorMessage(null, err)));
+                    .then(() => $state.go('password.send'))
+                    .cacth(({data}) => ErrorPopover.show('forgot_email', Messages.errorMessage(null, data)));
             },
             auth(action, userInfo) {
                 $http.post('/api/v1/' + action, userInfo)
-                    .catch(({data}) => Promise.reject(data))
                     .then(() => {
                         if (action === 'password/forgot')
                             return;
@@ -41,16 +40,16 @@ export default ['Auth', ['$http', '$rootScope', '$state', '$window', 'IgniteErro
                                 $root.gettingStarted.tryShow();
                             });
                     })
-                    .catch((err) => ErrorPopover.show(action + '_email', Messages.errorMessage(null, err)));
+                    .catch((res) => ErrorPopover.show(action + '_email', Messages.errorMessage(null, res)));
             },
             logout() {
                 $http.post('/api/v1/logout')
-                    .success(() => {
+                    .then(() => {
                         User.clean();
 
                         $window.open($state.href('signin'), '_self');
                     })
-                    .error(Messages.showError);
+                    .catch(Messages.showError);
             }
         };
     }]];
diff --git a/modules/web-console/frontend/app/services/JavaTypes.service.js b/modules/web-console/frontend/app/services/JavaTypes.service.js
index 679914f0c5b49..944fea5aa893a 100644
--- a/modules/web-console/frontend/app/services/JavaTypes.service.js
+++ b/modules/web-console/frontend/app/services/JavaTypes.service.js
@@ -40,7 +40,7 @@ const VALID_UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-
  * Utility service for various check on java types.
  */
 export default class JavaTypes {
-    static $inject = ['igniteClusterDefaults', 'igniteCacheDefaults', 'igniteIgfsDefaults'];
+    static $inject = ['IgniteClusterDefaults', 'IgniteCacheDefaults', 'IgniteIGFSDefaults'];
 
     constructor(clusterDflts, cacheDflts, igfsDflts) {
         this.enumClasses = _.uniq(this._enumClassesAcc(_.merge(clusterDflts, cacheDflts, igfsDflts), []));
@@ -101,14 +101,9 @@ export default class JavaTypes {
      * @return {String} Class name.
      */
     shortClassName(clsName) {
-        if (this.isJavaPrimitive(clsName))
-            return clsName;
+        const dotIdx = clsName.lastIndexOf('.');
 
-        const fullClsName = this.fullClassName(clsName);
-
-        const dotIdx = fullClsName.lastIndexOf('.');
-
-        return dotIdx > 0 ? fullClsName.substr(dotIdx + 1) : fullClsName;
+        return dotIdx > 0 ? clsName.substr(dotIdx + 1) : clsName;
     }
 
     /**
@@ -163,7 +158,7 @@ export default class JavaTypes {
      * @param {String} clsName Class name to check.
      * @returns {boolean} 'true' if given class name is java primitive.
      */
-    isJavaPrimitive(clsName) {
+    isPrimitive(clsName) {
         return _.includes(JAVA_PRIMITIVES, clsName);
     }
 
diff --git a/modules/web-console/frontend/app/services/Messages.service.js b/modules/web-console/frontend/app/services/Messages.service.js
index e679488e8066b..fefdae9a4378c 100644
--- a/modules/web-console/frontend/app/services/Messages.service.js
+++ b/modules/web-console/frontend/app/services/Messages.service.js
@@ -24,6 +24,9 @@ export default ['IgniteMessages', ['$alert', ($alert) => {
         prefix = prefix || '';
 
         if (err) {
+            if (err.hasOwnProperty('data'))
+                err = err.data;
+
             if (err.hasOwnProperty('message'))
                 return prefix + err.message;
 
@@ -38,26 +41,26 @@ export default ['IgniteMessages', ['$alert', ($alert) => {
             msgModal.hide();
     };
 
-    const _showMessage = (err, type, duration, icon) => {
+    const _showMessage = (message, err, type, duration) => {
         hideAlert();
 
-        const title = errorMessage(null, err);
+        const title = err ? errorMessage(message, err) : errorMessage(null, message);
 
         msgModal = $alert({type, title, duration});
 
-        msgModal.$scope.icon = icon;
+        msgModal.$scope.icon = `icon-${type}`;
     };
 
     return {
         errorMessage,
         hideAlert,
-        showError(err) {
-            _showMessage(err, 'danger', 10, 'fa-exclamation-triangle');
+        showError(message, err) {
+            _showMessage(message, err, 'danger', 10);
 
             return false;
         },
-        showInfo(err) {
-            _showMessage(err, 'success', 3, 'fa-check-circle-o');
+        showInfo(message) {
+            _showMessage(message, null, 'success', 3);
         }
     };
 }]];
diff --git a/modules/web-console/frontend/controllers/admin-controller.js b/modules/web-console/frontend/controllers/admin-controller.js
index 70043016bff1d..cf7fd71adeec0 100644
--- a/modules/web-console/frontend/controllers/admin-controller.js
+++ b/modules/web-console/frontend/controllers/admin-controller.js
@@ -15,79 +15,220 @@
  * limitations under the License.
  */
 
+const ICON_SORT = '<span ui-grid-one-bind-id-grid="col.uid + \'-sortdir-text\'" ui-grid-visible="col.sort.direction" aria-label="Sort Descending"><i ng-class="{ \'ui-grid-icon-up-dir\': col.sort.direction == asc, \'ui-grid-icon-down-dir\': col.sort.direction == desc, \'ui-grid-icon-blank\': !col.sort.direction }" title="" aria-hidden="true"></i></span>';
+
+const CLUSTER_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-sitemap'></i>${ICON_SORT}</div>`;
+const MODEL_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-object-group'></i>${ICON_SORT}</div>`;
+const CACHE_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-database'></i>${ICON_SORT}</div>`;
+const IGFS_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-folder-o'></i>${ICON_SORT}</div>`;
+
+const ACTIONS_TEMPLATE = `
+<div class='text-center ui-grid-cell-actions'>
+    <a class='btn btn-default dropdown-toggle' bs-dropdown='' ng-show='row.entity._id != $root.user._id' data-placement='bottom-right' data-container='.panel'>
+        <i class='fa fa-gear'></i>&nbsp;
+        <span class='caret'></span>
+    </a>
+    <ul class='dropdown-menu' role='menu'>
+        <li>
+            <a ng-click='grid.api.becomeUser(row.entity)'>Become this user</a>
+        </li>
+        <li>
+            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='row.entity.admin && row.entity._id !== $root.user._id'>Revoke admin</a>
+            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='!row.entity.admin && row.entity._id !== $root.user._id'>Grant admin</a>
+        </li>
+        <li>
+            <a ng-click='grid.api.removeUser(row.entity)'>Remove user</a>
+        </li>
+</div>`;
+
+const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
+
 // Controller for Admin screen.
 export default ['adminController', [
-    '$rootScope', '$scope', '$http', '$q', '$state', 'IgniteMessages', 'IgniteConfirm', 'User', 'IgniteNotebookData', 'IgniteCountries',
-    ($rootScope, $scope, $http, $q, $state, Messages, Confirm, User, Notebook, Countries) => {
+    '$rootScope', '$scope', '$http', '$q', '$state', '$filter', 'uiGridConstants', 'IgniteMessages', 'IgniteConfirm', 'User', 'IgniteNotebookData', 'IgniteCountries',
+    ($rootScope, $scope, $http, $q, $state, $filter, uiGridConstants, Messages, Confirm, User, Notebook, Countries) => {
         $scope.users = null;
 
-        const _reloadUsers = () => {
-            $http.post('/api/v1/admin/list')
-                .success((users) => {
-                    $scope.users = users;
+        const companySelectOptions = [];
+        const countrySelectOptions = [];
 
-                    _.forEach($scope.users, (user) => {
-                        user.userName = user.firstName + ' ' + user.lastName;
-                        user.countryCode = Countries.getByName(user.country).code;
-                        user.label = user.userName + ' ' + user.email + ' ' +
-                            (user.company || '') + ' ' + (user.countryCode || '');
-                    });
-                })
-                .error(Messages.showError);
-        };
+        const COLUMNS_DEFS = [
+            {displayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'test', minWidth: 80, width: 80, enableFiltering: false, enableSorting: false},
+            {displayName: 'User', field: 'userName', minWidth: 65, enableFiltering: true, filter: { placeholder: 'Filter by name...' }},
+            {displayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
+            {displayName: 'Company', field: 'company', minWidth: 160, filter: {
+                selectOptions: companySelectOptions, type: uiGridConstants.filter.SELECT, condition: uiGridConstants.filter.EXACT }
+            },
+            {displayName: 'Country', field: 'countryCode', minWidth: 80, filter: {
+                selectOptions: countrySelectOptions, type: uiGridConstants.filter.SELECT, condition: uiGridConstants.filter.EXACT }
+            },
+            {displayName: 'Last login', field: 'lastLogin', cellFilter: 'date:"medium"', minWidth: 175, width: 175, enableFiltering: false, sort: { direction: 'desc', priority: 0 }},
+            {displayName: 'Clusters count', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: '_clusters', type: 'number', headerTooltip: 'Clusters count', minWidth: 50, width: 50, enableFiltering: false},
+            {displayName: 'Models count', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: '_models', type: 'number', headerTooltip: 'Models count', minWidth: 50, width: 50, enableFiltering: false},
+            {displayName: 'Caches count', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: '_caches', type: 'number', headerTooltip: 'Caches count', minWidth: 50, width: 50, enableFiltering: false},
+            {displayName: 'IGFS count', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: '_igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 50, width: 50, enableFiltering: false}
+        ];
 
-        _reloadUsers();
+        const ctrl = $scope.ctrl = {};
 
-        $scope.becomeUser = function(user) {
+        const becomeUser = function(user) {
             $http.get('/api/v1/admin/become', { params: {viewedUserId: user._id}})
-                .catch(({data}) => Promise.reject(data))
                 .then(() => User.load())
-                .then((becomeUser) => {
-                    $rootScope.$broadcast('user', becomeUser);
-
-                    $state.go('base.configuration.clusters');
-                })
+                .then(() => $state.go('base.configuration.clusters'))
                 .then(() => Notebook.load())
                 .catch(Messages.showError);
         };
 
-        $scope.removeUser = (user) => {
-            Confirm.confirm('Are you sure you want to remove user: "' + user.userName + '"?')
+        const removeUser = (user) => {
+            Confirm.confirm(`Are you sure you want to remove user: "${user.userName}"?`)
                 .then(() => {
                     $http.post('/api/v1/admin/remove', {userId: user._id})
-                        .success(() => {
+                        .then(() => {
                             const i = _.findIndex($scope.users, (u) => u._id === user._id);
 
                             if (i >= 0)
                                 $scope.users.splice(i, 1);
 
-                            Messages.showInfo('User has been removed: "' + user.userName + '"');
+                            Messages.showInfo(`User has been removed: "${user.userName}"`);
                         })
-                        .error((err, status) => {
+                        .catch(({data, status}) => {
                             if (status === 503)
-                                Messages.showInfo(err);
+                                Messages.showInfo(data);
                             else
-                                Messages.showError(Messages.errorMessage('Failed to remove user: ', err));
+                                Messages.showError('Failed to remove user: ', data);
                         });
                 });
         };
 
-        $scope.toggleAdmin = (user) => {
+        const toggleAdmin = (user) => {
             if (user.adminChanging)
                 return;
 
             user.adminChanging = true;
 
             $http.post('/api/v1/admin/save', {userId: user._id, adminFlag: !user.admin})
-                .success(() => {
+                .then(() => {
                     user.admin = !user.admin;
 
-                    Messages.showInfo('Admin right was successfully toggled for user: "' + user.userName + '"');
+                    Messages.showInfo(`Admin right was successfully toggled for user: "${user.userName}"`);
                 })
-                .error((err) => {
-                    Messages.showError(Messages.errorMessage('Failed to toggle admin right for user: ', err));
+                .catch((res) => {
+                    Messages.showError('Failed to toggle admin right for user: ', res);
                 })
                 .finally(() => user.adminChanging = false);
         };
+
+
+        ctrl.gridOptions = {
+            data: [],
+            columnVirtualizationThreshold: 30,
+            columnDefs: COLUMNS_DEFS,
+            categories: [
+                {name: 'Actions', visible: true, selectable: true},
+                {name: 'User', visible: true, selectable: true},
+                {name: 'Email', visible: true, selectable: true},
+                {name: 'Company', visible: true, selectable: true},
+                {name: 'Country', visible: true, selectable: true},
+                {name: 'Last login', visible: true, selectable: true},
+
+                {name: 'Clusters count', visible: true, selectable: true},
+                {name: 'Models count', visible: true, selectable: true},
+                {name: 'Caches count', visible: true, selectable: true},
+                {name: 'IGFS count', visible: true, selectable: true}
+            ],
+            enableFiltering: true,
+            enableRowSelection: false,
+            enableRowHeaderSelection: false,
+            enableColumnMenus: false,
+            multiSelect: false,
+            modifierKeysToMultiSelect: true,
+            noUnselect: true,
+            flatEntityAccess: true,
+            fastWatch: true,
+            onRegisterApi: (api) => {
+                ctrl.gridApi = api;
+
+                api.becomeUser = becomeUser;
+                api.removeUser = removeUser;
+                api.toggleAdmin = toggleAdmin;
+            }
+        };
+
+        /**
+         * Set grid height.
+         *
+         * @param {Number} rows Rows count.
+         * @private
+         */
+        const adjustHeight = (rows) => {
+            const height = Math.min(rows, 20) * 30 + 75;
+
+            // Remove header height.
+            ctrl.gridApi.grid.element.css('height', height + 'px');
+
+            ctrl.gridApi.core.handleWindowResize();
+        };
+
+        const usersToFilterOptions = (column) => {
+            return _.sortBy(
+                _.map(
+                    _.groupBy($scope.users, (usr) => {
+                        const fld = usr[column];
+
+                        return _.isNil(fld) ? fld : fld.toUpperCase();
+                    }),
+                    (arr, value) => ({label: `${_.head(arr)[column] || 'Not set'} (${arr.length})`, value})
+                ),
+                'value');
+        };
+
+        const _reloadUsers = () => {
+            $http.post('/api/v1/admin/list')
+                .then(({ data }) => {
+                    $scope.users = data;
+
+                    companySelectOptions.length = 0;
+                    countrySelectOptions.length = 0;
+
+                    _.forEach($scope.users, (user) => {
+                        user.userName = user.firstName + ' ' + user.lastName;
+                        user.countryCode = Countries.getByName(user.country).code;
+
+                        user._clusters = user.counters.clusters;
+                        user._models = user.counters.models;
+                        user._caches = user.counters.caches;
+                        user._igfs = user.counters.igfs;
+                    });
+
+                    companySelectOptions.push(...usersToFilterOptions('company'));
+                    countrySelectOptions.push(...usersToFilterOptions('countryCode'));
+
+                    $scope.ctrl.gridOptions.data = data;
+
+                    adjustHeight(data.length);
+                })
+                .catch(Messages.showError);
+        };
+
+        _reloadUsers();
+
+        const _enableColumns = (categories, visible) => {
+            _.forEach(categories, (cat) => {
+                cat.visible = visible;
+
+                _.forEach(ctrl.gridOptions.columnDefs, (col) => {
+                    if (col.displayName === cat.name)
+                        col.visible = visible;
+                });
+            });
+
+            ctrl.gridApi.grid.refresh();
+        };
+
+        const _selectableColumns = () => _.filter(ctrl.gridOptions.categories, (cat) => cat.selectable);
+
+        ctrl.toggleColumns = (category, visible) => _enableColumns([category], visible);
+        ctrl.selectAllColumns = () => _enableColumns(_selectableColumns(), true);
+        ctrl.clearAllColumns = () => _enableColumns(_selectableColumns(), false);
     }
 ]];
diff --git a/modules/web-console/frontend/controllers/caches-controller.js b/modules/web-console/frontend/controllers/caches-controller.js
index 8c01173ae509c..e7521b55ad403 100644
--- a/modules/web-console/frontend/controllers/caches-controller.js
+++ b/modules/web-console/frontend/controllers/caches-controller.js
@@ -467,14 +467,14 @@ export default ['cachesController', [
         // Save cache in database.
         function save(item) {
             $http.post('/api/v1/configuration/caches/save', item)
-                .success(function(_id) {
+                .then(({data}) => {
+                    const _id = data;
+
                     item.label = _cacheLbl(item);
 
                     $scope.ui.inputForm.$setPristine();
 
-                    const idx = _.findIndex($scope.caches, function(cache) {
-                        return cache._id === _id;
-                    });
+                    const idx = _.findIndex($scope.caches, {_id});
 
                     if (idx >= 0)
                         _.assign($scope.caches[idx], item);
@@ -487,21 +487,21 @@ export default ['cachesController', [
                         if (_.includes(item.clusters, cluster.value))
                             cluster.caches = _.union(cluster.caches, [_id]);
                         else
-                            _.remove(cluster.caches, (id) => id === _id);
+                            _.pull(cluster.caches, _id);
                     });
 
                     _.forEach($scope.domains, (domain) => {
                         if (_.includes(item.domains, domain.value))
                             domain.meta.caches = _.union(domain.meta.caches, [_id]);
                         else
-                            _.remove(domain.meta.caches, (id) => id === _id);
+                            _.pull(domain.meta.caches, _id);
                     });
 
                     $scope.selectItem(item);
 
                     Messages.showInfo('Cache "' + item.name + '" saved.');
                 })
-                .error(Messages.showError);
+                .catch(Messages.showError);
         }
 
         // Save cache.
@@ -559,7 +559,7 @@ export default ['cachesController', [
                     const _id = selectedItem._id;
 
                     $http.post('/api/v1/configuration/caches/remove', {_id})
-                        .success(function() {
+                        .then(() => {
                             Messages.showInfo('Cache has been removed: ' + selectedItem.name);
 
                             const caches = $scope.caches;
@@ -582,7 +582,7 @@ export default ['cachesController', [
                                 _.forEach($scope.domains, (domain) => _.remove(domain.meta.caches, (id) => id === _id));
                             }
                         })
-                        .error(Messages.showError);
+                        .catch(Messages.showError);
                 });
         };
 
@@ -591,7 +591,7 @@ export default ['cachesController', [
             Confirm.confirm('Are you sure you want to remove all caches?')
                 .then(function() {
                     $http.post('/api/v1/configuration/caches/remove/all')
-                        .success(function() {
+                        .then(() => {
                             Messages.showInfo('All caches have been removed');
 
                             $scope.caches = [];
@@ -603,7 +603,7 @@ export default ['cachesController', [
                             $scope.ui.inputForm.$error = {};
                             $scope.ui.inputForm.$setPristine();
                         })
-                        .error(Messages.showError);
+                        .catch(Messages.showError);
                 });
         };
 
diff --git a/modules/web-console/frontend/controllers/clusters-controller.js b/modules/web-console/frontend/controllers/clusters-controller.js
index f92a2f1e100fa..7f90b90e198fb 100644
--- a/modules/web-console/frontend/controllers/clusters-controller.js
+++ b/modules/web-console/frontend/controllers/clusters-controller.js
@@ -17,7 +17,7 @@
 
 // Controller for Clusters screen.
 export default ['clustersController', [
-    '$rootScope', '$scope', '$http', '$state', '$timeout', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'igniteEventGroups', 'DemoInfo', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils',
+    '$rootScope', '$scope', '$http', '$state', '$timeout', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteEventGroups', 'DemoInfo', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils',
     function($root, $scope, $http, $state, $timeout, LegacyUtils, Messages, Confirm, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, igniteEventGroups, DemoInfo, LegacyTable, Resource, ErrorPopover, FormUtils) {
         UnsavedChangesGuard.install($scope);
 
@@ -31,6 +31,12 @@ export default ['clustersController', [
             cacheKeyConfiguration: [],
             communication: {},
             connector: {},
+            deploymentSpi: {
+                URI: {
+                    uriList: [],
+                    scanners: []
+                }
+            },
             discovery: {
                 Cloud: {
                     regions: [],
@@ -38,6 +44,7 @@ export default ['clustersController', [
                 }
             },
             marshaller: {},
+            peerClassLoadingLocalClassPathExclude: [],
             sslContextFactory: {
                 trustManagers: []
             },
@@ -276,6 +283,16 @@ export default ['clustersController', [
 
                     if (!cluster.eventStorage)
                         cluster.eventStorage = { kind: 'Memory' };
+
+                    if (!cluster.peerClassLoadingLocalClassPathExclude)
+                        cluster.peerClassLoadingLocalClassPathExclude = [];
+
+                    if (!cluster.deploymentSpi) {
+                        cluster.deploymentSpi = {URI: {
+                            uriList: [],
+                            scanners: []
+                        }};
+                    }
                 });
 
                 if ($state.params.linkId)
@@ -699,17 +716,20 @@ export default ['clustersController', [
         // Save cluster in database.
         function save(item) {
             $http.post('/api/v1/configuration/clusters/save', item)
-                .success(function(_id) {
+                .then(({data}) => {
+                    const _id = data;
+
                     item.label = _clusterLbl(item);
 
                     $scope.ui.inputForm.$setPristine();
 
-                    const idx = _.findIndex($scope.clusters, (cluster) => cluster._id === _id);
+                    const idx = _.findIndex($scope.clusters, {_id});
 
                     if (idx >= 0)
                         _.assign($scope.clusters[idx], item);
                     else {
                         item._id = _id;
+
                         $scope.clusters.push(item);
                     }
 
@@ -717,21 +737,21 @@ export default ['clustersController', [
                         if (_.includes(item.caches, cache.value))
                             cache.cache.clusters = _.union(cache.cache.clusters, [_id]);
                         else
-                            _.remove(cache.cache.clusters, (id) => id === _id);
+                            _.pull(cache.cache.clusters, _id);
                     });
 
                     _.forEach($scope.igfss, (igfs) => {
                         if (_.includes(item.igfss, igfs.value))
                             igfs.igfs.clusters = _.union(igfs.igfs.clusters, [_id]);
                         else
-                            _.remove(igfs.igfs.clusters, (id) => id === _id);
+                            _.pull(igfs.igfs.clusters, _id);
                     });
 
                     $scope.selectItem(item);
 
-                    Messages.showInfo('Cluster "' + item.name + '" saved.');
+                    Messages.showInfo(`Cluster "${item.name}" saved.`);
                 })
-                .error(Messages.showError);
+                .catch(Messages.showError);
         }
 
         // Save cluster.
@@ -774,7 +794,7 @@ export default ['clustersController', [
                     const _id = selectedItem._id;
 
                     $http.post('/api/v1/configuration/clusters/remove', {_id})
-                        .success(function() {
+                        .then(() => {
                             Messages.showInfo('Cluster has been removed: ' + selectedItem.name);
 
                             const clusters = $scope.clusters;
@@ -795,7 +815,7 @@ export default ['clustersController', [
                                 _.forEach($scope.igfss, (igfs) => _.remove(igfs.igfs.clusters, (id) => id === _id));
                             }
                         })
-                        .error(Messages.showError);
+                        .catch(Messages.showError);
                 });
         };
 
@@ -804,7 +824,7 @@ export default ['clustersController', [
             Confirm.confirm('Are you sure you want to remove all clusters?')
                 .then(function() {
                     $http.post('/api/v1/configuration/clusters/remove/all')
-                        .success(() => {
+                        .then(() => {
                             Messages.showInfo('All clusters have been removed');
 
                             $scope.clusters = [];
@@ -816,7 +836,7 @@ export default ['clustersController', [
                             $scope.ui.inputForm.$error = {};
                             $scope.ui.inputForm.$setPristine();
                         })
-                        .error(Messages.showError);
+                        .catch(Messages.showError);
                 });
         };
 
diff --git a/modules/web-console/frontend/controllers/domains-controller.js b/modules/web-console/frontend/controllers/domains-controller.js
index 2d7b875e36f5b..303110e086696 100644
--- a/modules/web-console/frontend/controllers/domains-controller.js
+++ b/modules/web-console/frontend/controllers/domains-controller.js
@@ -756,15 +756,15 @@ export default ['domainsController', [
                 Loading.start('importDomainFromDb');
 
                 $http.post('/api/v1/configuration/domains/save/batch', batch)
-                    .success(function(savedBatch) {
+                    .then(({data}) => {
                         let lastItem;
                         const newItems = [];
 
-                        _.forEach(_mapCaches(savedBatch.generatedCaches), function(cache) {
+                        _.forEach(_mapCaches(data.generatedCaches), function(cache) {
                             $scope.caches.push(cache);
                         });
 
-                        _.forEach(savedBatch.savedDomains, function(savedItem) {
+                        _.forEach(data.savedDomains, function(savedItem) {
                             const idx = _.findIndex($scope.domains, function(domain) {
                                 return domain._id === savedItem._id;
                             });
@@ -792,7 +792,7 @@ export default ['domainsController', [
 
                         $scope.ui.showValid = true;
                     })
-                    .error(Messages.showError)
+                    .catch(Messages.showError)
                     .finally(() => {
                         Loading.finish('importDomainFromDb');
 
@@ -1382,10 +1382,10 @@ export default ['domainsController', [
                 item.kind = 'store';
 
             $http.post('/api/v1/configuration/domains/save', item)
-                .success(function(res) {
+                .then(({data}) => {
                     $scope.ui.inputForm.$setPristine();
 
-                    const savedMeta = res.savedDomains[0];
+                    const savedMeta = data.savedDomains[0];
 
                     const idx = _.findIndex($scope.domains, function(domain) {
                         return domain._id === savedMeta._id;
@@ -1400,16 +1400,16 @@ export default ['domainsController', [
                         if (_.includes(item.caches, cache.value))
                             cache.cache.domains = _.union(cache.cache.domains, [savedMeta._id]);
                         else
-                            _.remove(cache.cache.domains, (id) => id === savedMeta._id);
+                            _.pull(cache.cache.domains, savedMeta._id);
                     });
 
                     $scope.selectItem(savedMeta);
 
-                    Messages.showInfo('Domain model "' + item.valueType + '" saved.');
+                    Messages.showInfo(`Domain model "${item.valueType}" saved.`);
 
                     _checkShowValidPresentation();
                 })
-                .error(Messages.showError);
+                .catch(Messages.showError);
         }
 
         // Save domain model.
@@ -1469,14 +1469,12 @@ export default ['domainsController', [
                     const _id = selectedItem._id;
 
                     $http.post('/api/v1/configuration/domains/remove', {_id})
-                        .success(function() {
+                        .then(() => {
                             Messages.showInfo('Domain model has been removed: ' + selectedItem.valueType);
 
                             const domains = $scope.domains;
 
-                            const idx = _.findIndex(domains, function(domain) {
-                                return domain._id === _id;
-                            });
+                            const idx = _.findIndex(domains, {_id});
 
                             if (idx >= 0) {
                                 domains.splice(idx, 1);
@@ -1488,12 +1486,12 @@ export default ['domainsController', [
                                 else
                                     $scope.backupItem = emptyDomain;
 
-                                _.forEach($scope.caches, (cache) => _.remove(cache.cache.domains, (id) => id === _id));
+                                _.forEach($scope.caches, (cache) => _.pull(cache.cache.domains, _id));
                             }
 
                             _checkShowValidPresentation();
                         })
-                        .error(Messages.showError);
+                        .catch(Messages.showError);
                 });
         };
 
@@ -1504,7 +1502,7 @@ export default ['domainsController', [
             Confirm.confirm('Are you sure you want to remove all domain models?')
                 .then(function() {
                     $http.post('/api/v1/configuration/domains/remove/all')
-                        .success(function() {
+                        .then(() => {
                             Messages.showInfo('All domain models have been removed');
 
                             $scope.domains = [];
@@ -1516,7 +1514,7 @@ export default ['domainsController', [
                             $scope.ui.inputForm.$error = {};
                             $scope.ui.inputForm.$setPristine();
                         })
-                        .error(Messages.showError);
+                        .catch(Messages.showError);
                 });
         };
 
diff --git a/modules/web-console/frontend/controllers/igfs-controller.js b/modules/web-console/frontend/controllers/igfs-controller.js
index e505f1c450948..b3c6043ce32b6 100644
--- a/modules/web-console/frontend/controllers/igfs-controller.js
+++ b/modules/web-console/frontend/controllers/igfs-controller.js
@@ -296,12 +296,12 @@ export default ['igfsController', [
         // Save IGFS in database.
         function save(item) {
             $http.post('/api/v1/configuration/igfs/save', item)
-                .success(function(_id) {
+                .then(({data}) => {
+                    const _id = data;
+
                     $scope.ui.inputForm.$setPristine();
 
-                    const idx = _.findIndex($scope.igfss, function(igfs) {
-                        return igfs._id === _id;
-                    });
+                    const idx = _.findIndex($scope.igfss, {_id});
 
                     if (idx >= 0)
                         _.assign($scope.igfss[idx], item);
@@ -312,9 +312,9 @@ export default ['igfsController', [
 
                     $scope.selectItem(item);
 
-                    Messages.showInfo('IGFS "' + item.name + '" saved.');
+                    Messages.showInfo(`IGFS "${item.name}" saved.`);
                 })
-                .error(Messages.showError);
+                .catch(Messages.showError);
         }
 
         // Save IGFS.
@@ -359,7 +359,7 @@ export default ['igfsController', [
                     const _id = selectedItem._id;
 
                     $http.post('/api/v1/configuration/igfs/remove', {_id})
-                        .success(function() {
+                        .then(() => {
                             Messages.showInfo('IGFS has been removed: ' + selectedItem.name);
 
                             const igfss = $scope.igfss;
@@ -379,7 +379,7 @@ export default ['igfsController', [
                                     $scope.backupItem = emptyIgfs;
                             }
                         })
-                        .error(Messages.showError);
+                        .catch(Messages.showError);
                 });
         };
 
@@ -390,7 +390,7 @@ export default ['igfsController', [
             Confirm.confirm('Are you sure you want to remove all IGFS?')
                 .then(function() {
                     $http.post('/api/v1/configuration/igfs/remove/all')
-                        .success(function() {
+                        .then(() => {
                             Messages.showInfo('All IGFS have been removed');
 
                             $scope.igfss = [];
@@ -398,7 +398,7 @@ export default ['igfsController', [
                             $scope.ui.inputForm.$error = {};
                             $scope.ui.inputForm.$setPristine();
                         })
-                        .error(Messages.showError);
+                        .catch(Messages.showError);
                 });
         };
 
diff --git a/modules/web-console/frontend/controllers/profile-controller.js b/modules/web-console/frontend/controllers/profile-controller.js
index fd595d9df8c66..87a880596c4bd 100644
--- a/modules/web-console/frontend/controllers/profile-controller.js
+++ b/modules/web-console/frontend/controllers/profile-controller.js
@@ -74,7 +74,6 @@ export default ['profileController', [
 
         $scope.saveUser = () => {
             $http.post('/api/v1/profile/save', $scope.user)
-                .catch(({data}) => Promise.reject(data))
                 .then(User.load)
                 .then(() => {
                     if ($scope.expandedPassword)
@@ -89,7 +88,7 @@ export default ['profileController', [
 
                     $root.$broadcast('user', $scope.user);
                 })
-                .catch((err) => Messages.showError(Messages.errorMessage('Failed to save profile: ', err)));
+                .catch((res) => Messages.showError('Failed to save profile: ', res));
         };
     }
 ]];
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js
index 2463d246a4446..7360ac4c0da3c 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js
@@ -20,7 +20,7 @@ import fs from 'fs';
 import webpack from 'webpack';
 import autoprefixer from 'autoprefixer-core';
 import jade from 'jade';
-import progressPlugin from './plugins/progress';
+import ProgressBarPlugin from 'progress-bar-webpack-plugin';
 import eslintFormatter from 'eslint-friendly-formatter';
 
 import ExtractTextPlugin from 'extract-text-webpack-plugin';
@@ -61,7 +61,6 @@ export default () => {
         // Output system.
         output: {
             path: destDir,
-            publicPath: './',
             filename: '[name].js'
         },
 
@@ -111,8 +110,10 @@ export default () => {
                     loader: 'babel-loader',
                     query: {
                         cacheDirectory: true,
-                        plugins: ['transform-runtime',
-                            'add-module-exports'],
+                        plugins: [
+                            'transform-runtime',
+                            'add-module-exports'
+                        ],
                         presets: ['angular']
 
                     }
@@ -126,10 +127,8 @@ export default () => {
                     loader: development ? `style-loader!${stylesLoader}` : ExtractTextPlugin.extract('style-loader', stylesLoader)
                 },
                 {
-                    test: /\.(woff2|woff|ttf|eot|svg)?(\?v=[0-9]\.[0-9]\.[0-9])?$/,
-                    loaders: [
-                        `${assetsLoader}?name=assets/fonts/[name].[ext]`
-                    ]
+                    test: /\.(ttf|eot|svg|woff(2)?)(\?v=[\d.]+)?(\?[a-z0-9#-]+)?$/,
+                    loaders: [`${assetsLoader}?name=assets/fonts/[name].[ext]`]
                 },
                 {
                     test: /\.(jpe?g|png|gif)$/i,
@@ -186,7 +185,7 @@ export default () => {
                 },
                 favicon
             }),
-            progressPlugin
+            new ProgressBarPlugin()
         ]
     };
 };
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js
index cad913354d7f0..34e1f6ae858b0 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js
@@ -20,9 +20,8 @@ import webpack from 'webpack';
 
 import {destDir, rootDir, srcDir} from '../../paths';
 
-const devServerHost = 'localhost';
+const backendPort = 3000;
 const devServerPort = 9000;
-const devServerUrl = `http://${devServerHost}:${devServerPort}/`;
 
 export default () => {
     const plugins = [
@@ -31,11 +30,10 @@ export default () => {
 
     return {
         entry: {
-            webpack: `webpack-dev-server/client?${devServerUrl}`,
             app: [path.join(srcDir, 'app.js'), 'webpack/hot/only-dev-server']
         },
         output: {
-            publicPath: devServerUrl
+            publicPath: `http://localhost:${devServerPort}/`
         },
         context: rootDir,
         debug: true,
@@ -44,24 +42,22 @@ export default () => {
         devServer: {
             compress: true,
             historyApiFallback: true,
-            publicPath: '/',
             contentBase: destDir,
-            info: true,
             hot: true,
             inline: true,
             proxy: {
                 '/socket.io': {
-                    target: 'http://localhost:3000',
+                    target: `http://localhost:${backendPort}`,
                     changeOrigin: true,
                     ws: true
                 },
                 '/agents': {
-                    target: 'http://localhost:3000',
+                    target: `http://localhost:${backendPort}`,
                     changeOrigin: true,
                     ws: true
                 },
                 '/api/v1/*': {
-                    target: 'http://localhost:3000',
+                    target: `http://localhost:${backendPort}`,
                     changeOrigin: true,
                     pathRewrite: {
                         '^/api/v1': ''
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js
index db667203ac722..1194568f79217 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js
@@ -37,8 +37,7 @@ export default () => {
         devtool: 'cheap-source-map',
         output: {
             publicPath: '/',
-            filename: '[name].[chunkhash].js',
-            path: destDir
+            filename: '[name].[chunkhash].js'
         },
         plugins
     };
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/plugins/progress.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/plugins/progress.js
deleted file mode 100644
index 5f753c722c592..0000000000000
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/plugins/progress.js
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import ProgressPlugin from 'webpack/lib/ProgressPlugin';
-
-let chars = 0;
-let lastState = 0;
-let lastStateTime = 0;
-
-const outputStream = process.stdout;
-
-const _goToLineStart = (nextMessage) => {
-    let str = '';
-
-    for (; chars > nextMessage.length; chars--)
-        str += '\b \b';
-
-    chars = nextMessage.length;
-
-    for (let i = 0; i < chars; i++)
-        str += '\b';
-
-    if (str)
-        outputStream.write(str);
-};
-
-export default new ProgressPlugin((percentage, msg) => {
-    let state = msg;
-
-    if (percentage < 1) {
-        percentage = Math.floor(percentage * 100);
-
-        msg = percentage + '% ' + msg;
-
-        if (percentage < 100)
-            msg = ' ' + msg;
-
-        if (percentage < 10)
-            msg = ' ' + msg;
-    }
-
-    state = state.replace(/^\d+\/\d+\s+/, '');
-
-    if (percentage === 0) {
-        lastState = null;
-        lastStateTime = (new Date()).getTime();
-    }
-    else if (state !== lastState || percentage === 1) {
-        const now = (new Date()).getTime();
-
-        if (lastState) {
-            const stateMsg = (now - lastStateTime) + 'ms ' + lastState;
-
-            _goToLineStart(stateMsg);
-
-            outputStream.write(stateMsg + '\n');
-
-            chars = 0;
-        }
-
-        lastState = state;
-        lastStateTime = now;
-    }
-
-    _goToLineStart(msg);
-
-    outputStream.write(msg);
-});
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index b511ca1e9123f..fd50d5b6e7552 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -29,97 +29,99 @@
     "win32"
   ],
   "dependencies": {
-    "angular": "^1.5.5",
-    "angular-acl": "^0.1.7",
-    "angular-animate": "^1.5.5",
-    "angular-aria": "^1.5.5",
-    "angular-cookies": "^1.5.5",
-    "angular-drag-and-drop-lists": "^1.4.0",
-    "angular-gridster": "^0.13.3",
-    "angular-motion": "^0.4.4",
-    "angular-nvd3": "^1.0.7",
-    "angular-retina": "^0.3.13",
-    "angular-sanitize": "^1.5.5",
-    "angular-smart-table": "^2.1.8",
-    "angular-socket-io": "^0.7.0",
-    "angular-strap": "^2.3.8",
-    "angular-touch": "^1.5.5",
-    "angular-tree-control": "^0.2.26",
-    "angular-ui-grid": "^3.1.1",
-    "angular-ui-router": "^0.3.1",
-    "bootstrap-sass": "^3.3.6",
-    "brace": "^0.8.0",
-    "es6-promise": "^3.0.2",
-    "file-saver": "^1.3.2",
-    "font-awesome": "^4.6.3",
-    "glob": "^7.0.3",
-    "jquery": "^3.0.0",
-    "jszip": "^3.0.0",
-    "lodash": "^4.8.2",
-    "nvd3": "^1.8.3",
-    "raleway-webfont": "^3.0.1",
-    "roboto-font": "^0.1.0",
-    "socket.io-client": "^1.4.6",
-    "ui-router-metatags": "^1.0.3"
+    "angular": "~1.5.9",
+    "angular-acl": "~0.1.7",
+    "angular-animate": "~1.5.9",
+    "angular-aria": "~1.5.9",
+    "angular-cookies": "~1.5.9",
+    "angular-drag-and-drop-lists": "~1.4.0",
+    "angular-gridster": "~0.13.3",
+    "angular-motion": "~0.4.4",
+    "angular-nvd3": "~1.0.9",
+    "angular-retina": "~0.3.13",
+    "angular-sanitize": "~1.5.9",
+    "angular-smart-table": "~2.1.8",
+    "angular-socket-io": "~0.7.0",
+    "angular-strap": "~2.3.8",
+    "angular-touch": "~1.5.9",
+    "angular-tree-control": "~0.2.26",
+    "angular-ui-grid": "~3.2.9",
+    "angular-ui-router": "~0.3.1",
+    "bootstrap-sass": "~3.3.6",
+    "brace": "~0.8.0",
+    "es6-promise": "~3.3.1",
+    "file-saver": "~1.3.2",
+    "font-awesome": "~4.7.0",
+    "glob": "~7.1.1",
+    "jquery": "~3.1.1",
+    "jszip": "~3.1.3",
+    "lodash": "~4.17.2",
+    "nvd3": "1.8.4",
+    "raleway-webfont": "~3.0.1",
+    "roboto-font": "~0.1.0",
+    "socket.io-client": "~1.7.2",
+    "ui-router-metatags": "~1.0.3"
   },
   "devDependencies": {
-    "assets-webpack-plugin": "^3.2.0",
-    "autoprefixer-core": "^6.0.1",
-    "babel-core": "^6.7.6",
-    "babel-eslint": "^7.0.0",
-    "babel-loader": "^6.2.4",
-    "babel-plugin-add-module-exports": "^0.2.1",
-    "babel-plugin-transform-builtin-extend": "^1.1.0",
-    "babel-plugin-transform-runtime": "^6.7.5",
-    "babel-polyfill": "^6.7.4",
-    "babel-preset-angular": "^6.0.15",
-    "babel-preset-es2015": "^6.9.0",
-    "babel-runtime": "^6.6.1",
-    "chai": "^3.5.0",
-    "cross-env": "^1.0.7",
-    "css-loader": "^0.23.0",
-    "eslint": "^3.0.0",
-    "eslint-friendly-formatter": "^2.0.5",
-    "eslint-loader": "^1.0.0",
-    "expose-loader": "^0.7.1",
-    "extract-text-webpack-plugin": "^1.0.1",
-    "file-loader": "^0.9.0",
-    "gulp": "^3.9.1",
-    "gulp-eslint": "^3.0.0",
-    "gulp-inject": "^4.0.0",
-    "gulp-jade": "^1.1.0",
-    "gulp-ll": "^1.0.4",
-    "gulp-rimraf": "^0.2.0",
-    "gulp-sequence": "^0.4.1",
-    "gulp-util": "^3.0.7",
-    "html-loader": "^0.4.3",
-    "html-webpack-plugin": "^2.21.0",
-    "jade": "^1.11.0",
+    "assets-webpack-plugin": "~3.5.0",
+    "autoprefixer-core": "~6.0.1",
+    "babel-core": "~6.20.0",
+    "babel-eslint": "~7.0.0",
+    "babel-loader": "~6.2.4",
+    "babel-plugin-add-module-exports": "~0.2.1",
+    "babel-plugin-transform-builtin-extend": "~1.1.0",
+    "babel-plugin-transform-runtime": "~6.15.0",
+    "babel-polyfill": "~6.20.0",
+    "babel-preset-angular": "~6.0.15",
+    "babel-preset-es2015": "~6.18.0",
+    "babel-runtime": "~6.20.0",
+    "chai": "~3.5.0",
+    "cross-env": "~1.0.7",
+    "css-loader": "~0.23.0",
+    "eslint": "~3.12.2",
+    "eslint-friendly-formatter": "~2.0.5",
+    "eslint-loader": "~1.6.1",
+    "expose-loader": "~0.7.1",
+    "extract-text-webpack-plugin": "~1.0.1",
+    "file-loader": "~0.9.0",
+    "gulp": "~3.9.1",
+    "gulp-eslint": "~3.0.0",
+    "gulp-inject": "~4.1.0",
+    "gulp-jade": "~1.1.0",
+    "gulp-ll": "~1.0.4",
+    "gulp-rimraf": "~0.2.0",
+    "gulp-sequence": "~0.4.1",
+    "gulp-util": "~3.0.7",
+    "html-loader": "~0.4.3",
+    "html-webpack-plugin": "~2.24.1",
+    "jade": "~1.11.0",
     "jade-html-loader": "git://github.com/courcelan/jade-html-loader",
-    "jasmine-core": "^2.4.1",
-    "json-loader": "^0.5.4",
-    "karma": "^0.13.22",
-    "karma-babel-preprocessor": "^6.0.1",
-    "karma-jasmine": "^1.0.2",
-    "karma-mocha": "^1.0.1",
-    "karma-mocha-reporter": "^2.2.0",
-    "karma-phantomjs-launcher": "^1.0.0",
-    "karma-teamcity-reporter": "^1.0.0",
-    "karma-webpack": "^1.7.0",
+    "jasmine-core": "~2.5.2",
+    "json-loader": "~0.5.4",
+    "karma": "~0.13.22",
+    "karma-babel-preprocessor": "~6.0.1",
+    "karma-jasmine": "~1.1.0",
+    "karma-mocha": "~1.3.0",
+    "karma-mocha-reporter": "~2.2.0",
+    "karma-phantomjs-launcher": "~1.0.0",
+    "karma-teamcity-reporter": "~1.0.0",
+    "karma-webpack": "~1.8.0",
     "mocha": "~2.5.3",
-    "mocha-teamcity-reporter": "^1.0.0",
-    "morgan": "^1.7.0",
-    "ngtemplate-loader": "^1.3.1",
-    "node-sass": "^3.4.2",
-    "phantomjs-prebuilt": "^2.1.7",
-    "postcss-loader": "^0.9.1",
-    "require-dir": "^0.3.0",
-    "resolve-url-loader": "^1.4.3",
-    "sass-loader": "^3.1.1",
-    "style-loader": "^0.13.1",
-    "url": "^0.11.0",
-    "url-loader": "^0.5.6",
-    "webpack": "^1.13.1",
-    "webpack-dev-server": "^1.15.0"
+    "mocha-teamcity-reporter": "~1.1.1",
+    "morgan": "~1.7.0",
+    "ngtemplate-loader": "~1.3.1",
+    "node-sass": "~3.13.1",
+    "phantomjs-prebuilt": "~2.1.7",
+    "postcss-loader": "~0.9.1",
+    "progress-bar-webpack-plugin": "~1.9.0",
+    "require-dir": "~0.3.0",
+    "resolve-url-loader": "~1.6.1",
+    "sass-loader": "~3.1.1",
+    "style-loader": "~0.13.1",
+    "url": "~0.11.0",
+    "url-loader": "~0.5.6",
+    "webpack": "~1.14.0",
+    "webpack-dev-server": "~1.16.2",
+    "worker-loader": "~0.7.1"
   }
 }
diff --git a/modules/web-console/frontend/public/images/cache.png b/modules/web-console/frontend/public/images/cache.png
index 83fd98790df1d91b2c41bb39e5c97fa277e96725..3ff310379c05abc75e687ab0e08d9a2026a3b083 100644
GIT binary patch
literal 24791
zcmZ5{bzGC*`@VpLw1g7UH4vDLQa~ml4bsg3VIU!>G{}%f83HnxqZ=tfQb|E_bThiU
zyZ4*#PrN?A?;m?TuRYIs&V9~v;=b?ey3R*!EoBN)CemBCZc(VJDC*w2MF72Z3!jUG
z0C%TiMj;#b=UQ7s|2Y<mJw863o}TXR?!LUb`taey!NGy3s3;bDc!<R=FRu&_4~=bN
zv1iLz?1rI%;m8@bT_Ne_bkab*Z}ALUvWER7lY?E&%9Tyn!(u-ST})xImD^Z^<}W#g
z^6kd9d`n4tmDXaX7<uW+nL})gW7%Fj5_{f%I@F6@pZdLzEyz5;Vkbq#)E2SWo4v7v
zrgzxmt*f=N#?f<WQ8DcoRr-a;ViM-qqn5adD=~{78(3`T8FuyCuPo6xQBf%uDbd*!
z-4aKsgOio(<rW|1hV2L3tH;NE-cPZ63n4LIZ&rrhH*Sf_xa7P5T1ki+NJyv}3>I2f
zoHUf|o?)G3>tTwb-=F1+DZy4MTVKeRb=WH(-fTn4*PRSa?d|QiHn;Mu1~0l?O$}bF
zh>Isn##N)wCc~BARP199e@>pPi)tbCV5_O|sPpmYv)%ygWPgrEo2lck7>J>s?bK3<
z<J{M(y;_U;tKGx$++m}bit1UNxB1`RXjNK*=69y0T$`3G<1-hqyPeyu@9PGm%*zXQ
z4L)?*rjCCF6;!Fk`<rOHcV5bxPHIGse9;tjDA>%-&eyk$tF_i`c`aG!J+crW*0Yb@
z>!0rn`03cZ7c;PZHdb{!o|cR`>z=!|Lu~cMHPlob02F&`q8zJj9SpLTvvjJ05-N5&
zLM)9czI&A`X<1dfYikG>T6#t|hzDyWP0NXE3P@E#)XKnqJvDW8^~)!tT}Sf)DG9&E
zUqIg13@jWRw>AgsZ`(9lgjcj~pB!uOIVH91ttJ0?nY>UHpkmsPtC3KmP}`YqS3G_6
zOrz>}{-!E@Vw+1NptxsYxF$%cV)L8&M$yz{B{HvjJl@O7PDf6Rt7s6NlWT5n@O3vg
zeBIyRO?&xD;pdJs?CE3Yl`~hVj_sw4_RY1qzQM({_MfQksoy^;$_CX@+ak`}ERP-a
z!_urj2DlhpCTmZBPHN2!{tA^2v;H;Qu#(3bnLri1FUXyR3XH2hOlOTe%ktY#FH~3C
zeTQ2QciyO&xZS!%M*H`NcPlZO0e6$wT~+fr@f<lFu`qf2a-YMkTkN+~73K83C${S;
z%=JAQ+Of>ZkL`(28F&jit$Bndau3?Q2U$q)*`D>opX+fhuMj*;Q29x?qVT<n;oC1W
z{<WPRF||X-{MsO;n!==sFw~0aSgl=RC1#?r1{*M4dwPuZ4_NKyBIvee{TjZTH@)7C
z5+#1Q@-%IB*Gu;BnuZf-M3k-#7*jD3=2QGmj>ymk81ZZK^DUDwk}m6Zqg8L+&}##b
zY9Db=gbi=$hRJh40d%-aMhu9KlwaYW%W^CN>SnGadlRVh4Z`1b{q5`GNrRO31u73_
z7KnS%f?pb|Lx`+soc3<aQJIxkZkIM+zq_6{dJk6_LLB<U-u;0}fNqOIQhZHH)XkSF
zF5GQVK;gjK(Cv1-a*F29gvq5k9x=N2y!Sy!{nO7y2JHrrj3gF##tKnN!?s{jSh18s
zdxV}Bz1u|<vkD_AG_q_;P)!ZiEA`2QgPG&AgMiG-UaNUTv^yi$I~VBsX~V^`*A$*a
zwnoZ<&f9sDuVL$YyHbheUnoVu0$NWaagCM$s0Z#Z_=cfi%v+BW44yQ@n+CmX1IdmE
z=^BPv9wio$$1b-t#eklVk!TOJLjx~24)8h%{@M;MbC%%G`Z^%Dh*yp>_^g{xo((9b
z(iRA0b9}q|VL+##;(2`9!}!(6^|ve9m4**U717!NX#j1V;2XT@TFv)}MW!gkz~m^Y
zwS5~2fxeBWMHmaN-jSh*{h!t?fZi?_*6;QRj|W6i<=I6C>f;!f4+5*Q-*?{hGqT`D
z=D76Y?bOc;4zu4q^cF76(7g}d$&Ap6mt3m&`tBr8@Ri{JeJ{k1_$zg1o`AT(+F@oQ
zbcKnthk}C~%8se_%gM`zk09Tqaw=Rw#A|eod3@Mb`%RDfj{fgV&Y~skc#N-ScX3g6
zZ{U$crCWMY&SMYs7wMYUWl6J!2nafF{OxNmi&3|D<y=x}Z9sO>YDm1Gmyt+&j;HqS
z7sAw<?)6hoqAL7PZlPuua3U7}_qWX+s7~M+<^Rh5{IK-`{|_Z@V*PP#NY}=7I`r?1
z%q9<&$9<x|mw(T`znA~azQ0BOr{sY)t|-FkIX($#=)0W%nWg`y-2bWZf9LbxlAaq$
zb#-+kW8a4+t1%Q?k?G9UZ0V9dY@E2M`Ny##)$3BBObobNSSiG}R;<CkdxDKRzu&t1
z6xYai=KFl#0{!DoNBn|Hig^?XD>_vtsNS^eCb}Y>VR*Pj!(G6$djdRkDS@({$dWU1
z9W-ow24!NdD1C9moR^Q8Y;N12-R;Z9UZ_SRH0vvEJM}(I@u+`(wT=z-lT|>OGm8BV
zpwC83UWb#C+m`U&*z|$jyU!G6E_d%B>aR6tCI<K#9M|Phft2Z6xSeE}$G($5nf|Jk
z`Q3c#u}M5c4J89em+22MV5PR@vpaZK!N<SQQ@6DpQCD4fFaw#$LUz)HEIe;>rX1*3
zP4w<FJ&NmP=GP<YSh^8J={k(fs@{v3Q_(e;*(QcrIi0w-JU(f;9CIy68RLT5xV(mx
zEfGOCoey<F=OJTM(!V5i@6iEHHzk8*9+7-}Gs0ve#4M_dUc1NBt;iT~UG-Kxz)>3N
zcu02jTtc5a`+Tjqyv4huYm?_t2~E^-eYMTbX0>|7#3R?kH|J~8Mdf4ZuOc~{_AMt%
zdHc9-5-WLbiKyNj3wWfHP<!4{Nh}e*U+r=`mH2c}5c5DDep>VG0so-|TKfwxRB*4U
zjID5f<sqYZi7OmKHU7u~CEIA*xsrmY;!zi#o_Nsa-0fbn+0VkS095$p%Yhp%+|_su
zx&{SM9h6mY3Ui@eB|aAB8cNmcyHwlJMF%ws=~a6ud7%ElvJH2^FYBw-o(53(GNKRb
z1trxRqo{ktYT1(n)2dBnM?(?$CzTeukYD9Zq{{_macQ$t;@~PWJD3WiFY118*Mj;_
zG<BxwFLdg+QOW8TbpZ<xQJVmD6dPsrrceN*rubIB{_ZpMdj@%EuGe(~NEP8>yL}Jf
z<kG)h&3$0@3mxsQ%4}MG<iQaF&mZIQu>ImgjJf=Et~f*cJrJ&wJJEi|G$hPcUZ_!z
z6&y&&x2!L3`1GTlim}78E-N(v6c@$5+Q_8DKlxsHS}s66l~Qbe7Pbm~-{TNUfY@hK
z!g!cB15-a%od%v254B-rT@Fsm^OEXS61>jk<_-B?!xZNS(n!Amq_y=J$PoF4ql76y
zlbks|9{QyRt7_2|bS&%zKL<}DTisN5R_;2Rp4=Q)TRa9vx*7v!3NeE)x3L83PnZOC
z0;j47pwfGVyaP28JXNR1=jZWdq3$mY$6z_m!RC-5&-M4mzM)`kJJn99N(BVrEUn$x
z+vw|^({)eqZ>(o>&*Bo-DouI*hY<@=bW7aLl9B8Zv{qa*%I$ZT(m<oPrvnK%9~7pj
z-_3J1O8$JL$-VgnNMqt`lP$L%n~MA5PUEl6lDoq9tpulOHJ31EEvG@|`RrE>#|Mc*
zz<x<%JZ2H}m{1qIB-dT|wO<n(p!_}k^4%`qbisMl6(4rn9;?NAX0pVk(+G!Oc|m?M
zpO;(y@OMkS!g8{8{<62)d%7j+ew6+5rzdE);b$JFVcWE%^NMZ_Px7m+EMUX$J_ech
z9E>2?S+kC74=eboZHL(($sfeyckh;TKe+CH<}IA%e5Am|8TQ;HmEC*ob`2pPyMi4R
zq+#4$a=y5!%ScAr+Z#_rI)7s)unBnGWK`!YsrtnG9-ymu^lh>Vs>++IKi<46aE)?k
zbK@S6ESF13oCSanZCQh6FMl>Ih6KExE(0vqX*7HnMo|jpukc;W*A)??!>)YW*w)NE
zHv0a+4)mmo>+R>hrj>lCeevhsWP2Ul^7_@z;?}OJNIsL&i~Ay5FG|K_E9W9tiG_Wa
z*Ln1E>zFtbdu)c*+qZ}NKP;E;u>87i8t;VD%4P9w!>Lo5q1jtsm+$a~le%W*XLP=p
za)Noa60a|;yLD@kHtIL01>9JsO3BBrK@~kLm4*ab*Bs)34E5bACL1{)KHSP-hB-v~
zU@ARc+pES0f)?wXb^dfd+JH&`9%}2Jw?>HkhM>i;`$;G(lB;QgK)^-rGIsyT6vagV
z_5f+G%)rk@C3Db)<>rJ&hCR>Os@+Y4mpvTX*uC@)Igj}qB(hmK2C+l1_}g5a&D*6p
z-M{gj=VwiI%G`R9z(-w2u0>6NK<x-qQrvxkt-z>afR9hGKA!2(-i-MIqs6+}^slq5
zOmqwl?r!H=nws0dp-ynO6LGX^VIWBJn8-TnZd*)&FM~)J^?56^5_+Lf@_duYh6>X5
z;_BG--45&H71lVUP+0AY^8E}I%roz$zC{g0Af65|Nz}gSehK_TM-d817%toZmhgV{
z`v4`X?84^w#680M=KSvdp1B$(OObHYvwP#V=$Z;tF!_UyT}`d=5y;nfNcB8mi~EX=
z3oA8re+Bx>B+C7}=tr}h2w*2eX<?>VtnNK|(@)iRM@G@518;M-CB5hw48i7_sEwa*
z>bE9pgtn`hR(9EkbF9(<5a3Wk%7aq%dRc+$dE4KuWv|pQlv3bAvbZ!<fWA+m-VPPQ
zq~TMZFyBb4J9cm433lJrow6+=$OrXl+MgYCAlWQ?&>k>Js{}i6&iz>S1mmuDk~c<w
z6_mpN_H975aViOAl#j#S*(!d*!;TJjsp^Z+=J!2|yCLDL0v&Y@f0}i3TkF4<%HS~`
zHzf`6{k}6EF|Z|^g4DF^)^e3vANCTx(>NL&&No=bHpM|WaBslJ#3{C6huq8RVM_9J
z!Q9VZ57$EkZ5`{iUMRaRs5KPw)0VSYbDwe@-<Ux+t(yt;#6>Qf(gc!beUdXraNDG(
zegi_Qt%}X4usY=1oP*?55XgZ*P3q;={P%B`H^T(j2Md5v$x()<N?&boeNm8t+Q6>8
zzph@3pYgT(oNb1p-XDP^>$;rn#FroqM4^x{go#b|M9xg@G#>gv2kh&2&O^UU+FU;7
z70ALx9YUL=adm1gRPkmNnH9t)<K@!+ZEVK-(0yxn*89*aSE#iJvq%=irm;$_$#A}M
zwj2q5>?Ku|Cd#=6g)KsX0tu$Vpc<h|>rCe@;xw84gWC_$*&*_0Pv_&pwZ6*GjWR3H
zq(*YmTEA^s;61w;?^FQ$?F~IaltsOvXSeNeLtQ7`-B$yWeK~&u1zi?w8aIA29B>?i
zf{o?5FiM78-fIWDyJ?S{B#yH_{vmFB!1aVqo?nmR6U#v-Ei?bqEcv+C+9%^j09e~e
z#%!9nZtewpRe&fbWF4xOlcqOX4gPTA=<P)<_2e?+;@P7-VUwu7gvOh7c;EPF@W(9q
z`|6aT+GwTFusTHQUgU!fowSLAvXE7}gIGcZ;;W!22L5r5Bh_ZAwGw!QkuOR{WxdaZ
zS8u}Pc=*5oULw4<V{LB_=2ejYV`!M=|LdRwuAb3(#p)}H=4`~<cuLody_#BQxhh|6
zYpgqqYH|&W+o^ZnKUL_d)A05TfGXAvc9jNsS>C+DFw;%TuPtr08^3vEW)hX(mEY%l
zzYd-__;Njix->_pi(lD3JglxE)?FIN%yih2M1-bEr5$~IDa5r94+-NOtk)2Kz5C}<
zp{82O-VSUfA@Q^Y_x11qk#>h5-$V1VC2f){XA=Kj^~tq5=T{^)k;Q_ERdzk8^^d$?
zbq~)!dMV?-6zrDQicf!?a94*DN|F(gF=hweGH8S<_IF(obG7pyc5j8(Z#2S5d>Q4}
zOhjEQyMxn_=VVG(QpYnal-j&cFPpGNW#<G<RJlp6jBFl{shTs{<xdD+vNfNNlSjpq
z?SB|#$ITippJ<~Tzxbmq<-KcKLD)?|IO<=;<=IJJR@DKLqi5*Rq2A9NsE<HkCYn*P
znunKoKd4}Bv%Nymfyau5S2p|Ujn8R0Nb$|K23k+sKi6YLl@Hl(XR@y#(R1&~r94|F
z3Do{JE=vDWcc?bFzY{~%Ea3UUjPOt84|afhj4&IcP?)?`qz~%-QdRy?nx_~Q&Uogd
zR@u4tKi-UK3*R0t$&thLyQ9mr%Kgy70XNDapICkm01^%#>GFoi!x6C@X}uS=h(A8m
z5kV~s^;>{s;M5W{=%P}L#23Y;dg<Y1l#`NjZACA=hV!3xy$?`YN-lKr9Jj0H7@lok
zx`!XK3zE{fBqIr%&PRV+7z!(1QEiP4vd^qTzm|U4ffo2y)}#{`y4js5^uA<U%8|?q
z|NY)c;Ngv27<n3Nvpk1$<Wsd&?<XrFxUS37g+RgVCmhMq#Uf?zIGS%&S*izn=O8<S
z@zDg=M$33|TIWG2e70aa|Ao|Q4Qv7tZq5BW{7#vDzkjyXzC@K<n`a{jM*e+7MtZah
zRPQ1PPJYXj{5}hL%wD9@pT}gF_;VT;Su*Obn4^o7G&gtHabc_qO1-{l!<IsKg&5M-
z6<Y>7EE~XT6g@}3pY0MG8}mG+G`?ol9pytOf(z!poJ5!Z>8gNISqf7qqh)5g#A-kN
z8jeFvaJslyCU=)fuGYz9e4JaO^>K|eRb~119G(6abJhE;0#sO7JPkOz(rZv+3{YxM
zY+IKc_o%0@?<T8rKJpJ$Cirg4f^&%;Iq*u04IKl+;wP&gPYw8H?662Rlp~v%fBsCm
z%EL`HRIWNfP6X%}WSLc@)E-~emDsp=l;oR+<bXovp_Tdl-8m8pu?Zu@Xzxvlp%nz<
zu1dOKsQ7ISlidFC=Zy2wK)thjPzMup2TsfH{Thp;m;L5+{D;qgdacDlUJg5Y&lnry
zo%9vy_@2{h)_-=Y%@7Ks{F(hLP+chf@>s9Tklz&Orpm%m;1E8ZBz+T3wH{Zpy6pel
z-hO}Va0W~CTkV{~Ra`rPC|WJEyB~hYpAH9qpGoge$RApNq+2`_knovv<nm8?*WeX4
z@H%B=lfFUO<kRONuVFX0bHZd;k~=&GM0Hf=ofRC2Y?TxjP#I}@1(Tc4Ih=*st|W+|
zRa^`$`cc{LnL@%(uYOg1<C>fN^y+4T!n(yXY4Etm2q^1${qRY^^)&}oaysJqQnA)q
zSZ`N01tTLg(NSTJsZQfiBqdQ%Cb@U4qLO&a?4Ee8Z!kRD!0Wboa$}gFz+`|!I+d@9
zL=wUyL<hJ?iCcvsNVz04T4+{a8HpNZ6mFZjtzUX~F?8^b;*k3X!61b~0STIs>NbPo
zoa%g!D4%EwN|aj@`+ic1ExuKqEn1^oBwYiVzI9dx7}U6?P=brU=HNbC)Lh7bXko0y
zhiIJ8hWzi69#8&_1k_Ipq(PTjD1xV!-M1}_Lx8`2)Uf4}vga1CdnUKK)^$I_MNc!A
z8;`AVfzV5~@8NWC&`3k}wX){Ct`chHg38BdxAA^#b%IWtrKQk>G14SS5h$vyWQv}U
z*ri*6x-;AfT8@3i_7%w@A#W$~GJElepQjZ4vv_Q^Z*nOH`K14_HMuOR?Xao(u_pa9
z4FkEY$B~WQT^fkkIWsT!DAK?(bRRxG{sAG7Yw0T5B=kpEo{<iEP35gnf2&e79Q4OZ
zrv_>AuNNNH4yWK=xZYOF0m09RpdT<an<G@8o$9JS%uJYHm%gWWlJ-2=D|Oq1{l4F7
zNUrvPUO<1J0pz!LK(D>Xbq}y;Pkyg3NKnGymbHQ0coVICG?qnL9XqnV)+gSUsLya?
zK|RWV$s|fa-u>!n0Z68{ly4WrEJXGI)F(A&XHDATm4%LhMEcv8yF%zo?d+|Gze@Ek
z>bl=W=~R~J9F>(jX9GSngpvx$$}>KxYk3DDdU%cx^_3R3lE8YSEDZe^Y@egzHet8H
z46;uUI=uS(A1Jqq+LS@=*49E+AVOl%QQi+iC#@TIMB$!et{zes16>iM-;ow{CCH|N
z_IM)yrb^ZL?bXIYIO8;8;CsK_h(GM+`<qOm+!9~KxAwpDKfMkci5}pS&4!$e!~|Tw
z@hir`RhJ67@u%$}t_OyVW8vaTnNl;O!VsXDiq**CmGTCDnM;7*?}o9L`fOemL&Q?}
zy+Bp&$niRjWw%1MdW-lf82$YppKgOlQQZgD_G8x!^oit%R;B{)ThO95VrG%DdyhmU
z0L-nFu>H#+`_{uFqm<7#M<AN+GB)c8&ackV`KzRo_w(AS#MXXFT-ZM`kSqS46R2L`
z$C3GU(X%`&PRowT@Ze5&Qw85agJX|hQg%v*AJ)=`t~#JeO?G-9d=u*ajT968$hdM=
zQLJ|Ob$HhbpG3f4FhIVewX90fxYMXnO1_ISBE34vA6^EbF&rHv9r3Wmw-qkB#LK0t
zdHJFy;}dZfx#N2V02w9JK(1Th!*9b<;{z@Z1ijpLqZG^aI+Rk)&+m10^fAxneEWu;
z<+^`c!yw4aaCJBpSZ-|HIv*y$3I~1LmovV9PYI=c`bQrR=ep*cM2Y%tLi~kKEmJ<j
zjzPq@9b<Yhb`bf1OLyg~x#_8m{ww&gu-C}ho)eJkO7d&xE*R)W1f|)4xo3307MH!6
z>PqI#?xu%?9ozxBB?j5v+!PQV<8j^SV~Fyn0YSSrvu{IQ!R(A7KO}Dj9l(I)<=Tl<
zjgJ231li6$8B~xe-v^~o<BjrvJ&hygzsB^~s!127ntm67JrMF}`MRgUpv}uLfqkxY
zqu+|pE=$|_47kZCS~MqQvd^;-ZqhJec=9KKor=?&acb{{tm7Ui?W1ivG9$UJ?rwlM
zcX91jxX=%oh>xIy@wC$`xd$Q_YTN|%<jc1CkdO3$^7!jgo{r#d?!u<#aS8{R3@JW)
z!(S5$PcU67PAK1u*2tO5$T~wg)-z1%qo17tAkYEW<Q8{;5as=WE^YrHYFteIPFCrR
zGYgp^Gtvcd{}Z8u!p!pFs!(lz<z8=^!sH|goz7EtZjYe&G%Hn{v^%fDzuZJs_ASWS
zg1$vxxMhZx9QAa@(_zYmELP<v1y^l4;#il~aFCy?-)1^D8*xw?)>0r4k#8Q6!bj2o
z_8itu3imRLP(0X$L&6|LE+_j;m&)*IACqR$hb=e+MpE9NIyWKyZNE*5E)k)e%Ihir
z1&X)~_6~VNB5p>tV3MpFj>Qb}j=6b4=Sc24aqsg__81EQL31j)AgJ^ur?#kV(GPPK
z_&)VX#nO(}-aiq!r)sFmc$Ss`I0&)=E--DI89A{7JF6B^;ag^g1+i<_S@lj6BvDj8
z^GvK0d^w)1N*x%&+y&GjgFXvlBbW4lV&}N_Np8N|d2N(<z55euQtBVdd)<)+gc*It
zO^#s<WQ@0&sdW5fTpq_q(Q?ZWvrOmEao%O6Kl6CJ`PU>QK?pJT19)jY)TMDO7ZSuy
z8*q)3LzIc^g3$NsX?S(p6y##QMHH88ac{MK)4@gVsc*cahlB{A<q*GZs2Y`j$)C}M
ze(E1$5!#|HDvW+@q=up`9nFA6j<|V;>{{9g2yYJc9Y1|{Yvpp`zmaXc_3u~qCx2Gl
zmA_Hpw-zgv>ar(#@k5yc%e)7=Sb6rfsY7-OX<Xhpdt;V}t<RL~uBi~$B&?)I<k#d?
z_aY7H1V_mO6~>P~KD7ze?)j}P%suYiVeCCIzN7VQ6&e@bz?F})hq#uMPW6xj9x?+X
z8>N~Kp6^aFElxannbFoUYkp~CyR^uQR-sQphQ|5~g)QUy_@pr^^0JWBr8~7}1-kWA
z_O4oRhcb+bC*W`5YbbNMvKQ+75T#X^3$)wyGZei|$x$RgBBPMqZ7kg4P9h_eJxNRM
z<Mb#cS5-g}KN^I7&{va=<OsRW30VczTIg?GGwKG6A1T-V9(3cN;Y;9;)#qbV!)`yF
zV8$E@mZ05B=q$ci4@9QjX0g$BY5f*G)|Jc!Yx~t?hs@kc@7`e4JzjWGOtSrp`lDVK
zLS?_KqDX1E$D&5FJ{jayUO$^>;oJYSq8k_^&j@SmzF$|wd|kck&titIgI9XYmy_9~
z9RKM7aA9Zwam0iO;)@bA!Nq}v!(c(x2Du=n>tf4Q_oCJK*TlaP+oGt&qS@ps5MT4#
zIJOEOsL4+fd-cYE#h%$aTvaZIJPWS<R}mI&5;^+lh^kl2I@<;Pu1}g>mj%FsJg;B%
z436n2CNjHU2z|^>P>&k9w`uu$PR)eFt}rVkkLH!|SKs?NaIirO<!y9GSg`F(lX(5r
zARg!!SbY>(!<Gg9Oz%WiJKL3g6uA!aO!hA6S*(e1=5phFz_;7Rf+?uiWDKyaKBw82
zJnJkmUxdEt0;)Xqy{Sc0!*q4!df$FOQc)kk%QQ8)@7`b=AC%qigR1&=)G_>%JcGLY
zTqid#m$iOSaTQuj&lkL*&UXvi78abOd~~HwOnBjy|JcWh&CxhN75L56E}&WM6Y@!=
zCAqnTSt|LRs2_qbJXarUr!bM;rkC955k_{OVXxTL6K_GY*qH8$2%-zH-7(cOKan|V
zN}-?A`Rw<pX`x|ro$@b9vwla5Ez5;ekoXOMWmdfhBN&+`*j5VzTt34Do)r=FJHO=m
zH#(XFi_c)sYQC);G(t7%QzId+lis}r_@jquT#wMH{35Go_Mitq%S?H4p>FfEtzp<k
z<B2hcb<3;9<SAen+tJ3s-DVGQ4w&5RfnY?_0!JF^fE~Y7nkz)_IU~QcosTRhge8fo
z=4D@E{*b8kL(EiHE@|MG2j!1Qwe7ltOJ|owy@wIZB7$=LSa!cJ9R3F&wL4i+`&l_&
zXR3(kDMyw~q1V}h&yAfc9;5g4Ph4@?LWsB!rsIr`nUWK#mD#;uFl??l3}ab%4|MZv
zWVpdJzNQmpk-x$Zb1iLEkXib|Pu1)`UvfW{<_oV9`o`8rHV{M-+9I?6(Yi|!#UO)v
za8bbz=hWq0ksIF~q>N5nR=WKI`&b&+mP8~b+xe&heP6J?$R3^f9BoLb-9u*x#?2xX
z)WbT#n~mE;9uD@tsJP2k?Jqko?t}i?1q2JFwkm{~Rw=;$;&SqXw?TXYrByZwt2T%q
z+A7`OG8Ak=g>2GWsAcZtDnUuI7yz_~>|0*eO|<1x5qp+5(J3c93fX=po{r8kM>mmx
z8M`K`ij)<;V<pslem}nE0})>4F^1NEH3D7|xYGFD#KzsOk%5&c?4w`?M+<OV_xAJx
z2djJtardaqjh6T0czh6_HUH28#K`zK4No{k|Kt0mJkrGk1^~{g5<&#tJj$^8^?-d#
zxlFm|9hY;_xl68pXN;^laI*N3khOkTOv1uz#mY*MU<0?!@9&=m6~BC`2xDSRLWV{n
zP@h4v9!~QE4|!AxKhaHhet4+Ye6}@sr*8aB>DUNWc9H9Km#_JnmrC#UeH#QnOs;w>
zGJJeceR50s`|2yzG3C~~rFvrwJ!NbK7WC%KMN4iu)dG)QcGNx>k~;_uj-||dj=eJ<
zTzUD~sXLe1X<V)0T&G!Y9n-L=n@6{qWz7R#w_V}>{4kka;=z^&Popw=kVyj>sxfH;
z(=R-ojk#oi()hhq^D&+{2Cbe%-i3a$EG?GR<qfd=X~F+FHH)W;bvdo#bf|~euc1Yf
z?kfkMbDqJjtI>?W!#xqSqr167ZPB^3R%1Su{r+ZYf%$ryuNP=GS7T~rtcmdL&6)gc
zrl3&dL(Qcw`a}nF>z5@Al?AJ=Uh&A|6<V4ywp>5Md2+tDM^qvI(E?;Y>+a(%;!C`S
zch2@nNIrE~r(`=W!)3q->5c2OZQbf=t3%r3_fiSVjY3IaLP7O%^{0)Z=x3-`4tJD|
z^VUD#SNT-4Z3gs|h%KXI$=m(5x^h7F*uO+CdSK*-K?%=kG*Yz__7!Vy8Y=ogE6_cs
z`$+UJS>iKhKaSejGr$439^}q0UuRKnc3ot2S)coWy3VE}DRQ-I<{Fi*0hjc?mf1zn
zhQ~vTrylQN4Zih3tm7h9A=P>bJ-kpB>*21S1ob{96Js2@h6k58pfREU7QTM&OP0^M
z&y?!L{jQFvPbeg<TdB3IvIW0dY}tXUsn#hiQ;<3F&xc~?5*}?hN~=)tL{X9%J+Qpj
z3NH-Zi7PR?$D?Z~KKp18kjGXaex$0t+V0C}5<a4#Uc_&f)A4%2S<j9`)8nVpcjOc4
zOKY`P4E(h6_i+|;h0{ECJGSM&{PEW1g2p{}%lenOV^I2EY9lVuREqFZgE0T=Gy~Td
zQLLKvZbywRE)xFb-tujRMxI}tdnN)yQ+SuFJzgvD4Bm6s1}SyUcoNNk>zSO!9g0AZ
z%@0atJ$86mmbMRWJL$AuhRQ1``qun8hwz?Sp(XA-WaU4cj8}vDAbn5;4KD1*pptCz
zM`kbUFPh#)jos^?l$XA7I};_d_|b;L6NyFCU%{tTihGQU{tVD{3@?k?Gt~)BdFvX+
zydA|-!`kG3yiw6$8)VTsEkle<TTASIgk^R=emSL_XWkqkGVK^80xuTH6?e{laiELt
znCD421Q~bU2MqP`)cY_t?pPPHSK;u+2>y(wX2yRFO0KX`OzgTdgy@dne)gWI@q*1T
zv8z_vZ<q_tOz#P3`_ie$=)2ZZy%2T6>5bn$pRD`phH~Ju7i6tHxZKTAwg~1-qPIOc
z2s@l)cE`%oTrJ=AtQVX!GzrYDvH4x?_{nzj>diuZ*<pb@@Zbs;tUp2E#|qRAv1pxu
zZZ0Mh0%mC!u5np>`^_#dniD;|5wz(3*@I9*<Kg+nd5K}6WUPq$O(j;vG}yHM&Q&BF
z!($6JWg2FgEsz}WnY?0|d0_&)^AOkv8~?(^A0ta1miAy3aj0@q7BRW&@dCB~nYpez
zZ6t3O`D?IiYA>zmXkjMgOZ`?-$x;Y*>|@<zKnVW?l}{mF(b=ZYZUBKSQ8XtsRUWl(
zu;irmP{jZP+3kQ`!S>c!+IWFsZ^GsU%HnEDe;Vo5!y6kCV=C(SSe<$<V-)|5yqs<B
zQu)l?)?P{F;L_#o={ob+7$O~McjBz$ylfJ<Omy^!P~Rp(Cl80Vme<3HHy7Awl|IeB
z=dr<RUCt}qEiIT-K@sDS>zAk?D451#0oq%9iB~CM_=s3ERMgneW?^f;kxOf%c>x3A
z?XF8m?da<AyJR|J+KT8fd#HQwtzR)SY@HpIrpFs6<(Y*Ly14LG7QYR7uzvnhU8@&H
zZDSN+W%HQptADgl*pHE~iw@+#J<#i!$#-gF;>Iqvh-t{CA#hVYLoDq}O2CI(Zs@^=
zn=RhMvWC<5Z(p4JQNaLg9xJ?}K3q7>>-*IHIJs>v$cOQkI1}JftruX1uxAjdv4`+d
z>0Xp4bt2?+!@Cinp}OwCRG`Ex4bDD^IJL5in3#K@(2oZFAZ>s29S8Eh6OgWYX93!8
zA#l^jbtnKUGM`j8vU{hHts3k;#>8c~SGC(``=0)Vg;qgievPxS&6ZQ&{Zm`;ZM5;$
zABJA)$v3?>AyKp`F>QO!K(0ubvHF?ofeZNU!inEiqZLouLV_pf?sdR2u5Y=UP(WUX
zbU7R3$wiDBBQwY|FK4F{mYy6GD`^L=b7vKb2W^;5htW^R+bH@Z;<C(5XvnMDzNJO)
z0a1~c7%I<2cOY3axcp%q`Lf5~|G|(IAzXGW1q|K_f3qk7RQF!7v$b7HMN!|f)}byT
zbH=5r*&p$Ebyv1<b6<{?`!v?_#{F39r`FSO9)&ug(m+VqmnG;u#1qNO0e{vI9SN6(
z5-jKyrFxjRnH4UP9r3~iFSsN{<IO@zfZO3+l(Wa>Qv&hb-atSHsp~)WbNeuM!AhAA
z?@w1c!JXXLbYeypZx|TQALHt<_*=*FFkVh1RGKshM=zb@DY?`_R!$KaxSW8MOEv@f
zF{%0<_8IVy7|wo@F_Gi>d^%coRnB=BpYb||qR%E&wgtqVC2x~0?t{8Y?>QheoVM-0
z{rWNCmvgLImA?vGViOKE3=A1>kNlL=wYBTL`401q@MRpxa7{swK%PHtbg5NewgpY6
z>}izQK5nFRhEKV^a{tRHd7E@Sd7~w6m9&g2qkUIT;}a5y?}#eN>1&g5lNJPhrN8a-
z%4&O)6hJ^jHS#lb_k}_RRg<6bJ{7+vq~o9Hw(_lr6Sqw!`ObkT5!HP)3VSAgn@i*K
zyBR{{eu&q{m5lJeG|<1rF8#<|HQLO2rkS)uP)9IBL68`jwY84CVbK9Xz0|!aI+RYX
za^uPt#gJfgaZ{0h7T$U0>v|xc{#H6rLinLZlx=rCT=o&*>c{$%1*0Dnh^7XOy)r93
z9e=~%?2;ZU96N;gZ|cC^Q1e^U3I23RF8v<K@I&u0MU;tLOuTy#eG~da_od&qWwDaC
z{65uC!%R{g3F6Lcb9j`k2BOJA4YSmr*V8MgE%8K*2w)ShJfTVP@?wI#?P0wjpubVQ
zwUgjd`_Cs^v-?9{hmqYI!N2=sD67PtVV>>RI~r0iMTWQK$T3pWV6U=M>?)siM+Pup
zlYCzvGy`ZSets4If;tA{_#F#1lSkqLD_^$o0?X$5aj;XTK(vNkPp>h>mj;R8{icS&
zn-8cKUpV!pvB*C})rD8o+hTmjKoZ0cQ+9a+>hd7=yCraH^at}Z``p$ydV)8x&J8OA
zW^wo1Li=UghE>9jy{uP*YvE-K5{oueznK;vR7KYv<r@1azM8sY=Kq=~OA-AZ392PV
zTnm~J$~mUJL+4vg*v-DZ(+5@a)i)JoI06OimC4F%9c)6CFajiHuQng?(cOs!E3kB>
z{@8-M1!wU*75KqO?u!%8;8@VDa5?Ps(;!wxDqO1Vg5KJ+B*IX^QhawQsSZG@7^=d&
zvKweq!Y@jZ%K6xR>N}7G!<L-3vM9l~3W%zRq)ktBI$nCB2T{muI{9~rf|DN>C1H1%
z^_k1ougw-xB*r6n4<fzh7#=s3X=I)o7efqrSasN)x4;WR2n;4tuLIV>V-fH9iyI1d
z*0D?3q}FA|H6?ZdN2ud#@|#;1@dTLc<iT5vT(t!<x`YfUZV^OfA^E#X#GJpDy9OT&
zUzMZ%nZJ)z@EG=h-<8Aa%RFE&)iIu&;kiGJc7{c|t7tRxE!m#05iBoD7Ltv(^;jH^
zdEbGGOMS1f7`<Z2n;p(Ve#As4Dgk?7&zQ14K*f!(Ubd$1naG<l9Ig@7oj(P#a{55-
zS!Q;ngH<YZEgxrK`G@pfU%tlVI}~o5_C-SxP2nCr2{DHRCVnR&Dcn%?2)hsR290v8
zGVG$6&csg2n#PR|nw7rlB=e`mNLud?-#lO6mpl>bIwD1{rQ@UmK%-Z=Z$qC-W)Y+R
zQhAqdkd<dA!422S6e+(mnM=R+Qu%tElZ4^9-kVN$g;Bx7ac=tT%Qf*I>kr=Hx@~C!
z3#6{UFY7oR0z`6({*o}--*Q=Wf5w8IkrKg?<1AcIcV+;=EY#}*$h$-*EJS&Evns@)
z;Dzp}2IB5MiMLCUk#<7n;t|X7lsOD9Z+3L?AZ!jQWcKo+uj@f4tY1?6I*oea7CQBM
zQt4%SH^aYMan#f$A-2t%U3Ea7<D9=^+rK|7&iy>#fb2qkXx{c8{sl+5c=@7|eN1{1
z9-X)<nYf52WxFe&-cN&0e*CHsicOu<H_H@4!@Zs+|J8qPl4L>JSk^|m*763%IIh>A
zl(yhG&8J7A_ok`qwg#`)1sj`^o=bj3`k2dPQ$ZvorMXP#z0#}R1Ud4s$T%i4t^RHt
z`n2+eEQfaOn!trsL;|Vv{zFE=`TR3RL5Q<;n*Ovl?>GyG!J*Ci9kF9~%635TNbDV=
zh6*U_xc+fzseE}$Vwu6GDc)WQXaXksFFTnw#&DT<J(uS1b|%kJ!=bH29P=?|*4X)z
zOKA?cFESkW6+E4&zJGlDb47mswx%lAb@X`RzB_G-&X5ly_@n=K*1sd+J(NODAU3Ef
zla#|!a%t|91HU^`j4Y0(;kd0)xROQ>kS@?~K2?>$bWt*Mh%$@3Ghav07(=IcuR~_y
zuBq?Q`)T(LM5Qi=c}IxhGDuU5%62!r%!Y?J;+e^^Jsuw3`OZ-pg&L1}E5g2<%16r9
zR*9#t$`4ZYQ8p`U#%To_P7#ADhP^RgtRy@dUD*ilvjzL+7Ok!+0c~8XTNJo`8S{m@
zGqV(iB6P%G$Kx~s<$fm1?*=xACm*Yx%Zdj?{k(`B<btj^!H;eWK^+O82a)!n6gOvO
zw$aI-Ss;TRJeKJiv&iR=-5!{CQnm%qNk1E!({}Jo|I^P4ojCV*&z_Dk<n|Qcjy71#
zyFjjQRhD*{1&KD%Tn$Rh6?(`Xu_ug`N^fN{uvez+h(LhBP#PGEHm^CbaMt!GWMw{*
z?F15EwRJ8p_O|wEM;A`x;ePo+l#5ltE*cx!jMynvOH_Gy!y$Cp#8Ok`LI2)Uci4(2
zK)UkdBE2dLkNd&&;BTIh`njt3D{m<w8={;X7e&hwI;rZ}_O>X85rnuevJ5|1b;IFN
zXqK=MH0AmOZMJw3lDUj6sZQu}K)o4#7G{Ur7^hk8Aby$Sre#gQZ=btVFsk=BLXpL=
z4{3Ns;0B)o<fGmvTz``?8R~<qXrI_*#PoYRot&+62twy4B8b6@aRAO-Y2&FDkOOD&
z8L{1g|5s2P0GfqG8Bdl#R*X*;ML3XmOW%*yb5)lkLOby-X!gI#EKX52b&ZP_$MH5l
zK=&@NtwC?*pwCcMTk~Bx^Vh3TN;@!bc-u4TU>#3Hj-UorE$@OHjl4pQww+HzeJdEp
z@V9oYy~g^YY%tMt{jJn3B*wCCD;?VkLSyAP12ME4$e11K#RrC}v?op&v#UQir>U~C
zis;;`<%D7mpEE&I<FBoB$Vgt^7W|v`2tutJR_3q%_GfBOlQdz68c?p0SgJQ+@%Pp4
zUU`RxhpSh3R~b9us=Xh*jUT-9+NUZx0{O%YrcG<)R+Mk#H;!FD@@s$0vtaEJaoq~8
zE7y`W+&ke19$xo7$EA8a6kG|{#x+LMvFrDMqH1cEbZM;fT+jv}$EApC++@85?#>Yi
z;pK;m;D$Z7+=uGm#ndviMCvDcTwGkI-D145Tdr}qxa~j!584BU%*=gfGK@L28i<%u
zhMf<MNL#y6KHpij_N-Qw11k8SkROe5?)z`2N}bF123xq<A)=DbHXF$^5&K?L6}6hA
zIAJPtJ)F8YWJ8Lw_cq_ng`_A4Q|x7gMpeyqYT#e=+h)ha+AHaAqmVv~g_l42Ui;^u
zMg`damI^9jJ#FvkXkWc*Zye~vm^PZbD*_crEO@$Jh2W%yuuG&8@X!)&{=U9jfN8$(
zpAZvp1Tt$5i5G(#p9F&qs+!I{4x%>1SFN-9ad{4(!aX`|*0n|fDv~evZ0X9__{|`1
za*)S~how4e1oho<U?zOqcL53Ay{-+$wqSiW<=U4JA{sYV!D+d4gz_+uiwkQ}_iuS+
zdOvaDe#TMlEu8kpL_0koz{Hf5fqS13QFcai{D(%sdlG2VtI#^gve&Ai;1it>4q&PZ
z0XhXC@8F0o;afo-;5wRe+2QMmMy4+-L0jP_{}g^k$MX@}zWPRzgw!97Q<IFI?N#3*
z8NJ@KMhn(y9IH7es{1xDxUmYTczvxS|FkxBC{~Earz@fTZNKT#;@0i@Znir3$n`5B
zJ)k>INYw4~jeVMfn(z=bFHCnqE$t(hLG^8aJ^&9VpK07HgAgG;leGmJj-|Y|468C&
z#0BsChp*7SMRVJqx56Jgh9rlxATR~{fe<^P*Kb{Wp0DJzO-y+bLMK2B_;O8txvyxh
zUZZNZzu|lS5T+{?!IEXL!w!{awDByNp!M!qZ*VMD@){IlPDCfWQf=pcTDIasSw6lq
zuN}@`sG|^#tO(midzi^KVyKIyW<15#?r}c=SU8XP^7ki<K0DQqPr15U5q{yyAQW51
z#9}ld>~!;a$aUi`R|gFK-KbCqd>jMnI|7YVR0TWXw&gD4W)DRE#wmgL)rY$WTQ#<@
zj^Bo_{L~9pc0ULM`$@vUH+u=xzQl>~a!#U}$CQ@;3ztPtFNSOpbMSA{tDsi>OceKX
zzD$R}T^4?3<7ht!a`^5&%jRv@InwI1e3i@uovwjC^&5SW>S{f_$*q3%=jQVvst+W+
zSZ#()5Ds0!AD?!A0R8Zh{yhd5(|>_?Yv}t8K)3Bei&iZYsPo^TA#nS1b%F=DV=a1Y
zE@FVJPOuhaE63vxj<!I>zR<_XAC1Z=`NcFt?eiqoHxHaBm4=K`brUi-$sa51{+5ug
zD0s>I5%?s#V_K&9BVxq)j+qhCIa6bc8IUI}K9X|m-kYH7rIxMn%6-$Okgdp1dhD)J
zeYf4IXL+=8dXb|EE=@d#NYv!({Gi*4pdcu`Rbc99)Xz|w;LGgZ5=u`sN+H=Gl~N2V
zRZxCm_B@SYyU5ZsoqS@P6o-)Av#i(tN$@o5{^&4I;mK>YyMSPS$c-Ayy#((BW4$W1
zSoL-?3I6he%h;#)jlgxky9xv{`PGOO(T;lS=Ijer70(z6KY6(4**d7TAg5Fqe`%lf
z2BE1}b?@mUXf;g?M<HJiLMX-{QN{Z#KxyZYZs(Scw>f%hR`GAmN+7vqOpiFBtm{zs
zGpYjkdMfC-B*Gz5I*BQz_5yF(d8DYdxW@;>dvg|+2t|p@is5Sb(CvJzur=f2cl-x*
zc{jiV|8M7W`Az8Q==9wlvGjLVFo`7UC@I5_;tt!(iO_4#&nn;THo33r=H|;XQd7VA
zROxabz=iYVNh(g0DfqfJ<~CDv*G}+oq;YST)n7l3gZuIn&@uyE2_S8&j*b@sbz;pG
z7fk>%K8I%}q$%k+54Omqw7hU5M@J+%jQ>&82yDt=*$jxaY$pmSSbU9a*@V&+RD@M2
z`cU9J_zXLrP=j}U5wdU)PX#rGV{;-1%nAajdJ_bt?tV2b9x9K2?_wZPG&Z!~mBCaK
zOZo6TM_NV5nJoNE1?QUfYD@h4*NnBC37LlXsn6e5CR*DiTx+}jta7;)mnt|~-crUP
zS-Yp8UxR!>tvhHDVGl~nl|nX{AkG;Zo@N;oC+rDnwO@Wg^{s6Sp2zM*+Gncn1?T<>
z&F^#@TYT>h8THy<O#*>pL0)=xSBDBGHb?D0b_#`@dHmF2GAFgOd9>#N;m1mh(v<Z@
zT~_78C|b^JjZ7p?bJKTUU&u|9qA<k%09SQW?s67iYinhk&uQyZ`X8X325V8|pwPUc
z4V(Xd=#Su=dzxpA!QnveqTs`r>M&O(X~0u!ACPnG$MSaUXO>mXZJWJIC~OYed|ZYI
zrgv^Rd(`m}LgWG75<YJN_`R|kd4%?BDFxR<E{B7Fscb*j;`5LisDpwxs(P{@rin_h
z#M9NL7M+YpHh7ARd$Ub>`1d<@W}*^In3L2E4#@d+**_Rpxoenh>~%km<SF}?0>lT>
zwgDTrJWxn?9w_~W@ISqd)A59wUi-5p730ozX;S)2iTUWS4Y|VEz%sFE`|l0_RmcBe
zS;W3Cj;V33U79TC{5v-m&)6g7&Vq*n>K00L6Sg#Na_~nHBeG#N+_lneve)|X=y6nM
zS4T(GGm}S^Ki+IW73O^}W0p($C|Cu|eO{12F$S&$tT?Ubf1of=!>5(qu;Fhb3s7-X
z+$z*|`+>AN>?<HCPBYIoz}KP)IF6#Zf9e&<v9`A!8Cdq!q?{Oxl(9XTvs{`=d+hUy
z)7c_P#QTJAtFbw$Al%zN<vq?NkU!af-u8v9VHJmJSuNq_s`0@eVjSFK4GEJ69{Rys
zw%Xr&oh)!Sd0}@S%Zq_UTN+DO$#hq`aNNT`bU^J$mtizP6**bp_#q+xp#u)ZQpRET
zhO>{{@{w|E3~DH=SJ?ZE=X&&HAI>al?;nA%cg^F!Q{D&s#iSBKWFO5z-~0iee{1Tx
z`aVtOi<<RjR5z|n$TtzuT(txHZV=d{!(~oZp9WAX!*51+2<y8`aEQwxKh6`q`wMDO
zU;K_4v9jI@!(lOmc+^;@Q=Ak)64F-2w#mAqZ(O(b-NMGake%wv`31w&5}1+kJWcS{
zmqtUJvhOxd*cu@ZEa%ZMenP-?-x3%8RR6Dp@($>vJ@P7(SqYBrhCP;xrdRtfLSwfo
zX-Qs^>4@~*_0%<}8q>s3)gFE}{71bZC8ta3zZcvrw39S;S7d4#`w(;USHIEldnNE6
z-kEW{>|2V}^`>tQl92G?6q&OeNYn0slU55S9quQRNZNz5b(PL$Jrz(0dA8A*uYYTy
zXb1?J{<pMaGW<o|*iYPXyW^6``p5M&q8~lL>G^{q9Wd^bLL`q&x%|}v%A0GL*xJ#A
zQBv@e&VLmiv~(Bj<+x)WDpAnA1KYC&)T07s+#wNp0FCbNDwSrlU}TAR!gMGn$5y_!
z^6k{iDN+D0X9E5C<gaCsHihw_FSXK5HbYrwjFZGXx}V2o&nOJ1*rZQxjMKaALTNT@
zxQ$Q$g!7B4`iZ|E7;Bd<$Y_g|E@P|W9wiP0>L|!j10au9cLICDOxdM1A@@X_d$}W?
zHAk(h56e$S^X-HdzlrnEps)}DB%>&%FW6HU9K(J$HM2R{R_wv)_nz@iK_T~?WfYI)
zYUcGNv|{#>gJkVveh}TcB07kj%+-2fhroQ(=FD{Z^b3b41|wUI?#EP=fNMMJx{oo@
zkd0sxLOI@>3t%t+YBK{x2>J48WW&RQ*mdrjSz=VjqcC3uQM0s^yZbuY)QaiOqfRCD
z8tWOm9^<ADnB-d$If_&wVnHW=TLO<Q(`i%S{WGNE#YbH)II10d8%6+H8Zt?vjx7_W
zS8K5xr*p%hNW{Rdnct=j5r-g2Td=d1SI7ms_LSr8hlH6vNZs7KfXMsxTRJpUgmP;Q
z0`t7>VMtA)%?OV2oA7>Xvpwx}`!)O0O5=)5s~_eh51Kb3O6`lRpp<jMGSN683?tbK
z=E><hIC=L$zt>4Nvt!I7bkHtVcKVqKl9Z=8;JZmTZn4zzZ~h3XejD|6VkZ1FT;D>X
zNTf#UazMeK`iHj7lI!<t5zxU~T4?sC-md1q=N}Gm8ldFaw$^I{LP@(moS6KC)qaTm
zyyHdI5|DxwJN3za_CZ6;Yn~y8cw^DqpF#k0)P~C3w^KQEiM%@1UA3r0Snywvt)JXh
zm|@b$bNJKB#7%m_9K#3sY>hjv;%or0Mxa{FxxGzH?~2KePdfNpe6Y-WNU=_gekTdU
z)D1QAZ{{$EUPp34^8Sz5<jNl>wyDm>y=j<ru)=6D=0uFSmcxAIL842XDfgZJKQfi!
z_|GwjFY2j%GPASiQ#2*ID3!wicr*N|zmrV^dC(gN0@|>=Hs~+zrUf{i`|;knISB*n
zB_y=Q^Bydd_J~bd;QNnv)v1Fz5yj>?Br^uxIrji0?%F|jxpRsV7C|@qv56h7%b@R?
z3u-u>$bb$!7t&riJ(n`RHL8ed%{sdO(|b6LUwz>1#b;j^;l|&d?PgtD#1C%<Tp=r|
zFcD8#D^Xkg_I<xz3JAm1+%Grj@$Ais)U}ZH&I54cgVvJ)IX-Z3iqqCLZEl#>KL;`7
z+Wg(WpO_=&^TffKZ(6!cBgr}jgOv$-xpRh$eu>U?H@!;NFdGIO{a!Y*1DrWuhg~sE
zOZ-O)Ec-@mXdSW72_*&R!aZD9rJBDXF}To)X&;avTuX$@JNvu1k0XDa;+b*KKK?vQ
z&=s$sKm!2M?(*1x&z3$S`7~HbyQ1A^ljWRR9+KGX2H3w4Gb`3-JdH*64%mMLZ}T;7
z)FPWAYXnem1A(<y(C>~=CEkxET6u`4)3+`EKoN3m>tyaEs+z97hBZc$>+aMsAkqIS
zd2PXT2N6)#HK^A1eXiRg>@6Z2#hAss@X{ipuBZ>35Y&^4j6gwCd_)xNm10XNppbU!
zjuC~Fw3w|vpmynjjQjcN=?jLxheU!g4IdlN>SOYmE}Kr%01VKUpV1w8!qGGViMR~Y
zzX#TWETSU$sryc)DZos_a_m=|@b=M&R_L8?gi!Zn20eqi(~-Bm0dXAAwtvyKRNy2o
zrML>CKdnFwzXaryUzX%#FKy)`xU9~L{FGfJ37_4T_fjJ1ivsEXJs#yZ9$FH^kp==L
z)m<IGuy1Mj=cGv$aJ+O)<m9GNr#+LaO_O`Pf*K!jCup`R8bc-V@-{;E|8?@6;c#{R
z-g+5sC3>A87%iCGh~A=$FhaD9Iz$i=5+zZiM+t^8LG(mtMhK$UWJV8?32A!oB+76$
zxu55K-s?K&!#SVknmuc;{a<UZHEXT^uk2ESmJuR25Tjeii&!}Xz<8eEyIpA1OuvBm
z_a;~HNFW9JHUM3F)Kha&mp<|;CqnN!Dbp*u(3VXklawrMU$p%GPt!sY5<ZJFUjUZl
zGdW6!H+{&J?As6c>+AwudI@eriJ^g*nhjbXV#q2nCZ-Prcb#%G;W`>P${@;^veO;k
zdFI$%a<L3N)Y&Xa$^Lu6Oz&wRSFtBq_||L>&#rpq{Roc<PK6^Pw@}3_9dc=5Kj4Y{
zyO;Cf@cG#gSuG}VS2I$)+aPaaaLJt8jqaVHWTpaPCa76>9bc_d2Y>tpKEBr06f^mU
zjaq8kquYR#$J!@11xw4KKEMje`kW3u0V^M66T?;zI0<2{4|N7Pw+tJ;#v`BRX!e<R
z9T-H#?Vh^D)tB6BN5|th-cttawR3&73xOey4mQM{?J$SopYZzCw~D1|!4<z_7Aj?x
zYVGL@C8@M{N%2c>#BvaF{h!W=D*;txS|a}tS8_CpJjaHatelNWa7s&J@GrglX!Bmi
z_*X)`A2A|0*MXDC2R|8A_P!eTVZnz@91h`Xb+twJ$tq@1>pw(C+zA}kcB9-6%B<}D
z!T$O_`I||X-GfEk-G_a=+-}#V4q?H1vG(Q@MxA9Gf$zBs83Cq9%-dia=Qs8oc8}5A
zz6s>n;VCh8(mU7?S?ykPw%vy0(9vb*Az}2BklXeJH2UaTu^k3!Ujo=*Np@wZY7;6S
zX?zW*(u=p+*QN;z$V2Vz-%Cv;%2aPT!i5~hJiMC&l_n;xNRE3!DB?qpwPb<Z8(9V!
z^B^@Lxd)tGC|ZyHeIJ8rw9!XjJ3}5Lz6Ru?3~(WtpMKYrbUVTex~91%jQx2)*4mNY
zFVLw1p2F$@ciul=fO?6j*{MqVVj^5ZOwj#mZZ-WOu?qb22$O-3SRGZ*o6`<~VwM%o
z(U_8$BGdN?HwwDFuk)k{P~mQZyT2Ge31~zStRVyb!Uv4)Snz6ZZF4`M*0Bt3e<aLc
zNGUPV5g++P1Q30R428X^(Ne><XVN<o7!A2>PpGYh)taa9vzmsXkFKCc2g_b(5EunW
zrp@4I$Y?l}C;e&|2r|*3o$}l$Ra@XkQ@&<DKH&q?6={~F8pE>(J5qYbg-T}--`BVL
ztvx0%?)SdE9Y<ag=ESEWaX}Dx=xpRTyW<X2mJ7)3Z%TQtbjP3`bK%siUDVlYNn+22
z(v_o*yH6zRt>d0nhNx^kpl8(4irod8>D#3sb}p~7G7hH!292KDRezg)T`N(w=C63u
z!N{82x!%>jiW{qv7e+;`Mb(D!OhTss=IX>(Rajv(eFm}WJybtbr37}bj62ZS;LVk>
z$Nm8Ys6AKa)a8Hp10=Dgg5UjuZ44JUUxR?OP`4BKfx)d}Rozr9*!0S0Ns`X8hgp!c
zQAMg2df`C*_ax=+yIZT&D|6TgMNT7s8Jrc!<sAQ?sI(deqGIh<15g%%Xkh5x6O1sR
zd9t#H=6|3oQjU~@l&L!xGJDwmqzCeNU2u_JHJkeO_obmhtqr0_jzIol(aH1Ig?~`|
z&mcDdq~^`wJQ@xeP}@2xU3fN7{Ff-8q2kXHQsCxpYR~w*Dk*@JYsT;mo-wDF5nB6&
z@XE_I0VBaOM*^l)w{J)KQG3RDU3ClXRCD)?xdMb&WzV~G<G|r7<1V!OeN_x*of^nJ
z`ZxD8&W^y@kJk`6A>MABt;P(4_{dZ+pj3J}htNFP*3#dC9)wBjpxM3+YoLnL2Fvd%
z1#!PM%+9@Q{fjP1B2mYImjxfPxw5XGM?Zc8F7KexSVG#ne7SH#LWTRj@XGTY_Ir7n
z+>^21)4QV7^V#C_;@X>05AN${V&P^FvAsskfJQqjB_-vts}@P5j^?3o&BzV-KBCdg
zo_l}%RaN*TJ%?$=fX}-RvoedGSe`f^<1ghu&+^*%lcbl(b2j;`+BQD7>KpB+(uEoq
z{ZI<|!w-0QiNr6S8iGjzDM!c%fKkf!?INI(Iz?1T$tQlxl8Kv{2kk_Fh#5iv50!|C
zo$Oqu+5!~&hedt?mP^*1xlR4T4==Hd+7x}a7c|w4NwWKy+*8cL&gPRYOA{{y2e3>p
zLKn*D=R1p+X~ab}#!d*2!q%ryuD<~m!J*Ju8|oF?A>@cH>u6UYqlcnTlgyos@Zzee
zlQ+4{Eij4QsG5{ZZ5Q_z4>lHSyi2R@IrwHUS<i_sDO0)BTV~UjN#{JzcnJOBv}Je4
zjoD9E)U@{tl~0s%J3jlnd~e$&S9J>MouYFJ!|c2el(4-2nCkAg9CEYgY((bIi_+37
zhSoMC@?BS@h0^Z4;ATsBS4{m8NKzGB1<BZGY>@~s_wX#O%OR`i>u-`>r=VRKD3VH=
zCujBJX|lS+t(TC1Es}e3oiYZgIFAs=%*N=W`v7cW=;^Fl!Nu~v^L=gx{SvA@8ps;I
zDq@Zf=R$i934*>yb^3S1rAavV>d2{ZsfkGUNQ>^oqCM#Dy%wruF33}6q1WF}nWO0S
z5~|?5TXuw`h~Fp_*8(B~8QycApEB@9Rz}ngjImcT7W&;T9C~X-`~hRJRu{D+-lx=s
z`?U0nLVSnu`hR&qf(4wDdSWqUf-gp~zZ{zG1iT}y^SUioD5U$yjXyYyLAF{djQ^!i
zbo%iHazsPwMjPDqm*-`GU|}1|etLiSk5;B%sYsvsmilz9p>&RS#0f%7$tTavj;Npi
zdCQj4^|wL#YGxYgtyhscto~2Th7c~67VXP=(Z83X0D+>MGyr@QKOW~p^KT#?&HbCB
zab<)2ium8Uurv-!EsAxnvKTA1FJ7S>dM&G)j^r*ej6D8SwX1rZjuHox9S!^T(LI5j
z^6T=gxIdK2!^KME3#R&1Mi1`IilQ|1aZ`Lz+Am?9o!9YuM{P6EZUu6YitI}n#LTxJ
z+?xA!rI@mi=wIRaNCWq6K_nBi-F+87l-|3Ijp-7U7^64)!-k`KG_D>;)SrGFO`1nX
z0twtYKIIk%_(U-OrG8FIGfk=IkP?s+z*fi)ud9{-XcsCsXV_cSy;jMESWll%%+7f)
zWVkVKsXxiLQl$XJWY)zm)wew>go`L3)SINoss@xU+A%7BN2bJi`!O<)5$~LAZBt2C
zIQCEkGHF{ppYJQoM0NP94A4bn2L`)xBS}A;t`#zOJ&-(Vr|D5Q?sd~k<M>IJ?{fDJ
z)We4~N;_h}rpIscdykoP?aG|w{MAPUYum_=V#!-j%z9rzwQ9LZYqtu`7+H7_<;Dxh
z6$u(_s!5HY71#5?&}w3Nqax?C(9C_p$ZV9E{FzHYk|UfdN?RBGudt)HExDFMJma=t
z($0ID4tMma?{epP=KWSvkf7P_(ViL2qo;`r4H-2lSCfgJ22%kwQ9=rRYk%a+R=lNS
zWw7?V%l0uil*2!O3rF5!TmpbHGlzyo_@|IO_E-(Ba!SFQl(G^u7q)OTfaoJC)TZBp
zzDGTE;FC&_2SEHtYx_GuC73%qj1AyE#4okB(E<O&fC}_6z^Z^9Ay-0ILk$(VTlzy~
zx4->At_8~eXes}^=t6V9MCAFpmO=b+Bi(DWLg_&DFazls^_uQ%83Rz8&R=dVJ@_IQ
z3E+?arOYK`YxRYhJKmY7@Cz?#p9&w9s+k`HXpVGI3pc-gK}bV8Pc!`keo5P(%IBb7
z2mT_E|CPHmK!*jN!&dt_=h)!I_aP<lxO6S+Q`!goR|ITRR<4|vZ^uVAfh!TqQQu^x
zg*@1vS8kq{r^y$^L2ZAoA!q@hkVA;E74Di|x%mhOEY@1AK$Euuxm?tBKb4P{V)*9}
zy8i;1D%#{8enMO>T}vg?WC0peAsOx#WL>Su$P=aZWE}75XBnY%FY#*&x=zPbB=qGb
zsrZx+c77?<bDZ5>iap+b{2=f)KfVd<vfJmO;_)Bo6zkt8_6=RF822V`2G&CdIW;7l
ziv(bV)GJBMxyn7-;B?nvt(Y3<e5@*kzsA)zZgAbn({j^YY#x?8pG+v8VaU&l0|(sd
zqAj(_SaRQi{*T~QGn7QI2k-gNGG2Nqf4y4KYQ507F4f3TG|g?BuA%78E_7cU$ng?<
zg)2fD6c_THmKZL^r*@yt_%`rRM5*ZlbXkul0nQ}XU|v6~a?-<~As22nmTP@aq4nO7
ztMj8llgn|_vC>!ABqed?f15s;Zj5FyEX>LqqHw?GpvSDW@o9V|m3`F9$KLm;Nt<xW
zo>Q5_Mxp~`DWL0fM4Y-s|JvG3Q`{>T4E*|VbLl!|mqz*|*66LFEaQMNYxz+=038HH
zpt$~f4DB(3q<iE`=MTg&8S%rhIol+s#n`J<6hE-qM<3GGR<jyf$Dhn{?D@f67xVsp
zgwR#v^PHgUFmq7%DWz0gSqz_68^<-}?F{!xMwKhmhu%FqQ)lf(^|Bj_H4TPcp)ULY
zyNx*NnbxIjyS%p;h}j~*`1(JHvy`HIPHtBWo<kE6uLHUC-~a#6-9APOY&z4WOn%W&
z)KnL1q#gTeiDcyktG$*045Zcz$wG1Czg+IPG%&0gm&hrNG;tUbbUGb~oxU^EYUbE!
zy?}up%S=VWj&?+2MI9^-X39V2N3oPQ58bVtU>D&QllW_7b>=t+dc>i3Xd%P!rfJ>t
z2>Pay8w?mZB*ctdPxK=}ald1wiv8Xxm>UOOgb7SBD?wy&iyAE5y?zN$uPxn~RzM|D
zzAJUv6iOB~r~QcR`Ug?~Va~nD#nNHPmROGV^TqMa5SR-An$`S2$l_0LcrWB5a`ol4
z+u}O*c{0DCi)G#Gv3$sDIC1qtH=HOtlJ-dVM)?!XU8oVRo8@z!=jCG?kk8)J7go@m
z^n;`2nAbssXD9oF1v_aWI>Xv}&5uP)xCBD14|f!eZh{lFaGqqE3x~*y3`$BI{ZZa2
zm->eUsJv5u&UQpVgyL|Q`QD7R5Fpy7Hg0n>!h~ZjK;5yN%OP*d#xJz?EOlo0@!H=p
zn75REx`H^47`=AkJ4In_EU3LmzI4Zs_N{G46{`4BrGT7)DD{|cVlRW?aZ#s{&>QQA
zxw&o62Hr)yHd{XPGx4#1V$d6Qn{Bs3Y}&%jig@*uFCgQ7#Q8aODf@aR$|720=T(^Z
zjd;_a7E71sF@DyAeHeZ-+ux#Xume5>3|I->gx`no7NnJSN*iYSmb@bMcoB5_uA9M6
zj@=IOI=`umDkXTzm{}*EjGw8Xl=Y!k5g;seD&Ke^a#yXAO(vb?R~`G(%HaKXgfs#p
z=$i$p(UGfEX2U3-%S+5HmcPBlC1tEQE>-M$=@EmuzGK(>b|@|Iht<(+`u%~N>`Xm8
z-qOoIU8HNvYcJqfC<(~`)IzlR)upmP%F%W4vjF?vo&8bc7>VZoszE~tun&J9a;Hk<
z?h$4^=HZ|=%_HEv*W1i9*P^4Ey^4T%v$?2GSDVx{U0$2#AOLj;^e{_~cEkq_^K3|+
zJw`jQ6h%u&=B^9UJ{`n&XU7QK2Ong4RFqJA%wQ)_q9cod4TZ9B<*hT4!KCamAdJ26
zxEMHD0Bgb`A~wYlF5;>_|E`qdwb?-O>=dzWtOY<MtTIs?o4}#sp#(rTNvaHzBDBD%
z_EgSMiIo^Qy#0&<+x1)qLBv#R7gfwEi!A`~oon=dXiAE@1%`}YEeDYE@7TImM|+fY
zef4kY&)fmrc<%ic1<M7XZ!D+)Yr#0)>TB`@^kFnnNLO@D1rT@loir~nPREX*e1V+R
ziMZvjr<-n<08GNTH=CMr)vW7WDGCbM{8PEPG|L}6;`0r3+5`SbmCagJ$NOog0M9oR
zOU8fTL0rRMIBR|RY_8(@c_$b?4=f<S*$ZH}M#1OxIJeHugU`#bEtlViaL%m++X9`U
zYGeYNC|85C{KT!sRR8h<2poIzmnz&{2+7w_0ciW(^DC0$tz)}7_TY_@`Zl9H;&xiA
z)FPtfTXO{hj{&mdOCaeT)SpX5#2`%hJR)TDEhW%rg{acy{EGYI2vnu)q4sxnAeg7*
zhB0$T=g$uAcpvu(f1Kxiwij4~?+ZRs)RER^URkBK_x;jg+-i36`Mj3ei$^|QIW{1K
zDh_;954_AE9>C&5e3luX{R0s)`L8mXv6pb3_~bt&cCZ;zvfVz?I}3l}qPU)ZR(R^4
zuz)dSj!Ihwh{Dg#LmA{VHM}qZyhLrnGc#~C%2D;K<M3x47rh+8j1Qqnv@bG`J}<jZ
zg9U@`92Yz{;o3jz9fAVTUpF;oINXtr=S}<yAX~tB4W^rL9>4Ghz!9XordN9Zp9)5c
zh)UP)96Pn4{*))4K}}k~{pH~E8|L1SBHngppPf~i6IKIKWETK__JhXrLHx)O*d1tX
zCBGEwI&WBOJVJf8#Xi0F&o@R|>;0!+{j~o?rU>kpnaD2I|0xTJX`x>Zp;4&kKQ9dF
zqkO0y$&<z8bRM)bT*O7)YAbpDmU_+k3n?Zy6j_}FSU(Rq3txi&+~ThdfL+>(lV_6&
za!Uhpw0#*Vv0F^vLf-<UWv*=h5bp=ZlpsR?DLxVq4M!sz)1UyQBzjY3^hXl2P!Kw_
zT2|0?-HHfwG>usmJ8%I=j_?6>o=woJmjEJ9K2SAK`7Yq?dEWcLZv-UoDFrazPJYwq
zG}_PUqZuWbPAT|hUp)s%Oe(OHnY<BlPP$9_=o`)=5-72)+z{gH=Byh^XGVy1|7ZI}
zz|e8-E%K)V!3=WCkY59b-x><jYBXh)MPqrt0yU%4(j^er_^tTEwDRqWA8^&lHUjRD
zjV9ApW=W?%)PE;GJJAH3BUwRFTK}t$>hxy2I5tyL2uIv_Y?=f)=7bhB2c;CBE8-Q<
z&bN!VR-|ojBLl2a#ZsWGxE_$kKfaW}$C)eYf0(1<Y{ir<JAI<`BG)YW`YJm!X+U!_
zh=p@cmAB7+kqv$FowBP(*QS5WJ)XkkroSuCG`g!CUs7<4#%5m`t0~zP-RV&!@S<eN
z<lk7^!QMElFW;0^R=lOabh-*TEe70B*y_R5{qmd;qo%YsPgRTBx*%CL_8n`g;&j>F
zJ}aeiYb`0Z)gS4mvb~3wChyf?5KEN>owjU#vFNt2mtH)|7DlqDFQv_mMtZ-dzw^yi
zBj;R6ITTl}e)N=7cK7uW;+uAVmCjF%OOjjqb()!e^J3wf#g*2nVxy+1ErhB+gb8Ap
zv+c|re5YS(`SFo`@L>YLM1HiRT=i5olci7C3njKAmYeiK5H#0<EN&0*QSCBr8_=-s
zzyU_8*s^$X0*Rh{Lv}qxC|66zd(kuTTauXAlNi`Ffz&!yi)hnJj^>XlBC$%Z#P%eW
z)9Pbq>Q_e*vmbLG(7bkqgceI?UC)BJ`V9wYOchJ(k9ZZ5caPQ5K@dpvqvCB>mg2Hv
zPD|d}#;;?qEMcjfL9EJ3?PJD;$#}rS6;e*GYbueoZNdK`-+6$Z7g}%zWz_RYSf__d
zJ{zooAo{S8lnG-(f(W@93a$pZrHYEW2u0T5h;ZoLu=4HN9y!Bs?5Kb5m@hC;^%6+X
zKHd5X>_smtM>r@<oRDlGft*Nh9Rh<bawIfVen6%R$U>lkKJkUI0;Z~=_Qb3AS`Ry@
zf<<QxzdIrm-JvSOQP?UW-GF_hQ&)tO2`@dT9?6G>!|-i;+-JiR;ri4z1J?pc(aQ$@
z8oiJ-sUHrBp<}UCNcU-&f0#37oG}IJl>)cFc*V&*kN(!SS%@)u&tj>4=a&y(uK91+
z5Y@|51}wNy_>h>TF0RHJu#-@=3p+Uos;W|5Fr-y+X-&-?6RY@EJ$vn&Mt}u(6f#_r
z#qzoHEOa|+k1=VE)AwgslhA#O7e-#*5oSNqogB-?otnhG-Xq2r`(HaeuNi&Irz5I;
zpnx#3lsD-B?1W(%Sr!LmCvj%_fPLl1Cy%YwIZitvNIEh6fT2>HuM8`wO(O9>XUl(Q
z=8A$@CWTV4;)=yZe=BI@+er#CN-}EXn^wUZjZI>C)$anv#0Ct{%|2{_kEu_62<rd)
z>COt8t4u{nV8}qAIYBXeenS$JNg^-L^s;##%ml{_O!4B(bZ1I$V5-B;oL`?seTDqn
zkM=(&yh<3h-N0ZZ>1Fr~!$aS2J6mK*vNRjcO^m0uJoDpQzBM^8_TI++I=ElayubJn
z_uvSu!)Ge|GWA!L!j&ZKR5U`L3+D#;mrwGnps?NKZNScWg)a#8>3*^K<(0QwneSk!
zV~w>dW>Rc1hur(1=Fg@bYipn5szq@#VtB>E8I8j<!J97{icDnQ1j{$~h0IkLe-Ze|
zUJd&D+l{|aifLHcz12_iFuF}joj|2C-9QcTszbfz=ad5F?f_*=SU^$63^zGBM|-8Q
zeBH~SJ~Omr){JFN>HDi0J(xhW<6B8o;54(4g&QDOr*@e~)-5;SBv6e-48KVg--$QW
zgSil%eg<ORVy|9Df3Si5O%&id*wriIubNm|`4(>ex58qo(x7Po;b48Hp=p0xVmEf}
zcYJ|L7U0!<*gH66ttTtt5=L?F`i1ZHghMZLe^Pq||E0w>j0o`rT$>7VF>k9Je*Nv?
z-wQZ@4YbcM&L+$(gPT=t$w=j4eZw{%9q!-O&OcJaR~kCS{QYVUQ)-hvvPLyRVS|+g
zV?5Z5f}%lYg|^D~?-^B2`TT2U08I2g%qt`0^fs$mICcVbnD&6oWk83wDm70f8_UWm
zUFDWZn+6$pGbR)r!(Dn5_6#%WDWhD!{aQ+#GAL3`cV>Me-r|7u6%I&Aipph$L+Nmk
zxKiaX6pm@@l3dMU$4f<5Zqtx5r}63F!|Tv3QA(51q2i&$!d8eaiD#1g1&DL>ue%1Y
zH?G{&j0L9r#()>olxcs`GmkLm7a$$-yTuflkixJx66p#*&3saF%v`^G?aW?reRp+8
z$)Z18Y(Dlohny*uDEQ+wQlk+*rUe9RF<CLx0W4$0XHsdK?p;7XsOkMtjHn^-&_>tV
zg!lN1LC(P#e)sAX?LaQXJp566KYo$dz{=~YlP22T{|8r2>zYMZs=bHK5=tk^4M8w)
z^D@=-u5I)ZL?}u`nakLeP=dOQjd1lJ!Pov8j>k4)Cj=S9v;*<y2Gw_DrEBvlHt}6~
zPb6>&?+^;1p2Nbl9uP$zWVAV3dTupq4n_hI#DXxW#+o;GRUjqSlCaDoI0t}C=;hBw
z`s1q8wZNX`KA)Lo>Izw`CVz!2%t*<FS2jKwU!j{2{Z&y@$N?ujg?QT8?s?C<r|iij
za{R%&0o{b&rAaknSG{h+B*NJ+OpP2-bEv7#zF3jaHO`3CgYBP^qn+1Q_{-NR055hD
N109oVRay=a{{=cPz+M0V

delta 23078
zcmagFRaBc@)HMplH9(6?a19oqxVsiB#XU%&6fICBxD>bGF2yCd71!WUthl$uiaUq*
z`~LBtG0x4ocrtRa_gZt$J)f+#_Fh@&K~8Q(t_7h$4YZZzq;<WP4}X1DdhJ3r^l$yM
ziRafSOsucVC}k`oAMr?J90ghv5L9!)Ij9V%z~~A^!f7CO<g5HP#n4ZeMHp!EZ;JBv
zT%`dN7D2@qpfCYTlz|+wH-0A*R$m8~N2R;2ydxz~-tN8jK7TxN0y8t|0+8myGvZmn
z@Ar^2q1<rxI#ha;QWoJJF)E}#%JL8-k3DG!5=TB5X)_OuM5>EUOQnmx5j2VvqSG~c
z#irtujM9C={x>a_W3=y{(rerjN4yl*+yjy{E3WcuTkBZKGh9KWYn)9n+-v+{wG_A8
z#9+rmiZ|3$Ayozo<FgyQ=IAXuihk6n23m48befm1zG7vjViAKSM;mh1PF{SpA_OJS
zOavaBhW^wnmn5JVG5Fla^`k`G^3c>6D`u<!9ANf-p0Gr_AOVqVut31_<L}kG?>*HM
zzomQ}iU{z<JDztd&ia?N(M7*+n+TtO(r0b)7jmWJmjR%<mEvxYrX2BKM}4RqlR?Q6
zg92ELQU$!+vk;y6!Eek?V_P%ToVF6GdVcNU)TC=opBwwTbA3d=MNL6&;i#|a{FNTV
zHlO7x5?NZ#uA}lntX$<Wjln5vR^|O|ds6Ou5AYg|3E|J+99u9uN@)Su2R71qaeyT5
z(7vauz_DATzigCU>U(YT=7mo7BeJ1ND7DqC-_rM<b3F(*x_bKuJ=g<SNg=zG{yxPm
zcx?e$7hQAn&rq7*W#3!{l~IN(%G}0lCj--T$>U^eMR^A}v8Y18w-0CQMjy^T3$3Er
zW{*Bt{QVQwRse^|&Sy6mO}j!G6lg4lFf|8E?Dy+|ad>{!EM7xD5}&Ii)6R)|p`Shr
zPEq)_2Y%n~(eUHq>6YZYpz;G2A0{VU^3gE~zu^N)*&wAzTF8qWFxM0L@@j;iRgg1X
zoK`(_tNlysoZ+}bx$v#<a4Cv0RJ)vg`~&zm)l-atL_za(j@FHqf{cmr!rY6F{=ZrZ
z9z`<NBoBZ(zkJ}ZqY&V;qd33FuJ`4+R7gRV^5X@d1bmf&AaRxAN(sHF=>Ol=|7$83
zoxjF@xj4?-kZ-<sm_&q?dVcr~JIakiS%+SRC5u4SV(P@q=+T*qB-D4EtxGUE9~{ut
zC|&_6t=OXUslMcGnyys0^db4(Uks5Jbb3)7MRDT{dDWclcr6xt?Az!8R|ps9ppXrD
zH?UIxv51ihf30{h)buT70K3h<sa19%{NdtL?xSFea=E9;_E|YmMIs|&EVzcf%Wkrz
zRK25)12N{{Ckh{cHdKJy@xQ}8`^I_2hwuS{f}NQuhkVzc-=kHjoqZ*8-B$7HcyGhZ
zL^hXazXU=!PJ>w*#zwx?M+@?eM%_lMa}KKSkEz2#E7dNY%Mx7PLw;_6gAKFtDz2iI
zlRK?eQnH6r?_yun{(;Bc54Jp79G00~|DI8~f3ySMPmvB~^dT3TzoFQ+U*<DRnI<IG
ztFB}-D`0V~)A~5tvLxpt=kT>S3`ck`tBT%*ki%$UF@cx4%If{i(DS$FEZzAIGnnbz
zHywAOx`lV`iqM?nRO%az&fW8<-BFR~{J8L#arICG@0l!9&bGQpVcQBO&Gdj$u)c%(
zJcfis#$P6=8=EVI>G_qdq@!-SmG(kRyKZK67dKgE6|bIz5n8>PTpGkuODjEq2hks7
zz)U{|uAAdQT)976pBQ!*l=dC~)Q=8(C4%I0&In=@#YmM1aeYG)BcwjcNHM`8LQCTH
z^eM<GzRAPbN>efn64J#(ql+4P&#E9-nxG>bRSu}n!UFUJB+66OQa{`jC%nFJxt5*&
zYwz&)iqHO7E4)W*qe+*UG7FX2T)UkN9ATS&_Fcwfo8~h?-czk%=Wh>!JY5ra$wfRy
z&U(&>X*gjS8wc_VSe@QLgwI~$HBRTNQSf`&ZMuO3J{)r7Y-}S9|N83T6G?tIDVVaM
z4HJ}rj~geQHnRlzXXp7&iBPY`=k?CcRjIA0Nq8DPKwHn_5$sfdiURte3aYmvVo&TY
z*<kzz884f3236F=zAMACO&uwrj*T;J;T0&BK_L(qlz<CkF?afQJ^k*>?vw?;_($3O
z;Qsn`XSHT$8L@llAKOXY6z$_W<m$s7KIoqU7^6rK=&N27$Y&?I%^-NlpPdo9%s(~*
zZu0%!fdR4tcm=azb{-)EbpRnCL^Pkj>0N7w;}bJhhNBrk`un9IqURf(2I6gM+&>18
z&Z^NS+_P6qSd<-Xb@O(qf8j&SDsFjqq&h}D_GzRERD|=_>ebr|Y*vI!c~qb*)cez{
z;NOn|O7+XK)w}xA@!LLc6tVftH9Zg`450%;ycKpL+AZ-Gqm+Q7>=$th7Jq1nHpx3C
zL?rt)m}A?B-Rc&MU@Q)8N9PRTvLd1567VNLf;4Kp2r9|98V<>uiv@1w5WNjZvMe7L
zwPt;L++2}s5$Fh2GY!-)x<f_Km`|cSm{shZudK=8w!F?tK;^+;|C>kvkE}x@b(5cP
zE#7-I?<3cTK*$s!L%q3;_AtnL|6*|09KoZCAe&5Itp!~KCU~y9Ud}qN{xlWzS>N5g
z<VOf}R{O{M!=TrnO$7lb+vSI{&p--ZcAi=+W;~pZq4TwFi^oCrAquEw|Jq%@>)Rp8
z7P41FK~a6CBdWvms2xNPJmN^GIe&-bS2k3ti$PH>`CYb>C)L!f!HLLr7zFYAW_rDX
zvS$CdGwX*Xg-}F1%G*}nw{GFp8nX7+;aZwHF@Y#~IZT;n>(x>l(`q?BHbo;z@&q9N
zBdCF>Q1O;WiLoNO^B~aLKhcI&)gn#iC4#gUpo)mzK&Ap#Do66YaYUPiLAR{%=Z$wB
z7njQ%)u_&Yw<-J{sb)F=4i|hHh{fCH*p@y@0;QxGf7iqM_4dihAGV{-acvqV(DkDQ
z)B8#V?zepKBqW7)(bQd2fjp-UGm*29Ah3H3i)U4HAmp>h5iN9el~i>`dM^8ehttGT
zYUzo;*LRJPRBXSKJMJov5PEHdlz`*ujQ3f@m_5RxG#-Ki*o4htA3!koAwwkmO)~Zv
z2;6gc*-uWxrXDf`_zk8*C$KXQ!dyFcR6I0DC~eN|DX)+K<h);1o^ijue7q*T6s*Oq
zZojRbl%9l_K_5G;D8e4448AbB9oJ6YdD6_#@w(>vIB`Oj=1E@@1@-j=JOWR;!kHG%
zKt9b57+J6cd|xcYjZI{`!uq3Mwn(uV&()B8wRtu{1w=E0Sn{2O-JuF2EC5oR<k$Ub
zV}a~Z$slE2v#)y)AJ#f<I9$b>$XoQ$iCDHjM);-?8VI4Qr$Nsgl3xIS8J3O1>sk?L
z1Jpp6ioAi;z&{3iDzP1;w&vu-v~%!>2e$R<lQ;}fhxD1w9STWEyxUTX3(B9M-gf^4
zMc@%ordl4hip@^{=`)tdFq^`1JU;6SP^_QeW$|^k`&A0_^@}?%Dt)$oaqfx)hFt6O
z9XjZ1bThcqS--P&I=UXD3iT)4emSB$vIRFm`h#S@?87<y)5<yA3g=^{#(UB#bX_*E
z=A0z=EOC=DT*G8nyr6C1);u4OyhVN(gw1(wF|9XHCf;;J!>fw;;!9+(>GRZ=a?8CM
z<i<0~3g*_C-t#aujdRKE$!vC<>Lrd!^n${60eDnlprm@8`F>P%f3!E~u-od>cluk#
z%;Ct9Ld|XJYu@AA9Ofg9u4&0j?<kYVZ#p_0FbJAHt9}9hMpBG6$q?d3aOJO_XU9I#
zVlAq0(Bwp%`wyF@1UrEy5N~7E%WgEAyva2tG}K`u*vA7{={6TDH_3x~ZHe3@dr%+A
z2A-|jud`*n?BGrIy-wg^Y;S@1uXw{t+qoE{^xs0{6~9XK7C!3kz-y?3AO%WF3egG~
zuU0&m^nuQA99H9fqnj`Z4Zmm^kUUON8`g@{dJYA8$e!)|qIVp>w^*s@S-hc0E`nU)
zxizRYYCOth7k2`CzYiax=T056Za@k4jyaa!^g1o>n&FQ5ICQ3Ldmb55U#&zLeYq3A
z+=8*(W{3%L)5!`;BphXmnXAeejQlVWeH=TFP%_+`dif2?iGP>N)?9lnI}Bg+Q#>)w
z9IIDrU<+B@2z)m(6FmI2kl8&_5VsXO&mq499axl21{DJuG?5`q;^-$@$3hkRdm6_E
zu&5|CxV9Rj>-CHV@1?0Jv=llhM)Tlq3yk}F`80S~S7v`-PzHf?E-!I}zV2IEzf1E;
zCWMU-U)i|*37Tl6&@Wlux2;y}4{TJ+M;Z$Ah9b9di-C3?HIG#JkADhjT|NBEdfo6=
zLTs#4u7M5JfMl=QAc5@bFR1fe>Gog)$UQQPJf6^f<db`UMF0GTyv<iRDgy)=X(&TQ
zdQxvFuPj2w1`-h30sMVTv27Yt0v`A)9KKSxh^HU2X=|xYcX-+UrP_w{NJZc1tl*gh
zAk+m*$-?<l1o>Y>MW~;7P-Lfxt>{h^LF{Q-HG>D>xpWZ?1XN5DA(tsr3JOi3i!%)7
zpX3fZpFm9U!;6C<`KpOyXAwe8Fl8jZc``n;7MG+!@EU*?A1{WmMuF8L9gv-BQOBtA
z!ZsX)RAW#_v=5P{80Bn(4==c$pM9Io5D-a=CiDCe>kwTNOin`>z5vEbyzYaE(I>)|
zc&YmDEgqpXgvtmi(s{3>qgepGeL8iYLA=sw@5iXGl8fAhd9SBpdln9+qjoUz^#~^J
zr5%CBQ1>u}ZJA113b`3@I5G<Fdy-3k)*9t74RinLGl8!c^taLh`qCnt+%r$LBLkuO
zviuQ6kT1BKCZL+S<m=!?iN&mUo+-Cm+VrGPS~JjV9pFYKZIMFUR#Z;iLe@bU5H*6h
zXtof%41C-Ap|XnZfBXcNTbLKTuy1UMp~b=E{dP~r5{t+M%j`3D55-S#WhGU%pV_-d
zTB%A}*R?C@yYs#h39f$Z0_m5m$O`jhtK85G;U(G9kKbF!^XHtWHabJb3ntzDZ>11w
zA$}RG$q`W_jc7`I67%A$53(b8VFa`7M1bVXf2toJB-()6CrM<{Z8`Dgm44wqL3y_7
zVI45UP+AQ4985a1{<z5KLN`x{XPIfpEL=Hu>Oc0rg0K5!qr#RY%m-kOc<G<%;*bnT
z2(qg3O9%`L6*ngufAD?%gRxmjI|uzgoW>*!hU!c6N2q@!-`?__g~Uc>W|@%r{PW+T
z33W2ZXafBD2@b7)y*=!#&3f2(pYQ{)TS>VXc+~f$acmbn<ZqDCIyPY13B1C}oCC8Z
zbXVeXG<VxF!qV{-7r>E_y&eBroCzfmA9W&38N?@xP%#>1f~8|qLg&G-1pTWndxL(!
zW<3)uG9cz-x{(nRnPUG#+{^R<_+4KTBq4K4Z3=9%8I!Aa5wFio1X}<%dl<k2IhEV$
z;IG`0EB~%#CC#*`qcSvo@5HPBmHl-B9%D_)hbdgy^e_9Zx-I#=MMpF(BwzV?AQ2WB
zw%Z7@TuiH)P}GK+4aB5iM?1C8iL0O)xoONazyGz){CtrGaX$y!*lp7}tPcJusIL?=
zeOotD_GD`?+NW?**bn|2K*pIVe(JA{4+E$WkAU-&u?!R_Nh2ZFMp8cEIgns&NK5-q
zb=PLKr8MW?-szVzA+WMn`y<Ha{o)~zL$E>23LnP|#tL+A5JAy34@sbQ&i`TfmIp=h
zUF%zq9N%pg_^?e3`}up};>{C`)2*h4hAS1vhT<a$)d=EqpJm@(xU`op6{5hU@Rxek
zBGZ!cZ)&RXa>{4cGfjcgpC5s>ZU+V=uenw2b2vE6zYlBcQMAVqAu3Vm8$kVe=5h!b
z*!*z*1q2HGGMXmq1EtC0x0H!_9j#R5e9HqSbT@cyCR#@M>+oUL8b~78%Xlglb13rb
zsXw~uOW3Dh)XuA4tkZ3mAExT<KkG=p9)X`3t|SpOGdPqAwR+C?jRh6R`Tz=H15TO3
zGsmQg7a&S@2D|wJ*vMN|Lz_;5e28_XQ4WMK1Om+&hC_l2rJ5Br6r=B1n1g@KjPs;q
z8H#xGminI|iI?Om{QmY7fse1%dXPIT!V+s<X$LEgW0aMXT-kyJ{{1l+<9OuTf0L21
z^@R?xH=+nip+j&f75?Tf%}fHN-dj*vlaiQ85Tm}9y2;2R6{Sdsm<Q7#^7#LT+HN0o
zL_-h5R8PyZ;A%e!TJ0-H+G@wDj;{HQjV{zGia=Js_JVUDpDYbZ`Ux4h*ukQ6nN4j$
z)l)+g27}XwUwR@4hkh?;bQ38?llB&c*VS4dhZ9hLhW#s}<wp*w!Sbo2OcUkaHb5WM
zvE6dmp+b~oJAe5?quhvFcO5s!>z}8{28ZG*eoF~4$@wm-fAu!)J7l<IP)~&eQ`DoS
zRO3UM-3f?+zulzejxOmZRc6p`+9}S*rNcb|B-qGf9{0EL-%+=(jK3*L(UkbbM*}2#
z!EcAma<OIVrEVyulEQbG#FTyLgsHEPc^#s6QN(LkCaZX7<YTn0$<x0a0h6e}7JSfd
z&0`xk5amS#2^1qNMUnasDbw8HDo|ZrG|$#RG#Hq{$E_nU$5UH394t<Zf7n#sT+Vlx
z136tjdNvu^*N15%k;**+bEk$r3KCs79f!M$IY|bB{wn?Jre17~j$?)6;H<u4j$|hr
zXXcqpzSzmaZ!Hw2{_^iYCj=JwO$^E{S_(B)&=Zf<{ccS~DNtp9GCAwZ0K<1m{HyK4
zwxtc2f)FEl*&Mm7@p?ms{wVfep#)T7-`ro%rmXxK`i-KQam8HiKlZHjK*7+S*YsQ0
zWBOyW0Vm6~9=BLiRYIlz+{n@=zE46=S6dP(Bx?g@{0ZGy%6oI|t-!P^X?^J4Lc&%c
zXd5hX5Ru@7yzLMIvAEb_nK*JpLrtBvAf1$u_m)hayU+g_!<2M%&<f{`v&OqHdLP4a
zTZJUbTsT)%4wKgdUH3yCZ4Z_bIF)rTFo&tsn#HU97^>Oumvfr#Pxy;|ty?nzB?+E8
za6g<1X0um{d^wFaTwMVc0H76=KsMclO%?yD)!8w*@Du_OwS|0sw?DfpfHzFmOJV)7
zZvfWhssE;Xz{R5KXvoWN=aRj^(xpCcV0T&gJ^^Tc(IQ#V<QUXEDiVXHYaiNrYHJ#F
zYEEtwRfCCxMYUhb9Hl{wDHDBGgxxWQI?|@Y&W@Gvlegk-G9={-FVwi9mgU*;-g}8~
z_L#9w4Hr=<ag9PM5pC@&#0vU7Vmqml^AI%vu9$q3f{~BhIBP1MsQjW&Q!QEn7zL^S
z;5gbEZdZ9N)S4#E@I`S`rd0{X#{JBJ-+lB)C>Xil37CYrd1_dq8DP~{*%gqsylZ2(
z^dQt*<${h9v0AhK1sY`KK)zkzY%SoX-NIv2$?Lzz@TH&ZE`Y?KgtP(L(NV3d^)gZ)
z+!kKwke<AOw?zn+_On9A>;SF?+)0<hrh_`~^)wKN!-><JRHg{V0549QTNys3gU18>
zIY4hcu1!btCSEUC6mIQ~ql{pT2>;j#gsW{l9-z+w!1&O;!Vnj<x%JH$bb|joWC6UW
zU&=+|CCL?{B7FBx&`z26%!#{ZvJBa}=szgIMGzvXuckJt@eO0Kbi4lgSd5%*#%VrD
zx$&|!vI%qN_>~y{WjJjs1DGpOZh5&QAU9;X5Ol?DOamA>iv)ojrmrx%7pAuK4YY+|
zRupqsb(nxyt`}<iA3$lf6#eiNFBtN1-I^b5gcl6!Wn2qEl4v*H_r9RS0n~)kb%Pl&
z7JXO<VeTqGd>%f9v3Fo5HcM(OfjLm_gL3cv8a6FOkVY*OO%fy(Z%Khd6xzt7oHfCS
zN}2?U$2AO|`oP0zP59Ox7Zd8qrjC|xLC?s?I8Q*j<scMlO5*VY-^O|y{ludnNa;Y!
z@8KnXB2tt8J;oS#ElYLd8-E~SOpA%dJ>0d@_=5q*8m|VD^LVCc5jr4qN+b}2+>?Rv
z>^f!_&3!6Wm*41DO2bcKEhJ7(z+V$!${TJIMizvd9HyhY0~RvF$cL)#c*UgA5*@Pw
zC127%)T9PSJLG`wdpd1Ii!Xr$f%UNL$@RG6jGvUjTaQk?NiYmG$jgVzs0-b0TA1L!
zEWgV|`>(mdvD;*o{ME+i>^YSc0b~9M3BLYr@YeyV406j2nY7JhcUN+<zX1ziEosJ!
z|Kmz0j~+z<`m!dFodhPS)6ZOmz!tv%klVICNx=f#h%N**Xv4PHn0F=OCEx&9K#0er
zet*_rOrIc)c*983Z^?IK1Y)5Yxh))fQ~W5h#t}2lDxg--l^8Yc=hOB3*iu@euX6;F
z^y3J|v?2(d8RNO&>Vv@Gs4tkXFpzd=?c%I0*cM8jyw#zGlLWqxVGN|bjl)3qucSvi
z%{vF*Q38XeX<1c<I@5`tuvGUz%S6U%4q8H0qg?r2J{jaP8ps*gGSwUUgF~UG-{TSd
z`$x1J#Lu`dLXwI5v#nU<z0`G>jipKXo@@x{?&0}*+!B|bbB&cBTR{$48ihf@D8Dhq
z0$NKVH(U3U{ik+6%13DjEp~Z#v1M%!^y%{9xk75AN?f7$f3;2u*Q}~+#RshvY_tCP
ztgD`xXA-?-auHm5o*VzgSGc!+Lf!Q!p`!qi1iM@XPhx#+k!=Utns7O&%JohtNWS}l
z`PJ2p)z)lK4<ds|$^1EL$baqme#ItKQx!^j7@i`hH*>fUm!;Ow61_^;bSzn?DN9e}
z|91IM(cdK;;`z>rh1{q(UdPMV@WQ>Ki*39{1-#Bpd3_JzB}821cUhizIO5es6TeDU
z8I=uZKjLCwqA+IGf9(K>Hv7qJ80wtnE6;`_2`h_}uQZKjFIV?_D6=)yz;QKU-Ghdh
z3F7q-HZIZ|Yl=V2@c~A%FvwA?4SdSPrt5G`6kKUBepK1!v^%`k#WudNaMJsAXX&X!
z<ZI<q-HuSE80cuo!dZ!HjC!9IVc<dBo-(x+<c$%4DEh>=DYFOGbB;Gg4<AZq;+2lD
zWwM*N4|JHZk2expu2b{rOzXUhwT0%;T(oJ9VRdO>(onn1=crz$ya2$l7T#=%=m)Q;
zoS(myb$Vd@-BAgzZ~pc^v(46twy9WszL_I+q(3DUU9WQ4{b^cr377v5yT8NQUUr4K
zag+aAZDVVBL>oxc`Lx;{TvCZK=vgkbDdj-sEqTmUVFY@o4~o+7Bx`eQf-+_jQGBQM
zmxi8I3Wx@{(JIKKoGGDM3zx*nq-MLjd>ySTE6m046!Q?#X3_zuhlkISc%}oz0<AC$
zQH{ffgv@#;EOADN@2o?{2j#=`d!3iUqPOrG7W-97i}*u}#wwUK*)^GUsL2wXuTVJN
z3GNAF+0DIwGTMCY6Wvto47Hs=OZuJt6clVDF&MF9<t{E><G3{lA*>)}p#Ha*=-yNv
zc7yh?Lmm<6HOw^;%+n{4)=QH@;k<-Xm>XVUg^TUoaQR>m+K@7C62?OCMw%?N9oIV^
z$qbG&;p!BEh4{h)W+bY^H4bu;ectJgb*H&0Y>@cyMzIAqnBoNp^gkpfHZ^a`^a>>F
zHuBOh`N6HBN4cVI{u&vz`XqH<d1sD*H5a_p5oYGsF-?r$!hO5F3Fg{A|L}R8$h-pK
z{<rUvs>NzO>XCbAv;x7*p=Ax?!S*CSqecE<jr4+{pZg(I7R>l+8=l7{YCqU}uQYOF
z@^)z-4vEae?tmr{Q}J6Fpjwi1sr2PyMg6yM6b+H@Na^_f+-V3qRT(kVch1Etex4!>
z>a1>-h51KlPaOdW0_=3~Wt!EDJ_k}V&nc=jp=5Lr*R~ku856G9<_c!g#ddHOZu+9W
zUScXtYelu)uR`$_V^pKhmNoVEb10GB6OC09G<Mny>OAFcz-XC1m&f|)wIJ7js)+o8
zo2_I=>mSHf$Qijshx#r$>-;8B=|T$((V(g(SCphX-$yeZEG5d5vYy|P!dit$DI+QL
z2oJV;MV~Wy!Q2>1&&@dHUTFUd=>+HY>Zi%IaR&$w`H+Z!*(cNH_-aO#jSZ>@PMwXz
z>Y?(5)z~Xv-`(zIrx-xs9G*fh`|jbo+V2@}vGIEk!+(Q6T_Os)Oi4z*@_u}HWz^zK
zokOEvg{&~`O!Bom!AjCu#B@c*_H3a1WG~Y4Qd`S7t--&f-M)SC7wb0zPwDjTSR%Uy
zqJp+>tJv@EWydo81AV|_JF|vCn%A8j5yVic{`7}a6WDrn^3kfqEl~dVGqc`GStAL{
zLdXDIX+BD8n{*L-0n%GVP&bHEXeYw4>h%%i7Hu^IkG=vs37d&_{eX6#nW&RT;>DD5
zm<YtzUIC_8$wdzp-5K?*iL_m^uNjIEbD!+Jb%-3BxoGR?auH`y1MJ^*Ieytj>_R78
zYJqD4=;o5T&V#7hw14hCmY<|u?NgCl%(*LO?LFpksg`M#JeOab{LyOi4c>k|fa7QI
z`<mj!odSL+{_VO-*PR?-pPqT(F}mT`U+j3h+MO`I5B^YnK;uYKQG8qA<Dqm_C{cC%
z&+)My$S!0>&qmOQxbZYE%)sn9h=sPqLWBi^F{(aY|M0lFs=n`52wwjIXfmkjfE)$Z
zrB&5H%~=lbjpvU(F|n3?KK<4F_rtLB=0=gbrS7e<9=$f62rVp<Qf2_Gw2sGjduJ;t
zlply+`_G#CUNun7@5_XJec~yepomojV3$jjjI{mtb@@T8*Z6%P7osK|R~*{Ig(27I
zQG}d@zS{e@nM-4!`zPK|$ZnL5A6LbB31NvLMqm`RH6ax*>@9m~#ZN9H=G1DUGK9>q
zX)Z1oAJTmf7M?r#yZO4VH^aB8Tq2J*0?|8|V*DF$d$N^NAH`Yfe;L<(Ctx0J9Xiy;
zS3J6Obk&^ge}17xn{kDfvl&6D5lX+qojW+NjQ1GPXBGI<%ZD(}n`hZ1woS!z#BOb^
z|K>i@W=%<@{_m3V5N~J47=X@+vc!l}pWta~tvJZ@z3&Yf*I&jl$Qp9gfOh*r%h>#i
zJApk<38HkoT5r<7``gHea|@h$ARW^uGpUP7e*z6VO(EH>5s@os)GO)K_qsP$2Cf_V
zr_^Z_av6Bf9*fuLg`x%22xHEB_!49xf#6#_;+6Q|4`O)DkbYi<mHP&jmH)j;%Sav<
z17*#c4|8tom<e^pqrN{@z~ajuV6yNNOl-WE|Ihl!&dn>T)Skg>%?Er&iIN}ZW0mv|
zYv%5X$yNpv&{!kP1!dVB()2zt(Y0Z+uW@q5zE_2~t0=J<RB4Zkd&Tv@PxwY=3*E=-
zU4h#;!#$FYy_aw~eeU`NfX1B(vTF4;^XqN);c|C1I8eHuEuJ-EmRc(oGP5xE-nfZw
z{Rtwv)(xfSlt`*c{uIhPud%$rcl+(S?rU*U7VdTp^hv$jpEaA`KOxD7?0$N+8uzK%
zy|-n&J(f&Q8nc4TS~4g?hgOT=yJ!IoLi?(t?p^BU#M7wnIYf*rYO@0<VlJTj%^!4s
z#1g3VjPHFV1`w_866c)LCpYu?@_^3OS1%1|i{`jf`!RP4SNroZhadg^-)RDOP2VyH
z)kODusI`CObddl|*{qo$@uc-Q<1*)3>DzavbHT|dXXha;Qv5)~%UNj#*Zs3)dV{gI
zLD*a`TbQ{dPd)eEpABd9wrDd;ekMB)ay*k6fkmB=;go>fTWHrlpZ+S%EQF5uQJ=X8
z1WR~y{*(LIZA`gfg}UvG4yUl45I&;Ys_C_avf4*Ne94;ZhPVH9+z|H*h{bQUqM0Zt
zewzB$Qi;X*BPFl#1f8pWYriV^(*7xf`wk+hl*f*$`qX3iraq%Hk%unEqR7HfzYa8X
zUSLl7Dpi_NV%lgr8bX2-`Pl$O@m-JycC;K*hwJd;Ang&e#qlH3ggKh!@$>oaf`Xm}
zlsC4QJT*UGqk!vPSt`|JvxmlNZV(dmlW1}p)MH89Fa(z13eMpEY9rYV!5tA=1lXc}
z%ZAWoj91i>DP`=FQT7;k<oxA&SU#usl>pQOIX~Af#fhIB?Nb$|_ZZ2de^#$^{H181
zuJcFYY?7DxAmdRgl`gSo6W5G<LO6F6T8SN%&=$2>nQHRynp5s>yneqeU;MMu^Gv#L
zt<h7w?NC>ac=J@Z&CC4>^Sy;!XpetGbixgdNOXyN;fp$=*^a2Jo_<)!kMd@Kqj3lP
z$qX%cZiVk{&c>e#s`)b>1JqH|d^vcDGfz}9>6e!PD`|CV6<dPgDQpGVkKO9fWXqRO
z-42j^I923eMG9h#*bKnqgQd@Om=3<pSc*IKj-OHeEL68E-KHT<Izmv0Olbq+D^7nj
zryM_2yb;4}JDlS?@!E}ORSF}+Tja4yFMD}xt{8l2tgnl)usBh|sjZ<NT=LS@f)uqD
zgJ7NiFxc5h_U=oo&~bc*BFZL#{x^*;v+@6%&Bjobx`ZD&n^QoJ*piwym#i#7jOp8`
zVAMHwB!W3C`EJ9>5tjOeivXjz0`U{Yo1upNS1-pHz*Z?9Jj@cNDZL#C#oADDHFF^e
zY|~l;_G6fty)3FHt!^y&+av8YB8miY7YPB8D%qu46adtqC_!AadOK(*+t{Y+3ML=I
zx&r!$E6LO<hrs8LaGphgj4=oQTuLk^U<)&k_H70Z<7IEXmkfcs!S-vgI!dC8r^o}=
z&a0lqzC@@U+u5r~*&GpSM9ZlRcm?7m6LiK4`sr-Z4e`RAjWahO`$A&mLmV8rtN(wq
zb7<IkHNW{<E{md%amk+agMswCX(o0Z?s^eT3W4?gYvNmeyy4bs(;U~srH4oRoCjIe
z1X9pSEX25aKh(MfaoSHfvM?q>X*RIORXBug4-!pd`T4!7z<j{m@-57O=yiyfo{W&U
zq>_SPI!t(Q_>`8P6m$H;DZA%VL<p^_1vDTf$X<%;9{x|gcOdD1mT;CDxA~vx1Fjaw
z<$S^Iq*la19oq&N=4~B78iA#U%WPwne%XDaLCivBhsOX*X&9N#wbI?k5C&0;8h!Z#
z4?b-cVW%6uCe6)w#%xp`T-_J9+n0`Hd7jeVK{FW~{f`~Z^-ytuXAcC5A)D9B8p$#R
zadyPcvPoAgnXa#tQj53bsT1YmJLlDyneaE3+#XTAOL|%t9yGGfzzJ!^bqmZxC@`Zc
zdoyMiBYR(`%JdQ|Jtb5X598bgrTm9JFJ&}G(Q_K2<J_<KZi8yuSWJP&YcSOtb<)zI
z=Eb%c7oQ1<k@s0JlnK?oBT_AdMqhofcuW*WAiHz3KFy4q&*ZK0!l7d6ljA`M?S~b&
z%HYH15F<LvgsDWEF8>{CJli@lwXu3&3Br;CR<;;7b%(G3iS<WA6q4ufAlfK4j9#wU
zv6kt~S5^(R{x7fC$B(k#y&}T&VUjvys(lK)pPYBc-c%?<--L1%q9GAi;zoD~lXlO~
zhlkU+Nd=cxp<Z?Aq%}<qTGMwtb99DDAbr#NC8?D>;DN0}?r`y$UL-ZlFe~ek!2JDk
z>7gP|D)K^p!CUvJaJxuI@xYimS6x<S^b*${SMYA`+o({zRx|<4zNy~nD9%RFU%&RQ
z8V#Df&FU{2p?*JW2MqRC8``}?A)O0LCiP*M4Xno#W|>nq-Iq(5ABu>Cw2a!@f8bKK
z`*c*vty5@yX!}TOaRe-AcN05Qlns|&0;pIe(^UlCpsC@S_K)^PO9^fa)hLR|owtAr
z!Pn6;E#77A9S6^z>774bBjB=AO#;_Gvr~`@?!RHx(4uBQ@7ZBGfat@AM88Bw2dCHg
zIl5`u_4o*_*Oh{wYChyex6d`p&|aX{T~_`Rxa+QOBWc4GC4R3_rq?t{fXJp>Ksfq&
zqoG42H})kj)9B>{&B4@PfJTY2Z{vm9kR|=ax9FBF$=fQQe_Jn0KM)%;f68QoL1m%)
z=kPmjC{k+GC(`p#>tE6P)<hOAsrkir;r~$D`b-os+i-V!ssDgwreqH*Yda7YH9+*5
zms+Q8kp8r?de<$T<AXckZPd>h-|0>f_cg%Gs=z?nA;v@LCZK@w0RBtTRQZ8R);+jY
zEzV1ctgiCYdv3ZIoaSHz$u7Q8^IjuSs?UuN^ahBxI>P+-s`AzE$InXr118K;pu!)n
z8!T;=4L{V^ni2oyf>`3616Y8?rUCMPNbOH=h-WTBMXp>LqaW-dYZsfa^N?k&c7@_U
zN5Dg{YF>93?1mk`9{+HF-u8c7x9vnb{+gzwKtwsJ=AYY{{p3yc-<zytKGA~`_dKB!
z;kl$%RoTCulJ!`#{sTsvOVPMG0`t?CPGq73Y6Y9eAzX#^4Dw!R8YKs{rGFIAJQ|lF
zh@F}D^erI9`)S-+(f+i<aQAz@Ax`rbr_o<%vy<0b5(_DXF275PF-GgpXP&z|jLue_
zz(|)kL@^VNg}3+KQ-i5fm7)8Qx>^TZuBmK!78s2R$Rpia%kw3lIYoQCtOmeohvwgV
zAm&XAX^6K*AQI{II!A^KwVzgR2Fib@)OyssJ^gBcd4=+cm}qs1k0vp19MV)wO)#O*
zau_;_7t-puS>4~?@2331D!N-V@f3l|oAMIJL*CxwmLW*C^N@MH1#}QX*<<}@37~a&
z^HIVPkzF_DAApd_r)(RP<sSsk=$qnp%C1>8jO#O7#nqucNh(Ue40gl~?rQcnN=HOE
zsHOfBR`8YbjM?%`KYem&2*i1{Bp0{rb-BiOI|e!a(_$$Z&HU*vgxb~L#WX{XdiQMH
zpWhnq-A58pD&YZj==u*oDOYz}NzK&_?v>&z*8WJmMY9t>ye1ix<BbBc#yJWhB7N?{
zS;iUHKj0?Q`IX;h+_P5UTIb~rZF@L%z}1REeG=yT9%76B?NZ_ICQNQ^D9W#oMkISJ
zB>VS&wAqqBw)1i0*`6&(r7enZJJb@Pudg0i>dpFTWsXdiLNOX-yUVac4TsUJ?j@E#
zqh1Frn3HzUYhLw5$_hoedVIV>p+N$2pb7>4;iq`1HY%pPCC%?0YGN%7Uc~;vnh@}N
zdY;f7qsvU)GmXk>4BNQmB`mKx%X0H_-PZ*WQcO+=f6^n;CP|PND_Y7eAyV+9bt<4&
zS73hTNq<;`CfhIrjlaxrPYjxF!+{GtTnsKF^5217=xkA#3^cPgT|e-x$k)o4)pbWC
z$Mi{pXUGs!FBMy=uWpK?{uPQQ4z}e>O~L>nwC@kLrpYLBCGV0;dFYFV@RmATk~>oR
z;ALC~)6u-?S2EVta(SRA0?|i%VWMs^Wm$0PBr!t^lc4=OEn^u{TSi#(JRg5(rjn;T
zO3eiYkwvOnyZR#k4{&p=!xjzl-}vt7-}uSX9~cyd{wKYB99nC{kbgh0+1Iqa*3N+l
zBdFOeCae98jya0J1QxQg#*<Uzr89jjV8bN-agZ;rrpB_aot~h~ZI)7#Usf4l&&mQF
zRfK66Xz=}1TL#YaKTe#Owt9umw;wtAT<3gt-CyZzT~Ix}&Gwn_?8==!&^b{NJ@fa{
zdKJNsx=Q`yp}JwsdJdj(ORoMEcE4#K74D1KCh9AE9n9o)Z!=Ds8_M1o-4mI?ZSC(S
zr3%CNo0>1si&N;79Qc#lN>lqBq7V!XZ^W%4#8pH1QhxA}{Afr)T{|`)0cf+9l;G!Y
zu-8uA4i;hJ$1H(lAiXq1A<SydG{cDR#HJ{C|Kx-_8B8l>7E86Z2bJo`_|z+vKqPRz
zkB1yFB2lX^y*G0B;Euzm-z6eZVJ?DK#Tr;g#OXGMC0G%yK>S$`<KO3LsPj-v^Yq;H
zZ+(J2Z~1Ev=QBU{ux%Ee#Pf2_vQ|$#dad$#6f@;w=raUB#0+?MI;W;ERM*=8G!ryG
z*W45A(@QqjfbbOxiyvA(eDWm_ih!X`;MoSwAh-l#h;dK%OqmAb4NrN!1~}Gl4NmcG
zz7yj)kS)|KhW%D=`|Z~d^&2XZw1#z}IO9gS@*Q$;?o4%&uT`pzh!&CTgl*dLvtRqC
zy$j_?;}zLHm-58O*P^iigogcKqzO7gxmU5D!m#lr6R<_nDu$)s+1*3z{eVelkMNrj
z<XLO1HLfe~w6b;-v#~eZ;v3H74YhzqwXvJ2EM&P9!$Bgftl`H9NvM_bhpk7=<8z+E
zb`*+a*Of`h8QIr(cM*Rr)8qf;3!_B5Y+>D4<O<+iqaq9Pzb0j8;KCQq<Tl-rpuZf}
zzRnqZ0X&>KFC2}!Nq$Me(7mAdsQ&k#Z&~v<eq68d?We-0!li-y+p<3B1SOD<Ay4R^
ze{lzU=Q*xJ4MdV{eeD2hh&fCQthJmAe<caP5U4L2z{$1GF$YhUQ5=xB!L7fV`+|ee
z9_7I7>GqKm`uE%^wlO80^TmP~d%5NM75Y}+Ng)N%X?C?jPY2KxH`Ea3Fx>DOWIULZ
z!~5}*=5-VQ93H3wUq4Mgy<y<y#!WH@Dzf5xF(nP1eYaZ?Z*!}UOR$KJF{NYh)DyRF
zb~Vv!twLK<sq4D<`LDdJMzHM35N+vz@Ytw+$(x)7@Og-=&mSx;DQZvnSjUd~D8+G@
zke^d_cGPkE-!*fPO*?gYhdCYD@j=5r$XO~XrD_Mu_p8r1qhkCAEKr@xtoO@FmATu&
zBP}D;o@w%bY{KJvHA>$op-a<k_gpgiDgW7k<=u>g3Fd~;t5)%01Y6HZO*1)<5mRJ{
zdMyv;XKRMAQL1dg#*t#%Vj*n1*_L>G=IQz!)FaoG63){?D%}@+sq(P*^vJDt>x(4`
z>}z`sQontjNOU_q((a|vv&(Jy6b4Q^Ui#lA8pkpmSQx$S_sA?`*@S)1t{#0iGJ=}^
zIk^5A)IGssVZ_phB`@bYgrdASPT>=jWtx0^C~b7%vNpJMla5<hdTAT+K)IYEZz;7>
zMq5!^<qd@BAIy8wzW<ETfEoxRU;FY<?Y`R!8Uz=gcEIq0w2#iqs5bd)6BEm!=sVv?
z&?}KY1%GR4*$VvWyiE$3YT|L93fXnGY*Dq1)*mEcRIxM=tHlD?jURmbBhr3*8OHe1
z4g3A7u?ht_!rK+bxvyBoi-e%s_uqll{RI5yBakr^C1s67A04%xkO<CVY)(#q&ECd<
zcx8bZ+7B+3dwi;FC0T%jilAaB5kIG*U;;&1SQ^4sRaexUq5+Ia<79YYi7rW-e75zn
zoP4?EwjdqsNe6PZXSO|siwW=#QdseuFO}Z&#T7vPaS~12liSwHK+(4HuNHcZ0P$36
zq{NYz_QgL(wRf9S_k>X9%sMqXu^LEs!7g)B+1GdI#TqbvtK^r<QE0RnubETfCks|y
z?oVN}1l1L5Pk|cWMZbLZBSk1(3k;&e|4x0rm9r*)FerW!7(_X*HW(o_fawfgAv(xZ
z8XP5<ur<hA>(H`YX8vHnL1{%gFs2?9!<iiXS*?S-PgE)_g0zACDHXt9@n%vi#^r-p
zbcd-P-|it&<rm)pOVEp-;f@zl@0xD6{L_O??5VBNJ*5I+#cKST2IVEs&Q9&rG^m5z
zD&svVzI|uLZKi<p)W@Gw!+9LHOuv-zKH<nJ-^m%~PlR&ZvhB7oz}_JJn!GDSMpoh*
zxkvsIS07g+mPxrWyB5b4{`2|R%Ki0)LM{)9&*+d)muOd7+%&W=$9L>;QT|!l^}}(v
z1HB{>jTf$?tEb8&iKbE0f(v_)t=R3#z5}?P3?ViOZoRgeVyg&;hi1n;=%4mRk;^?R
z#xuY+g@}9j3}N89ewCMD2=Ge~XnPn9?Z|YM95lAwQKWKekBx~GR<ntY@Mg4TdlQ_)
z4>Uwg@O>VH;*BJ*^kRLW637cON83N}L!n4y;3FzOY>*For|yG1R95DT>(iwc<Gg2L
zub-3=$v0^KhHK)n<#-rSr{n{H(m0hrhqpV>q<dJHtS^bCj#COAzmGZ{$HV=;yG;5f
zrehYi_<16GzGLwaO~>l*W8M1)T*CZ<3xfAQ%xq>{q3`XnC6mrp;=1{o-l#F2q9oq7
zzP*-}-U;u9FFp1pngaiEzz=sqm>4t$2$6{pXLp=`ZQmU>AI3B^Aiu5l3Ku6dqU82l
z3G16nrYTV2<fDje%D;gkmHf5tgQD8+?f+ScOIRSkrK>x{WF@mRu>W0&BWhusR|K)%
z;7f))L20de@*s6@%TwRCBOdZYDdqv46(3aqFA?Rpx&miKdhT?QC;fFH#an)_%~xEL
zkF1>&!+@6>ua)<|m?&7m%f`IdUQv>V@@-EOCa;!G(YQ<_F7PKXq41M;-P>gmBPxMQ
zet**cHCc!HS!Pqd6$m=(yKac?@%i`zs#!Y>$ejbDbI*d&UxH@n%Dqbsi-QZ%Gvzad
zjMBy@S^aU+{{rjAqrP>O@A$D6rx9<6s{>KRZEwR^xJ$}4%2vAt)4Hr-n1^`)q#*B+
zn&N8|IUX%L*pY|RN)vS@#q<p>Nayg;c?}R`7zSMFyT&mrVednK@i(vrfW8vGP5<)p
zW&e?u2r(QKm$6n1(n+xh{%`W36{E2On%427^ACJn18F8)$>Dj7zHX#~|DC?O7B}?W
z<5aHoy6;wmXJhCyb^*|e1|RPc9go&Z*Lu<1aIpIdp^NJAk=#w+JLbAij?7mFalyQL
zA+bE*o~>M?$N397{JOa>ZTBCJxlCl7eY)2E_XY4nw1PZ8xQlA41Fd|Gzy0TZ{>llf
z;=>)5Eaq*EH1Kp@?OJw>FahYgzFhPCYFcOZO!=MHznD($Q*mpSjiYi?gqmN&BfQ~9
zoN`&vn3?|9a*6)JHNY;2VdY%*D*)9c5t7D0FL?pX_UnX_2Gq^>FXn^;fb8#B6P!L=
z{+Ajg@A{i-6*J7;Tily9f;Q_6{|)f{lO4v8b~K`xm5hetqDU*Lk6~`gVx!WXwi%#d
zs_Ldt8&=e#W-C)_O3TWe)yWfG$-n<8p$y<<40+`yhbdkN3yX}Xqi0#Jpl=J?u)kX4
zs?B4K=76pdVg1JRLSK_UvU51I<EX9N4<X`Bm0L9&)Gy(`M)`(5_Oxq<ps%Rt1dg6d
zJ9R2!qULX+6$@Uier3Wb(wc$CKNVZ$_e=wRGkv7Hw)&^Jj14QY^6gq4HYcET^h=QL
zA0wbzNpTLaotxpd?xFjb?j^3P|CQqR(JQP9Zy{(n+%eV*<02z-h;%6sBF>!bj$pDd
zEHP#M3BQrQWqeFMnar(goxjf)1Il5el0VI{Bos@vy-R@9d!k2zw(V$@^oQ@VV^AgA
z#zzi~bxybP7u$N_Y&D%tD2pgve;Wk8=b|YmvJ8g0*Wpf<7Mn#JSQ=O^#EZQk`1dLi
z1~qs+An_H0@opt~R><m*Ex*W!3u_j9DN_d9m3b3Lo{KO(eALVV1a0cMkf9OXu+@Sz
z%?&zAA=gP>6qv94QhJAmX&OLrDvGG-XDosCnNSeq-QjhEfMPN|l$R?6zlVxYtdPVC
z|A0<B?#2^Ox#T<3wJWkY<3s0(AcXN7)b9d6?_3)lUim$QXeLFqHwU4MtBAFBCNPLK
zKD*YS>%ZDD@bY@>-qK!3zij)&4^oP|yne7Y)+9vQdNAb4>V<*d=e)eR>Ax`&0wy6h
zqOoz(cY*u87nub!dJ7V*o>FK{ho|~XHeplo1UdiV!4a1=C#OkN_lW|{KncEP6ffY&
zokWd>*Z1QR(VPT;)(8U~qKn0ul9gH+zcqsV3XqXf9BaIN*c$s~#Lyxkh4T|ezm)hR
zY09-qt-HqFE9qx58NJqZ4%23QvpR{mFUk6nbj|R1WgD|^q;nog)%uKqaLQf~e%vwq
zx$vZe?F}hT?DIF`rf{X`fR5a6I!R)FSfL@-cvGt;)@Se7AwM|P!Tk;7bmng^Z^Wm|
zagGi9bkBwJvExCi0kcfSJTfM8Le}K>tIVbbHIq}FF?cb;`#BW{ukXzJ6>+jNIu<CH
zT_qhjL1nW&PYIrk6-f3?5h1hT4OG>}3-{m1TdewPaJ4Jl637i8f^y1~du7hT`!xs0
zgT6J#b^ytq@}&?R^6SZSQ7$)Vts`<0u?`>O%1^uMJN-l@+_OJnzhi~}m0Tta^W*uw
z^X;GENMZfZHG-x^uxga?f{vsHzx}NFQ)l=yw_#HS4{(%`dae<FA|Tf_yxQZP5X4}#
z-!_z@h3d`Lj;otq475yRb!?pd;Nij2W}X^D+Sz0btYMwbyj~Pv-i%@xB!$5J{8I8|
z6>kFB=fF&tmQ?=IPT+*N?TiCgNF@{U4cM0zB#~p<!63>_0t4Ec*;|X7{E~xpuD^+`
zNU^B{)Rs|C5Ir|#<~o}qYMHyT87aNfs*<H4zd(!MMv!1oPNO7ymSJ!ugI`C#jC_r|
z_(e$sf+-quY_a6(KSzO5JC<-A9DWMmo&%@OM@ciFw@DFQtE+Rs^FE;XF}$G@Ws$Dr
z6(==#dvM$m|6~Ej;YZ74_+`mi+ofQz2Q>1qcmHzndtma$YW)<$l!`t2acjn&#z3JK
zkPkE?4*LL=W=Yps16VJqWFPPBJc?cW&^;I<YE{~AX2b59z_1=M-U4{-?N2&6RI7AK
zn`m^|k(QCu^!lY2R6lr*JPh}?zVcGFiIzsneh`2hbaN&m%bD9*ML0X@lOke=xu!xW
z&0q&^r4aA;?UTetFRz>!57GUD%QQ;kAu21$7?g|>^9m~u^cu<YZ_zc1ZM0HxQI;=F
zgPNs54{|BDA14CT|NMPj+k!pPd=lcM|42~L-1jG(;RJ(j?K`BpnzS#!Jvc)s9)iJy
zg1L2LI|P(=i0p?0Afzo7kK0F2pXHA5208cp*|$XpKD}%Dew4;=!6~6G2csr4m*;$a
z2i*Y$i>7)@zNsHmxlPhmvi;wDbP}TM9tcHerD%R*08>BkJwpIDj5vmwUm14{w&<NZ
zMAs&A!$IH2@~`_i8pj$hE#(-{{l3k-SI^*g#yJM4Vj8?$CD+0<lOoeqei1IAWdx}3
zh^{>1Ldd7kuZl%0x-EV(UwPDlUd|s|LDu9~Bl?4nD+iU=DA$N+78cuO2XxP@EJv@<
zcgwRDA|jwPN+a`N$xoWD^~YiJ>28AY+rpe}^K)RQ!<D{Vvn)y%$Yt<Rs<-P0!{f*9
zQ^3g#_`^b!G)!Leb$B&4O(uA%V%l$fTeNrD&HE!5OC^J#J~P!s%G$uP5*k{~%d83W
z=w^ERY4FD}Xq&#zV(_Mp_+IYwXk<BNpi{2Rqu`06#_EgiBrC04rQe#2a4${7gbu-R
znkms2lB`07Js5EBZ@1C9it$a;<(tIuIBq{38;mHP42jAmwG3JmXLuGrTbvkLZN2<m
zbl1Y>OEXtXK|{c2dg7&T$xw8|DS6cy!~FQ=`1d^+PL-S`F%&_7(r<b(^On>415F<*
z$r_de5)?k-hV_4Rz-_D}su*OJ&{m%-y8GVsaGbuhWU#`i;kh`rozqY++BflFfWN;c
zYCGYgXoefsT>t1xz6Zgg^^Gp68=Gkb#vVttniJ-4CJ*gkke1M0%R<+Z&B0vyHq3~_
zjQa!yYah<~CoxLHFuEQ73@li2F67@Dz_GFv!u(Z|;7<Z8!v?&0-3JQQ!Y(18d2(sq
zS5QF;#F1;Ix%RKT|AW)!Joaxh3zwq@4%%)7(l{P|hbVLaF)x+6!E)|JcG$n=Uq!5u
z%W&N4v$t+6QH=lFgy!LL<YpQb3xj<J-6X{p^)(P`m;SVK3>HKf589Z2rIaFEbcSD=
zOUmA<_n{ToraxEEcW(Ew8SY(zJ^TI&<lG0w{M>`=eU*%ti06q!F$}B44+(j&UVPJA
zZFD)H!5F}AH$mVf83ECMigWW7l0m@zOa&_p!WL3L30<R5LMy6xGY%h-5Jb!_0T9f;
z@!m?qgoPyprN>JWcI3vivki_J+kJ$F%tRgOn=1fEQ1hT&^)(7P5GzTj8#}~f7!!-v
z5!+%3cZvSx=v8hKEEZbj=__Pq${hb@q!akp9H|ZlOaFhmxbAScny4Q{i(bMmVzm%$
zv8#rt(Mxoz1R<*j(L$7~L<t+B?IJ>yh$smwQKCn;UOkBxy@l0lEb;A|@BQO@zWc|$
z^UVCt`JFTKOnL6inXL=e?w&8tbh$eJ6;Tp+29}GhI9p6xa|2%Nl{B)t>I8Jw0}_=|
z+sLOq@n)+?QlpJ9UCasDgB#R~GzI}-@*+YGZBwr$h_N<270$H?ik|qQ+YVQAi;=nu
zT<i(MFiQSG0<Ta1Wb;^}jf=;2&*s|B95tDa7$4;1oijv=>f{(heTAYKG8tes^=p<P
z4_pC{<f-Ho1K<-}WK`lT@^m}eYi#kMD#I&&Dy+2I^i{Obm*w6zha6*8)dREzNYd_R
z$e$<WY&C9cNt&Mq9_~=u2)2peG>7akmbH4o$v%movw1%mv{jwNR28*KPW^VG6NgU{
zn4Gh1)}x=?N(lLte?hrah9E8jU?jxV=&dP8gU4DDVDY!{+F&46V<OIGaxx5i+Wo(<
zsIlFM!{udZNuCb%1DJm^#1U+Be}217@pIfk$&S+5e~bzzIIb)L@23Gx##x3WRk!R=
z-j-#vui`dAtIyx}4{z(*UV`=I(w?+bwZ+DKAB92usZnm$LEn8u0@^tNl3A?u1N$=f
zNxcm#M^(+FtIv|MR5xZBLdG0o)N!<Y7dGh3z*mVCVlP;K$R;Brk}|^lBcr98gny;0
zv-qP;u^#b?EYA#&BP#OOKRrkP1Ds<T{MTAO=Wwbzk}t&3DRv~FFM?jh&1EHiD6?<5
z?{phCECBEQfGcugd3s$3_(m`}c6I;p=35h_@0{nTPQ~|k(Z_wlc%^TgZ$}(*^t!7R
zuJ);n*9+4{b>qT6yea4I>Eowx%@zI7lHzd9Z=bC68&Hq>n|PQv=5Sw%%hC-^Rs3Q~
zYif#w>a+5%E9<<?Gc)xKBXQO+EVv_no(8(D_d@7Z$7j!1#mVY|FM-07qD8dryRU1A
zenRRD11Va(Y2(+>pl2-nu9VjVn?1i6Zk!Qmt$SmHYQk7S?H|@=b8Dtq+d1uT$W#)M
z=j}SC_GU6#Z&@1-v$qwdH(7CIO-j;v7I4<=-J@ro<(q=AR&Vd6K(-dLOh>RsOoFST
zE-s<UfG2Q=(mlJY3HaCR)|tH2y>;!iLF%yC{z{&cj;aR^4~x688zm$ZZ!qxTx-PQW
zM<8pv+I^;>xKz^E#17PooKHzdb0_qSr&OCX*1IHel~iF&V<#j@Q@D8p!G_ON<WF<V
zrk+i)8-#g=sH73pHiU~$Mq(j@uR|GfdH0=VxuH3w$S43Uj*`3jeZJR*Z*HOc-tbW&
zBP!sLd_R_)14r5~vgjf!-PLHDGBT8atY#`=ojRaPH9&!u=F14kKQ&%TrqbFtrvum>
z9M59`ll1TaFADUXG+o5~k;8}F)tjK#9~~~{9}}8}bSom{#|R7l#DAFMu<y^+e|6~F
zENP)O`2q4y^iTdIi+0XsbIz<m7z$K9W~B0lcePq(!G_UFG>H<|sn<g9J%+uc3Hjr(
z83z=XA_s<cFyV$Lxc<y<429;Wy#+14&>glO)q8bkCPtsq9j>@0JUY)D>^~3Ew2wGA
z-Q#xG{pXCz;Q|5Ep2^H3-|30_sjwtN4C0z|2ar*16+js`sFN8a5~B7@N9>M39HJFh
z;WoMMR+2a}l0|cEHbp5e^W3W5cSJAZl*dww;<Jwe-`0}oZ`0CwawkKm<-WTinJc-F
z$FgDSeCOwO((HbvlXJrW7J5-Lk{xY=fAR?Ssx^<J^*%SKYCxxthVQP^FW%d37top_
z069BSDWb)a29LONbx9+N)e(DA<autvIe;U!*}k-BLcVL%K;yy&f`<z6{U|xUCo*GK
zdfO?S!9e=F<F76cDB`YKrz1F8gHe!qZpW!5WlxrFvhO}cV282kZ)1pVTBJ;yze;U>
z?$iD_uw@QJAm<wPA}^#E8Ju<-fHE$(DVCRq(Dq{4cLrL6-$NJdX)YyA_oFG*sej}*
zU%AfC%2gQmTwwi)Zk<qTp>>+?bVH%8!8_BK-whDY%PLEw7Bv;BA<~5q@_bLRG0JGK
zVroleC)Ftcf0ow8NYrJ%Pm2C|dOYpJs*VfxO<)#;RnBQDIS4Z56Gi|BH#2b!<=%!a
zP${`QZgu`diD=R3KQ!fY%hJLj5x>iA*b<cgdWz8UuYpU5EVJ_rSSdbSB;3behGj=)
zH$Zl5K3Gj<K{`8XR9VryG*|yMV!742qBD}~4@-}^+X93JmC+Q5oRr9KO)y$wSM{a(
zdiU@Gx@hfA0?5rs=(AFd9x=YMcr(`}HKa#VKSWq?rg$mI_;8DceQ5$uaslN#(P)#g
zaiO6S_m^BG?r^-TQ$kkeo-cnP_la!|KUK%gPz0;=6Y3^)alN(+pueytNToInUQ)s=
zoE!DTTwd^8!!27>bv7jpooma9e}kM^Bk_s#2fnI2Lo0wD6Z0W{<&WksB8SLkuTjPV
zL>*-HCmokAOw|JvHr?Ue<qdU(vr-EXbsQJ!bS9<1)^8iz(}W7+p0QoX-DN01ZpUV}
zE>pHVF%^p<&W9bzvkd>(E(oBh#r-8ZF<~OaNltS6;#LuKu#8XNee&`+glD~@8}Kbb
z-)~J~Za=wEz2ya#oF=sU7}_3MyYGh#upU0x+{K{Ab27_b?Ya>UTuSd{ogbI+J?ioL
zWJSS(Orn+1SFNjT<*4yjK}?R=d;Wt9P-duC1f}d}_2AxqyN?Z@M~E_<sdmTzKvOXK
zB>tm!6xgRjL0>x}RJ$syXoII;1We4nXD?GLE8ep3^qRchDDsx!Sx%Rq%EVcyXN9c9
zSQvN2_I6uLwnr1?ma@T)FRe|9OW4Hdm9$i+8dF{U(V00=^(YQ@sU=*Wb-&KWo}Zj1
z>y?!F$x@DNQl-=eQM_Jpo$g)kIpJg2kDeIiL>aU&7*Z=88EocZNs3zrR!k`^gkJpE
zb=aAOa;QUBz%S^y8O!wxnfQlnXU1mO9vO>lF5O}ABEU~soj=X}Gy^x`I>wz3O~D%y
zn(Su9bY%YAzfS!RNe`#Qw(<@cahBtI9|SYx0)3L`&ENjHKh8A<?KH?IET~L*HppoG
zv$n9dot%ZPTxw+CCo@6<lUz5KG-}_H2y}TpaLaBIl&;2)^<F_}!ah58^deSCMp=5T
z6sjwfq}3+dABj!930UdAiF2!(+*-<4pNX*O7&SapJIVr_>vkA-q(m`aw3C?b%>*xF
z^0X?W-#7We4%&1`0%Zcj4g(L^Jp3y;I;hu!)2ME|Ek>p_c5EyH&5)Za<2HskCuIhe
zCr@sn77t)Z3}yJO9>&9VxH8`T%3e+zEXYtE1KW=u$uS>Q7uZ+tIRz>rcBgI{j26=A
zfuy+Ue=JtGUAjd0M+M=rN_Lq>%GzLWK99>}n<G-46^RBPsgJ(u`^s3$Mh<-$bgVzw
zj-Iz`w&6BrL>`d*(*h4-aSfYb0lQRJpmJ_a#Fg=#^(?c5rAiQ^y-lNTd{(0tD~b}N
z><)0W);w_8LR^Gr<_eFH>q#;iLmA(Yf>nRjt=}5xDJ!ha(e<%9*R;i?fGW2y5`HVW
zxPW2G_%AManM`C9x_zbCbfthI#U2-2&KxvOaWfNn15g>sL`&VljXNVn6>AsD#t<d;
zJWRML7&Z5blU*m<aK*q@ImGfC7Bh2PH)JR7?D;j2tVvoJBVWWRWhx=0h0=>*Mp$-y
zFrf&3ieAVv-P2tr25#_opIx6Gr}?x`fA6v{h67i&8FwW5+XQ`QvG(JEc>_d4L7^{i
z(ct3{Q1or}L+2+?@HVE;^XuD+;*^xZeg0g*`|W)`w#8};+nCHT4<}b&zH;rULYZ5A
zIL;JStMS0gr|Yx>je~Z&71L^Tt{c^9=$qD!d<&+z25lRZnegjc62{6f-8)8Wr3ct#
zJNwy;5T~#F+j>Gr5n^|29zOS4(9f@GJozCH157<EqfeH|atvg+eWLtMt^7WaKNTq#
zwB<+{V~BT0=6|PENxs@+TfE>F!#r)m&f%z-ml}--XE;nfrPXv3VZS3g&bpz@L$~;_
z<iqm8+wlA3>8!JTQ=dNw+Ru(p%#SEks1v$pDj$v!_Z)o?fDd!L#nW)^5%RlJ^1S06
zkpLR(ZOL54Qh!nuCXlAS9iOetFhd(ttwDD*;ca#=UQHZ6TOr^W-Wj4os*+(Tpf8|P
zGsN$LS1qOVu=D!$%ePL*VZL>i;ft{0rRgw?C7fGQn5|<p%9vqlVswl*Pl6?D^<sUX
zPZuqoYoAXk=p-4$rSYFEqm7^=&_zHlANcbALGGZei>SUzD%IipUON`5^+o+5)ub^?
zV3TG0P4-=cibR}#Plx4GRlVh~{d8ofr5B&92*SFgnz3^+kwDJ3ZN!|`Fqz$)jrVJW
z51ARjSN8VsTP<M{P?{`3q)M_MH&ZljF%~jO3MFgB_h`&PjV^P0Ui$+8JhXDgp2j(f
z5=s_&>>S{Jx}Eb8cYYGmguuCOdV^?`e8GqsNaI*dx7_ce6{Y2j*Qx3RIgp)o8jVu)
zV8}eytq%(kEuy(;W~n`>VU(@()Rr?IN^^TH3dg7MlaRXA*NO~ws*qTFER9_3!)&Ga
z^-nc4l407pLRIeW0HL~^nnwyLVQsSX45?k$56HQ<W`0KD9Fy@qr+-pf9`tYELK3|4
z7mt6RBjz}0Bl^j?H`UgNMr9a*6;i@|!P1HZrocfm49R4QLOXI^R;zh$f6!K{sm{4S
zVt5)x10}N>#g_-v+5~@=C#d=8Gu6}DL8#>%`tdz0i=Mz!+iPve`OZ-SryRskC7k=c
zc+_Dnf|#{|mHYVQUF&6*ofdH@C2IL_53+fQTYpp?iE9%l>?t{jD$T%KoQvz;Ua_N~
zBiOS=_|QOO(vd3bk4D;&RlXR_AJftQI->1!@JcxSo=*}5lq?0`^Kow<pD{A+M1qM5
zJqC*MKPbx>f}6!}*VE3_K-AsLh;-z36x)+2Noe7szsi#zzSTSXPel88X><0QYh1VO
zQ`-Pvu({e-Wd?XXt;k?R)sP0URu^j^kvLUcMDHSd2T|l-|MT(MDiRSUoIbz)+Rjhl
z-8T_!DXG&W!$bkv#~N&{r7G}daW-FY@Iw{sBKw$3vpD%Y%y%eaxl7_2_1}_uSC!)=
zhs1ZNSvER5ip-ZU4<uu7Zx->C3MtmU;FlL}?Db-f8TfdmanGl=sCn3Ke?;+=mHSg$
zF=DzmqF1A#q^^thUt8jplGBI(dQYV02KtF)MCMcc#mgQk=BKt^MB+grzL|(8g~o{J
ztUr<>GUebY|Kf;DEJP;ek_zxVHdy!By1^G?%5-{Y(lb9y^M?8z179$gfi?REDMnNM
zYlb+z9R;P?aRJSdWIHm|cmvj50TsMqN72+L5x;|MJHZ|Uk+>_wYTVVCfck<XiOp>H
z{r@Da+lom!NQhl%jFh9DtVGPwh}!Lndw<4->sqo2PuzoB*YvcCdxZlp&BVNZiM*h`
z2EU|bXslKd!@b=Lw6H40<6w$Dp@H`cVDbjY-GqKeuFQ5Vq-1ixV-XCbjjS>O1E?a{
zL@8~z3E0b=VyTo?RwU-JIR$5CySykGgZDM9T&JJU-R^CC?c%F<^xDKqVm@OqvGnTT
z`%_W4q7+b0cIAJcwv^Y3wIrZtBs^49Cc%0;Dw@Rs!;Mt|bscpKe-Wy#iq0f5f3#<k
z;ZY(&$2zHMZ_}L(-vFkiw29)G?S{zV<o@{rn430o*Lbq$132arQ|#khpY?FY+pIlC
zV7uE~ObYV7e!3FA?W+TZy!y4OajJ51&NZwwu_bH;FP%z^MX1_&>D-{8XY?*oNs0b1
z76r!p-p_?%;Lj6p2xv6y$Tit*|7?5}(X?|h=bUv(UyFLc{Qx*aH-PUplBXHcRS)#s
z1+Dg>k5UdwWvAu~4)qfh7_&Cr@cA9-I^#FYpFBZW2s#=dT)5qIUfyF1?qn*eQ61<p
z4yV4ih>1yCS!<J2bNY(CNw*cG?RMN=XV7fwiN)2Yp__$Q9(z(|2rPR1Y_F`js|#21
zIfKYJpBQ{cPXbAw!c(=yY{~CdL_2n<T}VoodIqqA1XWSC=mctsffIR~7hT_E*3wBO
zYqGYea5IS`_0Qi6gT{3~;J%dP;qRlX-cuzV#=epc)Py>f%0@IP4r3pI?3VHZhG~n=
z=+xPM>q<CfU6Jyh%#BMZH-#5%*rILvf=Fgp>z34$>w&D5)iIN#;=*n_Z4Z8Mqdr^e
z-Ktb*3(w82oAqEn8Pkza)nT4ai9m6W5v!agY3Jz9dgSA~_iE@Dk9X0jIb&sS1di;Z
z#Q0sO3Wnb}J?X<@`QpMTC9Mwg-f_FX``!!NpftXwWzoOxnsK)pDacf02ZO0C=A$is
z8cg~*BLRKHYPWG=T^w5cL2`hNRcKYVkI3wmIatmWoMkRI$Qr$+rPqs=B3UX(tcF_>
zuTN0h5eEz36c(?M0IpRiqQ7xtQ4xT|lz4q5$OH(|6t_YdR0Z=~tfSsRu<O$du8pHu
zovxjF&&i8KlVp3}bJY>rr0B<JgVn<rYS0OgcFv!@48)}(Gd$g<dBkH#6H%An@ZmU@
zuTSY5^F|rwV0%N7roQDX!!V=$?8v?K51&SX7olqc`LkwcKNj)Ye9qF}lV6v>-sJx2
z-N5)ZJj!z^_2lcbVrV{$Am4)S=)u+2Mp!JpwY09=|B2e?a#mNWnqh7nEv*wBXS)a7
z@Ol4PYARD}fKI5V1QznOf8ystHi{ZFb|)ROb`#WL@B@3d;`BO5aNS@8o2??GMW5A*
zpRT5hl%&;EQ^Zch-M9r3XvBgF4R`n!w}*DGv9F$Rh&FyacpMYu;Es4tTU!J}l>Yzs
zx~!Anu)7v%Gf<eU5DA5g7Ba7!TJH(NJvsmD(8Ny@{%18^n<On+y8xrPo8Fd+rrq@C
zYh+Ip!}Ljs#ch>^n`AELJ&6_j<RuTg#`t1%C)t`F5Gl1niXvnB=WmO|yy!ojmI{4Y
zTX^e=|Fan~_X5rJ(Z2R0rc}xTsKhUpyi!0aTb<24o>E-{!}i#ukMCM^knjGJ)Ww*q
zx@zPH8tU!5(8ss*j!V*Xhx^{I%`X2=d&yJUK{Ml1pXGmKhmlw#9-Y&{38cvqW8`&*
z_SDhUQY#n!a2{@{^<LlL&MVK0twIK1a}ifR4+zI-BlNu0tPV}DULX~FlAkBIyS)Q!
zOdG9bznmG`2>!Z1CDvFkZK5!6+d?1_oFWvBqfm;mM!j%we^u+BS!y~P6V$7W#k*Ij
zlnV;dgz}`;zhuK5rX_N1J<%xpL-KZj#^&1CZor`?l0A-}`gnKplHPt4Hb;8xJe`P5
zVZ*st&>|1)YOXWVzSsU*XD1y1iwEkvCumhj1ldZ1a!Ic;vE1|*U{}q-<w~LwbAtCB
zzbPPGIC8{m%u!Y!jD<l;SRbcVdxYQ@eQcPJ`uj<csngSkoo|HBd*&=2wNP~)mzA;V
z2}o#PCdlVs*cHr8kUsAWl(v=SP&ELZF%vH=;PKL!Bvmt>&Yf6`usE@}?GJ2DErhFP
zKYLExDGfC`>c3&V`Yimg;|Hf-N3Gmr&gbm5Vq+i4Z1&#-^^^<PejPr2k@zq>@9V&+
zNOQ46`~~nps3L>1ylyq29lE`1RS@)kzqo^BYuG=Q4kwmwKT2>^MEU9SJ?8xfP4_;b
zUQD{2!Q{g14-3JAPHY<^ulohaJX=jkMvTl#-F|>G0123d4SOpwtvP5UAs-e37y>gO
zn&!*i2iwE62z&p|&OS{WjZnRF^G<;w**cjy`vZzGCAFV0lU@44e!?H&V*TfxkECtc
z?pAoBeehSpHw*Zv#9(dJsH=TXn=QUvC!IhmEZi_@D6^)BuA$m5EDZ&+owIcYhY+b;
zMslk#Q}@x$EqxPN`-Rz3>W*rZ-|hj~Aq~xBTotNQLtU8D94D=itvjP_g`4uU=HSA)
zX)zB!szDlMWtxloJAOPDgIm(X5Av4gKB+<t@_5UNjj@}k)+R9PU~6r)T$8UkHX_Fy
zV@+%XAM?My-QBbfng-UeY&iAL&dDPl*(`AzceaEq-*}Tu`nq&qipM417aQZkWss#7
zQk~N~{}ki=B)-He0OL+G2tqN-jXofxvQIOF_VnU!@FN~Z{gg@!E$o5`RkU&}^Rr)0
zxLvbmOqccxto5P?WCKyds`uRUZwH_P&I+pwoxvB*=blj|Fs_VghUc8W2n?Y|9a=|`
SxorP-e_c(ZyC2jYqyGon8#n&|

diff --git a/modules/web-console/frontend/public/images/domains.png b/modules/web-console/frontend/public/images/domains.png
index 39abfcb8657be40d75645a0fd2c8b5a3eed97d51..41c047066ee41b8352d246a47da3dd31688ddae5 100644
GIT binary patch
literal 22131
zcmc$Eg<D(Aw{CDRlv1R4DeggwTXFZ`7NAI>IK`nzaSiV75Tq1$3-0dj6!**b{m!}1
zIrks9d7hQbtl4{IWoFI0-&t9|loh40&_AF9001l*X$e&T009gDz*C_jylQG!#PeP|
zP-S`bFE1}IH#axW&(GJ_*H>3pLqkIx9300apI%<>US9tG{rBVLe){?Ky6f$imqQ+I
zWep9DwU-xh(c0;o=a!Sl=j$F0UW?1yqw4(^0S=CrlZK7!*Q+<~uU|Mg1dmDij-Fm_
zFP9Z~xVLUD&N{xE)f_IIzEtRO4P3pfWhy_+1Zv8U`A$9PadIezEPIG@mgk>etI=>s
z2E4pHs${Lt?7tkh^d3{GZdZeCx-VO%u8%FdU-t4HRMlU0mre?|<K_NLg-EL@)lN@O
zJ}xBniKoK8iM^bR-EBe7A&_&{Pqus<xytONmi+mj;~od>ti;;2ts*ou+^(8RPpLjV
z%noxK*2gV8WUoD6&fXs>mON~Csh3_g3FZ}9O!j=O@YGklZ8L1|=x}d73auPHVzj;-
z^!9`O4TwoDc2X4g9o6vZ5!0T!5ToUnwaT6P%k!=4V)W8L)8yp(`iMWM#a87Mww=Kh
ze>XjUGu3cw%iizaxE=TVX6H%$dw!W^ZJRasW0|UypzLvB%WyELNUn}ox45h@hf^)F
zHq0_imZL4t^L#9Jc;oKgTWIj)9J+cv8rfXKUv!@$yZ<M(Rwh0tF3DK#^F~=~Irx_(
zYk5umAE>cmoUwYOqh40;#t+GsagA7$_@k5X>EzV(n#QF($)vT54Tto$`PzV8i&RUi
zuVz_e<9{+iRt<9*rfUId$I^D&<F!S0N%;!B<~gea3#$^k?tj;_1C#4Ed<D*%Pqf3w
zXNHUaG+ZkARQz@N-kV|h*HE%1d8%T1<Dteku5tHrcf7waC5<#LG{9|pp`T7-q;Yt)
z(0|7^prC~-VCeVG%-{tz8&ny*Y^2nZ1?fuRFXFM?@z$yTq48sIxpUEjW~_Vc;qhT@
zW9{hZ==}Wr;r@PmduMrhxkTR;8vs!Al#viscQY_CYIr&8P+Ra-VMCYf2nfC*A|w-b
zM>Lv^)L%gD;*Hc@DI84`qwZryN#sZ%mWaj^>*!#{Np#xV_{d=OK^lEy?IZ4|2o3f)
zqP%^RV*g^CDpkw3l$=@d&q&k-=P!E?I}WYvsye|Ulx^5<e>~b3TbmPa#0231IpdF9
z_S3L_K-PCI<w7!efC_m-fdRbl;R_T9P)7z7VFD81vVZ^)Fra7%?7cp=3dn*kNXC5j
zI{SH2UQg7hO9WsjjTa-4Rv-AyWs=TvqUhKta&tw&ox?bJb8P23815MN{;()riizx2
z5<5iC+V4a`%RCL`jSv?LkJ#VVCoE;q)K>jF)ozIgGe$;6jyGX8<=-Zo2ksnV5a@$1
z$H>fuWK&{hs0y5QsBs&imW@@agES?K=kE@zeFqCRjxgTD0_lYX#rlya!>BoS=e8RX
zo||tVR1!Smjq-yHEWqBBgt<0UBo0#R`KjB^siLcXO!J*gY*k?db#61{(!-&&lJ0Ug
z`t#<=@wK@(MDJ$c=i}hss|dVkV^8~(7KLv|z4=u{G@kIwa)f>Gw12xwkfgvHxoBux
zE3$fozE67-m7ePJ^ngaqyk6}4{DZ5;gT4&R6P$TFGov4LN>p2^Jcca$bwddR>T-j1
zLrEdZFYxL#A6QB|y-Ox|b7q02{>fci(A_Y@WWIxJ!{io_S#cBa?RGW%l?T|jj>)QI
zNcxL;Ab)G3{>UfYg!E(jQ9g!A@iw4ogq8(a#;Oi1Xev~Dei&!{mkRumcb-t$;WD;e
zlu9(rxbh=q2p{95_^En9f~)kBw@G+JX{GHv=wsajoNxLg6NfJC@rh&rQLH+cy>47o
zvZe+oIF+zX0!;*I(U2Bmln<JCwZ>r=$#Xvbp77k-TlM~QZI~!el*-FJ8P+t};jbTw
zvXk*c>Tj^R%-@3>>UlC+mQeiN-`2s|)i;W}y8FXTcKL8;7>wBlK*zPd`WJXU!QY&f
zs+b;qso8=<Lst4>3N_BxTxP?KLnQr9%LIUgx{wp=frrcIT4Bs9@GK!9;fe9z*!w38
z0SpMQVdy&uP=1X$gxBc&PxCc8Ujw!6Dt?{bwu6okDHqUQ@zhf59Q>WcM}R|UKt>T$
zm3fg9!}*v{HM)x^j8ly8-GQnk-COQq4LK6FkdGI=<DF$yPL7Ol)I5I3XLypj^U%d|
z9Qg62;}Mu;!AoC;;|KZS$}5}22yX)v>^tu$nP&G3l3XXT7ux^Fx!3LTuXF#`&n#%g
z|8;kbu_q~vSk!9WyZbFx!jlI;%jr$Rpd)E$NqkPxkuZR*WBnm-=u9j)2k)LZLw|B8
z2buv0M|(5;E+T=uK8XNE(pVTV#O{NuJC;mk`gO<9%u$N!o^rV7YK9P{6gEZ7fN-`y
zl#d-CF(5V6gX`#2g=_le3of73(l5?n+1Pn>h0*T~L*)tXdBB%%BSTR%j^-}h7H>|q
z6fS=K7-tC3@kv)~B;oC+1%KBr&`2W)s2wR4O*Vqo3P0KVFw=sh>X`pflm<>(WN31G
zupLyMxWA*HuF);BC761Wuiw_U>f3Ds7Lmq?3Hl1FnD6&j*!Xv8f>nOx1EWUxuMy@C
z*kA&NG1cj+cIGU1z|%?Ml9qX~;@Q#f;IUaH!FKu2B{8}*39}&Z@G1=1&}=A;s~<9}
z%AjAIV*V|8Vd`0+EkRTvn~dUHu%pg=V;Y}E0ciDmJ@92&r4(rTU^JBiu5~fOs<#rg
z%>;|>3%4b7P+q}v46UnfMvgywn)azBZx-HsRoLxW#r<lKuGH=33|?scQ2>?iNaYO9
zuX4TNP)%kvHGnddzN8$ggPuAyk=d+YPNmdy8xmCLa4!c^a)CX0^y$?6hj`-*>?{Yv
zgAzc~6rN^H{X2cfq@_iV5}TbN+UWpFtI6FgsE}4`$8_^}^WE~T$Xe9R9=ZF&j4!Hy
z*00CEd~);&KiRq1&PFT>`4waz%iDLieCw~Zy&D&K_DuM)SK6yuF@Z<?lAw9K?de0T
zfX1nrb?n)cn{72rk8;Z((c?%`^&JONu7(rUfYXQk&VjH2yWqXLTN+_rwT}MsRcvgY
zzRK~r!bCN4yGWnnSg@=KU(@7SVD6xRFpBqmb$#1s79v-q=y=ikY!xlt^_-DxcZZ=n
zCbhp2-5Gx(|6ayE<ifi0#?pf;wUR$3)M@SB?W~V!3`k{{VpqgblexvE@GJEG%pEL%
zKnMD;zgbw&RvJT`QCyQgljMC1mreQ8kQ3uLVli)cq!Xp2u7xk;KA^ag%43JT6Wj&(
z8IFp8>YJ?c@#hG%%0D7&l1uj8dkL%(>Wc9qTegD{{4c*Hxk^Ini++7vT;K`|v%(*i
zG@&Y)DOpV6p6Guh7u;RpIsLpJQa_2QzDmI1cp`Y3{vf?`(R8|GcSA@S`$6%k>zl%$
ztPK0%XiB!X?T<lsI}MrZyX}H26hQa0X#otysD7Y@mUgtUusa^AIzq>u{HVy4Ypt8@
z<MWIhn_3p+3>^gO9u3j<*&C*ybpz}0wPy=bS#ENS&Lg(<^zbdt86N}*a0;5$^b8^C
zDJR$RFLH3<7zoG)3OQz$$xwc=TLBV-cPbznvv_rYlxFLTfsYJXtJNL0XF?Qze|_3T
zo%It^wDm&kN5>y-8e2Z}3RQV!PH~X+1xf$8X{lAhX?7L$XfVE4&`h$c>6zGoeDA9h
zgO;c1y*(@3Bq!eq#@_R$PLFZ%a#rG@{%JbeSZ@={F8;{a);R{1FsAsdgI`QqOo|pP
z93=rl*7)3G#VNS-GhX{B9mtBEHGK>{*0TvJi8Yy^1{@|~##5e1aHK%IR8?g^Q?%Zt
zmq2<(@IKP#55G`CzF5of#|%1ZC0jJ<EBkQ1tE2$e#b-B=wIcl#!<0E+B0E>&prmKJ
zC76pvL&hD4E<~(P*}IwAe$s^5lkd6Kr=aS)m<0PX)f|D}y-x$HzOa+nL%tFffbmxP
zYr`wS8zcG#<qz32Vh$BP%4k()c7p*PQ0>0Z3Xki`OPH30-w0GwEfFIcY{!+@Js&pQ
zmH`gEl;UVc{`n=Jv`4iL#fh$?P%tV~;Cn>?AtIAAR>Jo~KO-!z9^E0$Wi^t_T@(+p
z^cWDGU@d~wBkgSwDC4INH?Cw^h$R!%6#%DC*<MF2NJ>ho&)1r$+D06awmA*_XP$9b
zg?SPv4-*Pe&T9)4e0Uo$cLPiZ=Ye&YIVF4`&xS;K*)eJQlLwCh+D;$ZwO+I{pnQ#{
zh)Euo0KhTfs!0fWAx$T+ip4|FOUG%)=$%FKo9$D%O@o^ni)PKc*u(DNb1eehP)ea%
z>heN#H=H(ywPm6>W!Ogvj>jnw4PK+ZjreZb$aN&TMBR|&sd)*@xtF0NnniNTTazD=
zQ!0r0%z8spZ(aTFF^l*mt6%V|<x8C#2zHl|$jf8DH|d09cMgNe^n9NR5Rbf#*4bvd
zPWp7q!K0<EB*9>E7kT~`=R;?0ULWroBH;=-PdBq?%;cR3#Z{r%wE7_n{T&2}8SIF=
z8zWKE=lPPy$6h!)Vo5#RvIJf9Xz+5>@iteaz#%=sb8e<UlnXBjqaW@OAB8&Ujn09&
z3Vkn7PTzd`yQ7U@bFKw_6SMYK_XgfRlPTISEf?N}7QG>R8E)ERx2?6&&Kj$qj*n<I
zj^-f#)?_E0AWq$PMAINilbusi(uG4;bFSOq(<2ZXQ68`+24uT$D8VAo|EYegy95JV
zknAwCq90(+_0G||AU)EU0PI#|6ToxbY{PQReecK+4OSS}89it`S^jzi{oUwM{jffe
z^xNxfm5L0uLOl1RO<S7oiET>c{CD-+>ze!hrQ&bx$kctxmcP$jcIemgvHXDb02xk^
zbu5AWdTDOm!hJ8wa$A6Hq|RjzmO7lbUbVXhrycP(nh$HxqQXm0YmxJpQ$}&AkEy<8
zg1&=EgcfLc_-8*cWv4%wPai*Uu!3i_xvA^PM2v`K-r?_g3TtoW_dn-Ast3as9>g*%
z;X}ZjyVT64qtHyKm(6$02(1F)y)MW|@ZV1pK;xcYA))VSYh^I_IHA9reEII?6S38e
zD_d)Eo=8sB+Q@Vtmiml4$hu2_u5g#T4T4t4fI#d@2wXP5AjC5zqh<MS1M_stl!;ZE
z2TUMktq;mjM1hwRlH)0FL+VV}rfJ5u-b@_jO`wr52mq_KeY=K-%BOH(5K&JV{*`sS
zo~*jQ{^PiM&k5=1+zGr91%rCOJD%+HZia)!v8;)eOvrd0^X$mWgN&3EBDyr5^>2t;
zP4iHFZd9+><o57h7Kb})J|;|nT)tHHo!cx<h}^`bG+m!0TjkY+W5s}Ha>hS;#Dn3h
z!*`DQfzM0<@uc9ecQExaa@skTm1_kWW@7$o-5#_jZqYG#@M9cm47|C-ui!ac*|Z%0
zoH3EoKt}8KaGc7MQWDJ|h~Jb~-e=|t$Q@fLbV!Q4kkJ_uvq1ryn5ghliccD>L-CX6
zn>NGxGp9Q~NrQ_|CNnLsrdK`c{5LO3_B&+$;5r-9GqtH>F8C!4O$hCe8{+nF`__E;
zx#;RL)-CiBG&esmi_xe_z76@8tQvW~7VNyR-ETi=Cq0M+UR1gqH=EC0^T4tv)$zyI
ziB<Bbi1m41uiFi?m(FhmW@8gF?v=bF(%ewT@ouI3`hZD}(Z;5mG?&R>F)?-3O>IUd
zjJS)I#S?)c!FW-he-?;Dh4)#C?CdA<BKAunH3K*Z2@7FmDd-WdMpJ1Pc%GG8u*_2L
zNoyTLl+|3gEF<0v^!VNNJ&{^FDyD|Xer){2Ob*Hv#tA{YN;kmy-5+<<68=d%IyvvL
z!bn7=SCBQAG*^;OXbPDRe<Va#;n&sd-?wc!kC@x;5|;tz6e1#73&N!Ydqn_IUB(m@
zH^nirFYN^EUU{drmt~N3UQfdUBwDSgn5^cvgGOnBeHNfcN~xz9x#TlB3hm5#{;QDc
zBQ235k<b2G0V$!SJ}R05`k&WVqPmUR&kWG~$*9$7R*pB49%<Z!R|pC9RBgx$3%IEV
z9^P7y+$!i(@VW5|yI>uv;kEp7a3$0+7a@vUY~`fwdpLJ+l-yv!A*~KJ7w8YzeYbtW
zRy#BX6gxq2aA+oP6c1o;*HplA;a10W@<SAl{4m3=M`T1QZi|I%9fGcq)YLTwM2J3*
zqy?>DFK1_0ho`dOI{P7#yRzEuWBu8iQeZ5pwqYRP(Q$=eA`OBN>Ck8k>0`<K^+rt*
z7q=kd(HAjc!SU0c8P_#S<O_DDl`4arJJWV98(Sv(b+VdC$KD_~1tTYO<c-iwJH=FC
ze7bd;EO?^<6By5_MULJSUy4D2YyI|j`9HSjKB)4v?#f60dlQF6{0;BIpM7{-2ft{@
zfL~;R!%5q0=){L1blh$-Q#f7MkN>&^_#%^W6EGo0LN1=ef)k2p0CM&07D8H<l5TCH
zy3;XlKlAPmK7p4(?q5EC=W!G1&xZ5G_!jtK^{r~yjDLDUU}%pMM3bc*NxI4zNeq9(
ze5vD;qXnf{sPE5coTN{@ySn1uZ@!)=4`wqX^cNIARDo^sc3Hj>@|4+hL8`;w9ansU
z$h*MR4|NM_o@^M{Fxe3q!Se98fZ5dZNOQob0ox<c3a}mG9=cReqS1>u2W(2n22vFv
zh)^r2b6+nOVJ5^ByaC{2sk_VxVmxmUoN62oDueD!bG@jGz!chQSA!G$dHa7R3}IJm
zLir{&#{-2!bRUE!YTxXxe<eA7T1Xf+>|1k4-BMyhUZ=g8xAi}}cTGshbl$GiCIbo)
zDf6O}1L5ZQyE3>(Y@*0%jH?7Q<^ljHHhteoQIP|4*37Kyzf&Qi<8PHf?O4=dUh9Y^
zR>~C4snZ3jI0Ikk0n1_WMPB1v&6c@hcGBOdM0Ep0aO=va=r$0_e!d&JnY$1OFvXPF
zgI@_ezmusxPW}}b?fuyzGE>0k_6btlWK$_$b6h>ip3pV}Jsc{*>nxwX`Le4RP`AIX
z$y_7l-{9(Isp!us=@nPQNY!DYEfrADw8&wc`xEdI4^FxsBGbiuZS=}D$LDLC5^2^8
z%jnjY#iyz-q(4s*nmQt1&y{?KU7O9(RsR9Ia2*yMM^52?|0URBg>tuyi1Gbbd==)x
z<<$JNZyPe?$5Uq1#oyLHK?Hh(57(x(@;--4>*N=T2<l|5^;|G{tt7><TXidvPzFEt
zvxgkvpN&$vdl_XfT`EtuM<mrv2_Tfa04(MQhCC*Gz@AjjZ5bS_mXKQ(az4bSh}7F;
zWX3$27NNaV;J_g{rxR+de{dA=h8B?4r)@KNZujhPHL(aipOEsoyCczYc|Lp5drAVC
z4|_Cl!NRj$l+TgPwSwT0jINMQ(k?eJ0HD~2?8sXxLw;@XxLZABEZNmbsa7HxS7F}s
zPcyb-zTgE6VrtmWoE;ry_uWsUhm2N2aP|y)JAU4Udg)ax&NB$%x&+wktGwN(eGJ$>
z?fjCH=fVTcriiAKU)|*(@9+5HgojmTu<<Xb8Xvs4*Y<Fc#|)F>n&Ei%2o~J?iijI_
zIeP~IogP&kjal}ss?3v~b9B?dBRr4`Zjfx@35;#oPW-@T5Gd>t&a2xT=^Ozvx48#?
zC2CzTVBV~e1WyCk>cTnrD(D86CIG7;%F4`5KCG*!YSBiNM+(EO@P4`l`&BB9tQVXs
zppu$Pv28h~&Eguyuf;Vb;A=T@{Q%t(a3>#35w_UGs@q6T;-Lat>+wpScAYJ5i><}{
z84o^2>jrqdS@+i&P>0nLy)J~`tm=?OILc^_Mt&)2?$wij9=M9#S+j+8)77xC&D&{D
zpBt^if`28~FJUz?-J@ZcXj9Y%gR)Gg-n>`)SK;y78FqZ%H(lT3AY8rXMwJe5jGzf%
zL{LO5`1)lpIb`z&eD4iQk7016&Gha@zBt}yR^$zYl*^@S6p|N?F_}*C_%tNAo-3<N
zO1)fN{0SrPpS`=>go*2BvipCfJB#-+EFeb}NWcE#(Q#mn5>4zG$EXzK-{ZY%b`m{d
zYuUBmhj-5t3fV`wx3Rf@n|Ts#g#8gU)swMBih|ql!5G1(?sl2B0w1BZ?^lbLHlAH%
zqeM04gsO!W?cMd}9O?5?{LOHA^P~fU;$OS%7U%xU+2*d7)%cryPixp}BHF<MHf?PL
zqu*CUo~CsM1ff4#_<bV6?;{l7Na(_?qWHvghT__iD#wfXp}puBiA}VJt9Tgant}Mp
zlon<eppS!?B<uRz&ugf}z_HN!#%rplOjl4W5=ims6WMdu8nGvR6^eJU)YyTQF%c&K
zR8J~T-sVuf!9BxXki~U}2%;hM$>X|96@kQni57oiD@B9Nh(Ptk#jDu!!_PnZnGit`
z>2@-IZ@BK2r7O-Dgn2nde-}Y&7;qH#c<l)t6&g<QkMslmDPXAo3KP};RQ@{v`5LHE
zKxtWL|2Thfv<(v2EDc11GQC~>-5lS-bQ(rWQG;U!dUAw=60N%nn&JC1!G^DczJ4aS
z=0nW?J=sc*2$F<+MSzAgQNdnqGsSWJ3E(-7Tb(raCC#Zt3@9-Sewyf-)h;-Rv-6Fp
zie7X<^Ttl_U>cBr_=*>QXR=>J{@t0#$eOTJN#sM4-}ONpMYAd2v$D^g24tS90}XNq
z9j7rZW97MtszPh@`-XrTVK{eP1zyRhu0JRCV?X}H7z56y3q<MUiF~qV;^1zXzPw#Z
z{7Z8OCQjQGH7C0jgdCqozEy)KE+s|Zk+r6#j%M0CAE%fbb>V7Y3T*VQ?g2!^;9lfS
z@-mGCTYb!N>olaE!D9n=i%1_OYukU;X3aGKm*bwB=^{3b7&Ie__xXs!-e79{+Hs*j
z&P(sXfhTFma6KRWYYs|_J#mpnyA;z#^Tt)h$H<~AVKe_k*mEPkY}Xxcs$7DI@z}bD
zoqdwFIH#n@cl`DX8vf?GnDz@?Q>OYx`H2gwdPrT0bTc(fbz0KZob9DKALsIk8QH^q
zc(S+z$o3|c*5c|Z;I7m?-K`yXeOP7s974co__w6~fJgXeXi8>=gw;|K+uXS&A#-W-
zcs11Mn*x7vW(FX`^Lld~n|m~YEW)LmpsFf@VOeTXj3f4qxb02b+tiq{*1P#)%ikTN
z{>CjG@KyKlu8JSTNzbKPU3huzM|&NXFnBxxM-CH{i`WuXkgrTF=@W~fTk;zDV2H+w
z<(EU9H0{+-D=I14OthWYZYnt+)j8bX&t9g-0TKk56NS6{es&AHAIm6zz2)AWS*$Dq
zH#2YTRT}NO0`g7fC-}l+b_$PThz5wAE^0K%+QK2{nyQIxYvZ3~T3BEAco6nn%ldJ@
z)*&-L>l=K3ZsuH_-q$7&`9cHsp;Sq`+N`B2Tpvv0{VrB(9i_-oHio)wwnS{7y&<m0
ztwIoueoMic=-#g+8>koi;5Cv5CR{pMvLtHXYsjH10eki~oFlkYn;ZH?&#=Sa3KJuY
zY4%Cqfc3HnAjv+VeVb_B;TnHLSu;d_)<I+jN|U7L=&ylIAN2Y}j35bo3s~B)2}Q9F
zsY{71=O190tkpvYO^5Q+P(ia~dpK&{MRs)FCqTx^o0JsqvGBLI;ph5}>}{;25?X*Q
z4OkV|Bgq%DJP@QA{mVuG*0rcS6Kd+}=K7>I+~hw-hX}E0h+DBB%h-7Kum-)AJa9sX
zi9k~UWS_^g{o6LYCe%6Z(d1h)KJDG5S8|=ZkZT8*wvlkyW@xsHlMJ(CSESC`8G?aR
zb#)`G>c{(x*3TYQs}2@<gG<GyzvTr!^z!X+xyoyW(dO00O}<OjHUEGvu)UhtLTDgS
z{X()=V_esnNylx$yPcXws?vy6!01&onY~6R4EGNN<B&}+V++)Y5@Ork`Ywl3nDzah
zsJ-r@blF9a*Z90cWs>-l<42#XDQ^S8A9H7X9=2OQbQMmY@o-knO86uPQ0UY2hHGYn
z4rT=V_}^C8_i%+v{u1VsI$CTKg~chLsQ$ycJk~~n<)Lp5BH0}t7+65(DCcJKodhGn
zM{l24<wUUk9XKm9RFW1Yr+f|j3%}esYs>Fv1oZ1MK1qd&svPI=j71G*#jE!KREPU?
z-NbnAhUodlDUFP2+Nk?AnHDAC{kx|_LL@jd5OP%eYLOCfjUDi8`LY<+LrHd(MS=QN
z{5?qe!B_B)jyAG_D8#SuT+QusTjY}EvPNlOrMHSO?(r)?FZj)D9Uj<c_kDv71*99D
z5g>pdpHc5KS*;kQA+~V-F0=Fh1F%JU2d)hPqD>hz4ip{(GGi5mW}qEV4r#L?oG9ci
z;y(hC#4#fh{YRp~|0@1V`R~9yu)J!J6R{jWQrIJ-MIg+=vyM#^LcQ?-NrWbDdz~Zx
zgVCivEF#=g0+qo-V$rXusTDTn-A;qxtZv<fn*a(}KhXA{XUePn*LUcQ%UMBuJA=wm
zEQ18Zsfp_-k?bT1{6z>~3>g%(Gf7tjxl7$mvo9%GL?%9WWB%qQ*vVh8iG3C9QUc)$
zGRKVm*%JrO{@;KsU{dB;1u*E{fe@vxSpD!kk#i4b;^-ZufH8N)xA$5F`~R8HkwN<e
zL#&n8>fm^k$6%4lY$^iPHjGv^=!{MOx(&ysebg`+j<<ZqVx<F|1$Sq-{3?8b`w!Ft
zkJ7-6aPsn<X`hBZs^R6oyQES9R|Aaj-X#PzVhON+#gt803#lZnm0&p6bZsJF=+<GT
zUQuNxc20nE9Q+FlIDirMfWlDRQ)0<#zEOL<7zyFmnV>f9L<v})#Ih;ayL2oyrrcWy
zKfl7J5ehRxZRuv$INkj=)^lZE6M32&t|{8@iyF1@ZwLi1<(Nq56lF|13nKh^g}=!|
zjF=~tfACKORmzMbh@wgb?Y=LL=6vOe%zveT<umFl8{K=0nm7zr=nPg%0&V*F>x*f+
zb};_0L~be{F-x~ebANmyaq<ZGHfE@jYhT`~39-U}n*Rs-f|zo3W(2oQ99z^_-m(*j
z2lhn#87qE(_8z=rqDs;2?JOJ(3BTNHC#}QG%?16e?Pk7s!26{Ais1uH9#jQvk>6n=
zxujISFE#7{X6JAxLu3Of+8P_0NJ9IyIllx34VaU2z}#p5A!dx!zLBY+-c3d+iU`tj
z@IIuD7N@ISFu-bFk7SjP%R1S25_@g`IIDOcWtdRm$x=n7rUbwBko|Xod3affm-l?u
z3{Z2A;cUa%tO~+qau~9AmbJicq9C3GPD$S)aD>%z3>4}FImpc6t<>%x43RQYxl;R1
z;HJ5<3FPlY#sRL4rI?MF9hcM6D;jO8xNnAqZp#S^$@#w}`zp%N#S=THa)|KqeUDcV
z-k2nvpkLiO3h68$k2{wVo}(Zk$+gSBL9M;cO<p!>(fznFXWCDF{rIuklvX64J%|?W
zKvQnv;M2=PbjYY@Y$deDS{NCR`qmlKP`Wq7K?@jh8{hu3o8~rb^Wc;4oH3n7{GLZj
zE#~+PIwMpbXR5F+05f~yk*vr~z&B|y+3LV6nW9QK*`SY(6%7F24=fW!epuc4WYmXv
zO@Z-B?wJD2v!^@nQM#BFwYw^*iU@F9$L~({m_4t;?qe)z%LirwX30k7CG<gDdj}+s
z$m{wlPE*ehOczy(=dV~&fK&uQ17s}U3LE}|Qz8&sg#WLQf+?<yZ&A-95BpO!OG$SQ
zxqf<Om{57*f7z+z>w9upHEHtzR5sD_JAxz<UnDipXK=QkZqNNpUAVS67X2}+!9#2%
zaWGzepisWbkwXzA0XnCU-Jy}XAa}bB*}vDtJo|Au8@kfyA~GVu%n`IeDHC(ALl=4_
z7@>5>$CX&dIm7Ku#a57SMm@;t@y?Bj=u`0=6rA85g)J=fC@RS;*D`~B`M#<TXbTFW
z_luZ*gXEZB7mF(mad)ynzUVOUo(E`&Q%8zW{vKjn8RB^pMhkePtS;1t$2Xw~-N_uR
zt`T4jO{^{_VP!X!%T2x4d;EqOTH?lR^T>OeYQ4BesC1We8XVs`Pf0z8LGjQB3c=kg
z%o)G&vV?q>{-yXK30xdYBXEZo7lH>K$I6$r@1Cvj>(*U9HYmceu8VV%o5Y6XnthRx
z-@d594jSjKm0$JY$lR6*sB`yr5mnd`nDe<n#nW(=xqt5P)DPS|xuBO~{;3~#74zZ7
zPpKDBP4;}gk!`DuthxHA;zPIa-JWtyGJ-~{ASWG1lkXrn-i1wnKh<l?T1jUn3poi5
zUec*T;FmM;S!a)4D$T@ySR74DI#ZX?J}|?EvCVYTIs9onT@=QKL15U_!TV?qHQnN|
zCH&aWD#6N1Zib?yL5h%YXqQTs3Hc44XFq~N;I4=!Q0S^Z>wk}jia{V;K)lCY1Q~dx
zCu?V!OT<;@czU3BkaPkR3{if_>mMW2%a2v$a)h-#vJ&jez-6MCaq42ox&6LrzV>4S
znDFrVs;a+8f6{NTN)V=;Cq&z7IgM3?y01F4ofhNvgw7b=`go}mR2#8*E&}%(lXL99
zsG4G*G(>dn@6C=5;9<TaPPuvonlkU^xh-vVZ%lsD=_muUY9(uJ#*OMr=fCm-jl-A`
z{B-dj>0_h?<l0yRXfqe6yreox-@*{D8UMkv^nDai5ciW|6!9$pp=BL%zWZ^8T?c)3
zh%j0Hy`aVmR=ci@nmeq+p@0N6x_4v>n&=c>4xp$^W?#4sukN2kYkXL!3Rn6Y_6PhS
zWh)NsU!mA+V+37pX4@+HewZX;+_&5&CEI8-v(q|{()PYIJ%76x>x^cob48Q4Q43#o
zkV4PF75rm7XflcvTBcUF>*47)8Fd?9Nh+IbbN_wTJPOfVOY+B`@3w|>S#<ThVFXSe
zOfUXWp)(M@i5&ku(S!rVL~F9TQ%lIT6BSUQ4aNWcwH=&6f^WV45WTm@2<W{Telw#D
zDWx6;8f+vv{*`HUpBO-2=T@P;3!U)(q5>b46FPyW^?E|4lD!O00_Ij+-R8b$P3ahy
zvO4awlm?Ng49dQzzR$p)VYhM#d|z%F4}RQ~X*Dl$bdVOOvtsc$pRS{ICj6ORg{dN}
z9vlVi|0+!4BQLJV4~s@RF2jdV%DtkY(#JU^_`O4>z*-&NcaXPc$xmf})bv%EPRh(*
z&$yD1eCewgk0Bs5JZstcTHK|kjSPtK%5rd{zeAr0L|#eLFOq%(2i)32dJx1N0`)>U
z^{G<oLj-h+;Iw}$@(H`VMehJu|Lzw5iva_`+JRM=RhUc&|2F^WFd@8l+JXNz7vns9
z=f65)kC~?{o&X9gw%wbA1@p09t3r(M1l;=JjZh-d001MLAh?Ulw~T5Xv+BiO#DF77
zRTqE&zrDz7U7<f<=$U$&X3OZI5bw3h@PEA8pT8)5MUecb#sr&e#lpz#aqF$G*Tq-c
z%@BCEPQ_JsxTYo>-$4C&Zias3b^i(7EJQz(_C)-#FH@b>R?%W#d=0|cdt+ZH;V9wd
zW94M6t!63cWGN&G#We8zg+;41c0Ddz(i0>HHA~*f;S4^!jo4mJ<{_RV=eRIul0@uc
z(>eTmN=%R&A8wp9(-YyyE=CwMK4RDWF88N#l51u(7_$q){;%lvve^)I{UcT25kXTu
zPYNAO3O*XXC0SpuHm}nx_#XE2wm@T;xnC7QhW;{CzWj3CitI}xv$e)9KcppO54P`_
z=^`xT+)<G)4(}ch6%p=g9p(avObe?J)|Z8zmbacYUK|RP;b-vZKd`Y@gwhD5^)l%c
zST65t;|F{mkJ8sNbXzHFH<wQ8gS3vxgCrG0cnUSoZ4LC^UC?cJ-p^+cGGOYOXXuE`
z1yfR@E&i^S|F#%rVur}fkpMPMIuiao?I_l?FE1r$(#ZfBJ6%S_GHIH*nT3?a3>4h3
zln^~(v&$3KGnxC1o(=*(TiN8ePRE01Gte$r+6N&R^A-cN5dF*?SBFw~7p&lR>M$B>
z<FYC>zcS%z3~tN5oKzI;b7~!$wuMrUL0~O*w2TJnrj#p#<bYP3cM9=1ZUwTVKX1_I
zvMJN#9i!PgF|*$J0*f%IDs`{VlYn5NLmZ#I)eXTA^iIzP(2sZ>LuS&l-R?7^Ij9-@
z0^z10<nyANF+rmAaEP}drwfyZvkz>7@#+b4jo7wgp4ZtiBF#P$S38bj>KfyHnH(Z<
zkv1>-zJ&roKQt*p-`mPIC?32cgM9GdQNq`Y@-&-B>Bl>n{`NZXK|vblT;h9~1rrs9
zUuS1HD4na9{Zm3oB@a=R^{}2ZF^c^~JJ%Mj6Fq<H;lcY5NT;@iC7aFsPjK@Jz&F9F
zGvlawD2;n?gi~S6KsjT>I#FF4!LyEdQBo!e+Xn-U*?hRpG5lP*c1p0|sC8DPC+92q
zN@39eBKOA*cn-<eNy@cOIo0emkX9O?%~YWuF@QZ<{OZ6A>PUsEdl~Bj)mZ?vkC2^Z
zt<Jm22i86xy0GkyDzLE|4v@!)`lrxPi$$_Qq|}}*=j<hk&NYS7^ltMi%s*Q74x*yp
z_6Z(Kz03{qDee%8X1)FRNTQFWT~GqMwf&&WGKT@K?eZ~DEn2cB3QE5iB!j6NIvEaX
zlXS7)wUX~}>&uE(ZwYkBb@>^?YBdS_tJ?Z&eZi8+EBfv|i8RP@&!wzs>F5+sOc<QX
z)vfmDskKa)v0D0&<x3{DX|QJ?XxZVa$Ip~V5>=<*J{B^t`&W{;^Al{S$=A9+Tc`I-
z8^1=HLo{|i&d(X-1FQ0#J{96W7KR*O*VzmlK+9T{Jl_+-?zLUqMZqM~du6cLUnWJf
zK&$sv@>kQ<;okSqSfk5rOQLk*=kS>IqrSmW;*KMf^3thVW+S4JdNDCwXoUczSbHnD
zR=)KlWhQQ*?XyvDl6)5WUkBPgLW~{4{n*-YNwcrX2XY`t8uN%L3fTv9;h%qEm@ym?
z=dpu0?hbFOJvi>LmRRHHQ3t8Rk@B;95-xBmY?4E#leh-W*d_Bt(M}DKZ(!<I-(W&Y
zMwMp47Q;04`Ao9Kf|p#XjPA)kOeS@vN!e9loYzE~C9GhgmAWEx{y5=n5cib4P#5HD
z`E>>7ZPSE~Guc6)7>5b@k9k+qyYF8N^Sa)TXNpZ@POYX$Enc&Fj0kF5YTD497#HX(
z`OME{qeJu9{E!*&Q*4=d^v4WOewhwc2rCkSYxtC$aVD)Nr|ZI3g@%ykx5aAUZ|$9~
zbBjm3r_cjCBQ3bUV%6=&56~tTv(5^o9$Zpa1#9rpi@8pJj)QPyDi}^wT;>qI#muZi
z&oDijPe9(h;)e9Zg(O~v6t-7nHe7Gue&r+T6&qbXm!*6z7oDxR{E8HQDOhC(OE*~>
zY)t?Fl-~AP=W=%khOc^|qO<Pu?-J@S%0=S>5!a7p?LZvQZ`^)hWtL159LtK=Zm|l%
zmTAlA*6bCv*=^>z6hB%m_|#2oNpTSpiscT@QhdhlX*b@+_x@YUaxuOo{E{02BPt?1
ztHtc{F@*3aJ&>#=La;vBD4_=#!-ODlm}7?hm|6Ezp^$E=PBxolnNn8MED7WO9mH4Q
zo)v4l{}^XPSRZaY2MqdGV1z-;rvJ+SR^CY|5oJu^hx@;l4-;m<&+%_U>MPjZw5>FH
zd^;KDQVj1Y`e40VqMAU+U-um`xi;A)L~Y_LU_pU@CR>xn*Oa-=pWcNLP8=DJW{tVb
zbOO|FTh>EYWK7;J9B00NGv~q5(fdh1<W~*zspFa3=pS6auCCLxS%gj%F`WNOaQu_p
z_^$-VYsTY$DW%Z<d*r|KAs|p&(I$j3rHNw1H~*GHq=e-c4_-e}jvC$}<uA|^`Cxi5
z31xVPQ18r6DJ9yzkYiD0AlP4d=7keCY{>=o#o*%kh^sc-lrIhihFUCzr;8`ueb#Wh
zOWxUZbKspoNf4sOZI1N~tB~D_f6tSdNj$v#c<!MOPdaNqgqaG)N9&c5d4hSbSAG?Z
zA@T080ZojRSiv+NK*UyNm2$er{H_5UvY<=r0@qLx5@$%)2tKIWN$L1fi@9rsJRB|D
z56nL5sv*#|Ua3oCvb(Q^dxLFkXEO8Fw`A=R*K|{N=ssPCduXoDE0?>MMB`ofQ&1cu
zL;KA;8(S8Tq?+Zy3k;Y?JPLmQM0}XqllTYByK-we=^$R*=;ZnY^1&d%=ZibO0M@VQ
z(#1TgjnTPir0@HaP=s5mods9{M!kAx{KG#hEP925NX5B#qJwB|y<<Uz*&a#GHq~Y@
zOiR$Xm57J(KANSFmI%jb5a)4rS`B!;ldiQBe-x<qoWcSyc(Syy25g#yeFJUE)U=v%
z*4NXmt^V?{#E;Ms=-c?T1ETU_x<{{(Klq9w5fX?205WS^<swURhB+JQU{#%4C7E(e
zCmHchm6A{Vo?D+QW?9D|i6GM>`<85738kye#f%sTn%=JR0j*r!$9l{0Lh`}uyA&k-
z&`Lvzk9CdU38WZDeNk8*1_t$O#}szzlM&!41}e~X(P*|t{+1kR{}Gc2pGi<+d<<Va
zssa3%)XVHB<_|_D)*UCeXHb9;wZb*|$hYn>bNwJ*`&p($Ln)CvudS(^^n*nWZUa(R
zF7=UB5#jeyeG|9A(_D*d{;ga;mHJaVDOFL0FZSAYq>3AhnNF2@$O>~IIxbCu10n48
z&(P&iDUw^X!f@n(Jwx7CKYD4(M<sknVBqC0@LO`LoP-nX$HE6En0&Uw;|YY)CmO8%
zBr<)HW%O%0%CZ1SE|74nXa2X%n<ghn@gwaU9nDL3>fb{rtEF2Y<QflwIWd@n6@*gE
z7$5Q_w)g#$(ze@GE*Z-2naoH1NQi#WN@Ij8H#5d=co?zvQ$vNwGX?DD$biE#v|F|n
zXSmj=7^fxN8F7<evZb}!WRA?dZRsb7nD_;wl$?V~+{?ziQP;<#9e4`e7$35n+t+8+
zF~79RUxMz%iSWU4J}*64;(z&uBTF56e1@WVY?tsxyE~<3f~suG=VPNa`BRAFsd(hr
za@3GdLZs+54$IR?uv#EoMa;B{Z8y84Nockj_mQUCu5$I?26MSuOaTK0{!05MvN9|4
z=tq8_B64kTb_=d_HU$POmL@PS8|TzXT?45~{FzeS`6WoB*ZI*7mH!1AkbX*dCoP48
zyV}P-KUU8PUljUGH+NN?wNaRl>tSk#@s44O7KVzAJl~97X;zGoJH!V+AL9Wb6_#j)
zhBpin&JY-;yDh#8FIOmyMzS-6y!~TA6a&iH90B^jo-deknfSfY@{{20IO4(aVAVoh
zp~oN3u$A{O-74GXvofj4^UKD8NbqpnO+OpUjRd$&LHUfF@%RgPMJRdr=q=OAT=>Kc
z$^gx+=+!sjUl_{uYPkYQT-iC5bH8{9dT@Pck*u3(noLu<_gT7mSQ!~&L1<vKNu0Pr
zn{VZ@+rbKaI&(iAwh&f|k;`=;K)s5(6q28sB?Sk^z9BOSK57~f3=fR;$J>mLK&b%h
z!Sb}?w7P(?Q}GHpqECUd*fDvo6Kj!%)>_h1To9r{8xecvO`YqomzMY1Gt;TA&r`*v
zdG(Er^YE-%U`^6emfDT#@QX|(ZVOKL*c1W8=nK>7dY_n-5?g|&#d?k5#wgK&!8z@v
zKl&#xGv67b7_?QAH$?}5&P(gJK=6`^i(f|U)C1PM{hChiE|?PX^#1Jtu+FmVh4>+Z
z^nGE?&3jxE{^Mfa^x)~@0@6R1gp<Ip_g0%E$KCa_al#nXFGna(eqmbPduL_}^6Bm;
zf3Vfgvso<XT?&k3P&ne=e6z^s9D_dh)$O+GO9i*CC{J?!^Q$o-w(zmGp*AB=CG!ft
z;J+E9?R42DBh07nSfx$EnQ4zJxi#Ta+_Nj6C?VkVI-BDnJ~`0!x!-On4b)jYx&s>P
z2nI)TGQst%l^zQBDg8{c&AeOcs<RRIw<emfnKZ#2+%z2~lCvVhrR)Y`x2CajYeU|k
zBQwGJ8~!4wG9_9icL1;UHj9iT7{L?nAi-w9R`S5%H)wWyUAT2J^lBKzK+%{sH3doI
zc<A<Es#t804sI9hq7yiwzz9y}bhM&NJzCs;HF~bB@jjfed?I1Je6gfL7p>EXh@Q_?
z;QbCbN*gq=&YLlNYOj~IE0a!a*U<lu?VK_RF@Gw_bf>ZsEzBu=mU&`LMB~6z<0cpN
z@sYW+*L7$iH-{vb#uO2z!f1ZM%>v(?JgvQ|>*TB3Qqnx>BHNVS{4<B~NQc!oOy9vo
zD6>Ez*2Q~&u7Xtyty2Qi={WI|g@FSxD<sf-RYyl%_$UmT(a-5FEFW?iz{%aF!Ye2*
zy-DJcQtxjZ5MWI-3+%YwwfHsgU@VqL{u5f}W^ZGoh3p)J<L=&t{A#=cjQun$E58!;
z-q$;tHHF7E#k+05lsHER@;effSPy1ZONaY4Z@E#Tl&=15&JJO<h?qiC9L|zC!blWC
z0|f0F;q{;(&*d%*LAe+GkXnA=bS{6jMdBP(#@K~C3;n=+R_Ib}nPtiXQ8&4<Vv6%v
z=~0dR-67)?FHUQ2FRkdAkf6N_4#mANRQQ7GA)RjjdKZub29Xk~m3*ygBaLan8mz1e
z@7lSld#h@|bF5c)LxoHM!$qn9=QLo~D`VgiP1Ndn5wffZZSY|-b<J%m7!bnU&(pUP
zy4Q|yP3jD4BfFQ+>Tp4@IJLPTND}0n9lCTPrux)HYjp_*=M$D;aq45sOE&gF1Xeec
zY_~JBng+qbY0i63(T&WXJ`|<TtLz5YGs8-(vAKT`m+d=y>GK@OzW*b2bg1eJuGb`d
z8W3a~9fVI&Z>MihZM`RpsUBigoQ<-w!)yAjK3-Rb96QK((^e{dSD{E7&Vq92VV~$L
zr_XcU)wUvc$z89xYLqB;B1O@mM@Zqui_hF8;ZL9>F$!I-w(zqwe*!~AK2n|z{?rOb
zX3uK%K3T-jOwluH6{|ZIRBm<}g?BMUh7pe&x4sp7<_jcb^BG+po^j3s??1@x15^%F
z_fT<W`OidjmE@vvbmS@n)Q~+D`5jOh`+;NBrrmotNWVIOmI*(ye5oXVM66CjPOYfi
z6ugVrg9p~U<T$2k2zSA4-H+;g*GxD8;2kC$4p^-v4nlyG*d0XyBVZz7gW7}JRs7?H
z<WM^?QReu#@d;>IE6IqNLmNUp5@%YTLwCRw@K*rM0@S`+>nE~)*AFxc?SQHj{j6pR
zk$|c~I>vC=u|MTA5ob-i96k*J#e7inWrtRx`h=hrI<UqcHS8D?LWW)kC8RYXOG)TC
zD6gdv@K5;Ra5AT7b9~S)s0c}<g*K*i41$^{un)wEYTItww|u)}aYx~XBzimAXzSr>
z)Q$?pY#hDLtdl4R7b)D09LtbHSM_Vr25L(Zcb@twuS;GMD$vPvsA-ozKUJRs3Ue*p
zWSSxDkh{bCv~?ZLEn8Pf7&L9h9m)iAae+(YgL!IW_5QO@k4Ul0CYXmFDgHzw`Df;x
zWYj1#2ee(k^`m&<-Bj!&jBaM^d7jr}#z5>T-CA>DQ1_Bjh6yz%gmShA-EXR0#!dAD
zA=v&}%3DXIEEaH7`03#NFjdJAbU~9*cX6czyoP;2CrCB&CRqxk`k_%2rp-HGJ;xWA
ze_YEm!|`01v`&4lgS`294Y{hvXnuXLF<tf(EDJ==E=y%h)$2)@3NjBTL_22VG%>TQ
z&<4Fe1|(Dl4HZEN5O!f{By&Ic4)FuZ-@VW?k!UTW>#ag|3cp3aLqrNoaf#axFBL~}
zz0lJ(FNh_R9K&rf1$8$RS92okjh@e5&QCZhWSEe4JW%X;)>R^3FiN;2!3i>MWiTg*
z0}sWF$_M=8>%-jrJIDkf4Cg3!Gf6qtlG%Fy=$ox8B+RSoi(SCSo2(!tJ=;NF>x^lr
zy%Wb8Ml9T{5l@9T2|~$M7)RCpI8I)dYy^8dJuNj}%Lwar9X(f7k}5wIpIfoC?o&}4
zw~|eDO?V}{`IC^dhEDJ6GANMo#X>1z0+f(IB!AB(l{CfA(JR%iNneRfiO_rblVwsM
zPZ#Opgs5V~A`%l$%p7<q>zPSsS#Io*`I|QZkS}@RY|L^wf{C84Y!M04<Apz6luF(2
z1+@{-OhF$8X^lVd1yY+-$qNaWQOAGMhE<G%f4Jjm&b3r<SH7lr5pwnpeohF?2(>TI
zNUrxAKZNWxYHG>0kmIt<PNvTy>M~B!?mQXIny~&2nLFve@bp5b4{o~j-O2a)*m-Q!
z7?dc9{oFkC?CJ2Aqg3BiLO2AeMuZTiJ_Mi5^sW<L^zq$16SCQ5+4+gYxq|?m=+4<W
z4h$&QDwpd=ak;E$Vd}Jxy(?wRRP*At$Y9zY5;#SY<ZLHcWjm8o7*43u`vVgByjYUR
z^wTA3T40lGPGpvl2u6YQ8BbA6_p6LWGT=u98ug)xLH#F0=qH9AcXI|D#GIf;LMPnr
z!Kb8Sou!AEs$H<nOsy&fY6fnDrH(q@P<qi35ROKs4f+x)qW*!*M3zX!gPhjiJ0EBK
zSExo$L$C@*D@LyK`0{GQ*B)Y;9}z13CE|AKPSMvLA=T8D2jUUtwUb&-T@CMpDat<H
znAVNLa?lY6et>6eP39fik=iT<>;k4R9L&_UTBYR6C+;?`HlA>Dy-ydV4vcEJ{_4Jh
z*KDoK1@BDVjX0$%61-8@17RTH3o<S8W6eZ8LKMx+gYmbJ%ANqcgakp81SeM%J#t!C
z6oePg*T>`Cw6K6I?!glqRVzNW`(ClVD~iE?M$0&7p^s5yP4Y7VH@RCBW%v0ezq)5P
zC~pj_aCT07O=$`TbQ<|_^k5B;?KA}h2=&y1=<M0r_-Xk$m6Vcl;+HgLGMT5yck-|^
zS#Zpx8S^qJY6VyzYW$^pfX8r1KJNyBTq*nQ=9{Xr8`}eDNKKEpN+`N&3>dv?Xp0jh
z{_5c@Fr%*sXAIi?Lm0haEady{j2!D#tAalM_j)&8v_0wu{v=3p8jVT=e*@%u_rXA0
zn#d{2B#_hO(Fs;^O_swFcB>;`*4TyV?5Yf5c1S79Zu*xKAjq0DlkXfNUYB>TZgkum
zfn74ZWo=GgsRzm7>Vq+b3O@D)eNfY;!8Gtq>u#8X4@ltk3z+$@`{Gy|Odq{^O(Onf
zEoD0nM?PGp)spuAGTFJ+;6!VKm0dr%>-XcfkDagd#2U1nOv^@}NjWHJSY{c~;L7d2
zu4DcQTSTbDtyql``R512=x!p<2$cx>7=Rbq)Q9g%uQ%2)#S8e#kS#|G%Oi)|ua$|?
z^%A|V<uh%r(yXx%HjfU~htuo`!v@GSMX7ccy-bu@a#k#Fz^=E`Ct9Bmuv0ah_Fcc3
zke_z1rsT0Z>>hu5<NOf0)DKun`^Fly`wrqjUR@a`AzrbSqac`MuEWxJeY}cBfHeI5
z-Wol_I6o`>BQoUcKL%tv>V{A|$ZhvKMLaOOR!MNY(j&^TLSGHER$|Z-u)be^>t4HU
z5qO)8{|U};jHIy5Wz9WYg&PAyHzTrr?l{EiO7{#3_FK}hXPy*4llg{IBSUY@_Ex(i
zT$On{g92U$USpf~<H`OjZNz^r$JKsf>)Wry7trN3hcF+lkf>-OaX~{XQ(af}Q@%5+
z)vXzs1niNpcSA|~4<tJ+>2K~^Sun*WBA$*F)4?_o<0O|Y1#6%0CJYq~o)Z!tPqGvH
zLybhqh3zO^N%dgS#1kRd{=N!Krt~-K4~@|aT_UBP)y(1GPtQ!7I`voEQ}3nwJ;{FO
zV%)o>e1g}5X`^}@y@l1rKlFR)XGPrl=^?|;K02m7>-dktj^1IXrYHzS{Vr+>Fp3XG
zGN^zuAbV+_Ya=6i8TIz*Dqy&d1OA==L=PQ`{*~#1QP>#;TK>|c(}}8W1m^Y4;dMQL
zT>p#cM&Dt!+uZBiLDsCp>sP5W^=f17uH6M}3s2r$vMeZitj!Z@7E<gXM6omfpEA8X
z9Lg_z-;y;MsjM-!?CS{G*9O@_hOvtwOOk!xLr97l#!jdh+YDt1WoO1N85zpHh3t&w
zJAJ;__xI25y8k-wbHC?#&U^0ndY<c?r!)2S@UnQ&I9bOJwgaZK2UZE~yz&ehO`PS<
z%6}an8Z-%&&a6z#@BL!4tNR?%oxkLt5X1UiGHqPQlkVb+_<8%som-<`haqIx5dwKC
zv*_~7Zo!Y*X3jV@6%MTQ!UE1{q~{8|M&psT%*FL9)=c{^8|n^Yj9fpI-#c%(aum5}
z=#pdQx*OuvZ)b*0H|9lz?7&b~_OGd#w$9TZF?;4+<ixHlUal?fhPBa(?uCeZX$H|?
z13E&G`H*7&@dr>rr;*-e<ThbrlB`4Z<ax^Yiq-M8i7%M=;RVKsEibYPL9Y`A+V{ej
zZrkDEIeL527lDDY?(f^5yzhqalvBK*pzjIQ$$22^v+DY-%8iWK7XJNLv%y66s{7sf
zy5?Rq^UhG=m4+PQ^^f+krgAIZ^4{K3ql7^)!90kP?+kd<HpiGhWCWYmXbmMsNH2Wx
z4}OPK30K-y*~@e(k?zeN4LF@5lHXz*V1TR3FFxV|KC|uv-8_se+JU;eKxbKUpQg#a
zTDkE;TpNMZ|NUNT+BQK8!Had2XqU_X-^XvpHVW#S8U8Wg!Q-K7rlg_|8Sp_aUs9VF
zn-0rcdm&Zzzs%=rmsKyd5Kf1b!$N3=7kGdR-iGio_6Lk>-7u0Kq`j2*sj+;w^{Qhp
z2S=$g!Uub|_hY;`=aQZ)eRr#X^?>pp+~_*XWcAqyWc-Bxm)LQwfAfim<-)xe>9FP?
zJurzi|916!D*U)v+Gtx>(+m#WS2W()ta;TvmjU2Nk-CzS!l|pm<v+|8;Ov<7`q5;P
zy$_Nqnz}RN@nA>@Ak91~oB^HsB~%||_5}UBq^e^pJh=B*q5MiKu-_uGo^Z1i;4Z~g
z9QN&vvfUrnt#dR+*zL*jaW8N#Ko1~ck%f<ec~N%8R*Nd>Pd%k0PI1R9Jo>}i2kLmf
zuyCW<kl0a^Wy_4^EN2~p?;xd=JV17_v#%FL)hEw11hSU$nCXM$3n;P(&!)!iWs1#7
z-aDyZezOTuX9!Z?o`I-Zne!{*3tYA@-ybsBI_tpNcShB(c$FJW5Pv_Y*j=#I2!zVt
zVGBTF1GZUR>rPA&rcgd_10<on0F>fg%FAm6>vR%w@+i?3ZBixmC!c7!V{lQOt{KfQ
z0_eijZS@~TrXOi><6+frd%&k@qaWITSX#{-lL^wSy-t$L4>VJpT8qBM$>RTVK7_wj
zQVjS)RqMdU9iWLQruz<D?daEL>$AorMK8_*?t3CtVk+$CP{$%{9miFyXojbm%~)nY
z-7Y0G;dIVT{b3)d-Zb&4t^tP~NlwMP#w{D9lc#eHUhG5tgGHj5FMg{P@VcrwxDT^D
zUvZy_3Mp9`Ln0c!C;S3g9BNA_0>(aVt(J&I#Mcv2XN__N|CxEQjh$Gp!k-p+0Q_bM
zlMnP4#<^EZQZIq2a|6R6?82TbJ2R#C=s#<E;)Z{-*k?6y5EljFmSNU+l5c6^bDv&8
zO|`Wg$v73gpGxm8AN<;`l$?!sNdiMCNdiX>eym>1RqQtM$R;NZ%gT-@Z4q>&xTJ~2
zbNuNpi*s<|siI(NANn2@Yv`JMP0XCPfa2B4KCi0{nJcY2B6|h0C*hOp8oTmiQcqd>
ze#30c`ywaGXQ@E9CQT|9I+=VkF1@M#0@=gEXK78ME227{RXn2=$iw^S-Be&cKh=%B
zBc5D^NUCYZ^_Kg-uwmdWTxyzHOETI{PRjS@GirCs|NGPtN(V^koP4|v!WQPcAYP0~
zt(^Gszs&j#o-At{-EPCRiwY?BYuaGa6(+1dJ17%n#wD42kb%84jfMpJUA*<0;)YHY
z&uLgpyeubU9Zju-NujcYlkWHp%To1v_HuN5VUt$df&K1-f&y+c{Mt+7?Wkv_Js1X_
zz?PPP0oXI}D!R;uW_rlKhT97I+Xw-xiTy#9a~>V|n;q?SG;2Fa_4wZB_16wiC~Fdp
zwPq2MjxRJU>#K*xs%$}dep8(!lQt_q<9_6$<oS%)bfN%Z1j4jNWxBa9&3SB+JkPIO
zpD0ab5@LAJ?o7r13uGSQy^~qw#a=9lNtl6q9$A=P9bJ#>^P|_1%}?fd#xnst?Dir3
z7jpCQYxoAiQYa`77C>+d%hHpq#I95(Ev38&p9BvyNcf%nbbEQROLuWd#w(_<vj-Wb
zUmJnpu`H+$*>1VYy>_QGKz*|bm||u7@oBxEt9YDBU-&dtM?vYVpE()Alt-9evrTcU
z-&Kx>>WJdon-kJX=xf$%0U8a@B%JXCA;wj)BB{FpjERK(7s6h7UauEug=^YsB(0_S
zx@-|u|CUfG>9_gRT{Ot_p{n0~?$3HNqQnm>{wXZ!XNyU0?5b)?szf2}oI4mO1U*&v
zrB|{}T(1WEphdwbGUI!RgWAewXOF9i?nT+OL`p=+HPz2s?GDCfx9l^z1)M8){8}zn
z<;^-Qmu^|!=?77Wa~9-@V0i%O*nk*@@!W|-6`8E=Td}vF(5CZDMAu)=Txdj=X5_rQ
zh@ock!M6jS37Sh{GGD>*cd{%x3%oYfRfU_1+^G1`*ezje@am(-gYut3?y3l@1={cd
zUv3Fw2NDX^%fy{OJ)VCo&d(Dpc{@iJlni^fotB}0vh-!vB()4$g@cW)C&D`P&~rx|
zj3d|vc&;r?4wXk6Re7>6zv8J$h;NnD+1q<888{5{Cr>=-TEU@BAe2!zU|KvEz5(is
z!(G<W&UzuUvB?7eqMw;#@1q8|Y@@bQ0%D?(9p@-=oyYjkZ&)VjDWqz((%G=l9x_yk
z3CV0<N+dCpN2Ul78*uhZ+;@{J5kEdV=~}|?%qySZZorTcbIuM~thRy})W>m3{>~<~
zk0Ao@tmG@alJ|ZrN%7!s8O+<dHilC~{^hoBJm@8pIoLI{IOX(vHkY6OQ=$*#167_|
z5HOcqXbb|shasDGbS5K-;wTv{ZXHKE9@DH7C@XDaYKqtuI>sfir^ZF-fVKen7agga
zv3inzmg)goVzhopuIe7gS8wc!yPI&Q8^d5qF}CY{G}rhGIuhouYmR~*Q{S*aaGr;?
zZZ@OT=VjwrzJ~I3&llAw@g)y6%Jt+7k#}^NvKb8WBfCI^mI_bma_*K*bVmp$)c{CK
z{KpXPy!UMOPlmIzGq+Y7(N}xY6?C;@1FiL`qGk7(jQ9CAn}e@D>88JHcyNq^pXP><
zIW0wEy~9HZQQvj_GqV8~dYQVnOz?Zhfb78m#Pwt9zU{1)lw7DW>Z{E1N6NGd$8c?T
z76Jpu7o*mno}QjkZVg?wq#c~nygYpS&W%n4Iqtx)an^MzrJ!00<&kA+PJLGiM19NT
z5RW`?4*!G!Ht1;Qyx>u!0jr2+wSsF$vfCUwmj^g1Ug_|jH)#mEs`n(Bl@TOxp0}28
zFp8OC`<#X~YpQma`nDBf;w#m|v?fcTn)7VW_4AzS$Z!y+7Z~x};B)ggSTg+VZ|dfG
zU(v)n<oZ)w<Q3N4LGIA8lMqYJ;I<QEwP7il(aK$;TYlj|@+0swa;yH~=0&Hd!Jq1J
z@Y}e^Cj@xG2rlxgU_e%xIxt4D=jISSt8&?<k-YAh;G@`hxUnpeJU7#h`Xp(tLOSC$
zs23o37XutMN^S)<dG$x$r_PMx!Gazg|1@J^6PL@<9O~@by>4|8Bd+aUD-O%*R|NFw
zI-by=#f`KOBxC39^`}-BnxwR2^u$fwF(Vesv+`~C`t%Zzl-STXuLPs@l^eU)4gZ_L
zsI=@zd+5K{8OjX&4*w-DcQ<tu%~SbThH8r{W@4wLe3uUMJ&fA~1q!pW3pMz0Pu~af
z+(a`#1-sSg)#XGTAS)_71WhY=qK1NlzP9zgQ+y}@vQN$hyg^67LvlJRg@rCd3Wl#J
z0>ny{*~Sf{L`sf&rhKwlBEB9)!_0aUmB?cMm3y83p_p};#ZV{x8e?;3RpXk>SzF;X
zPw1$wDefPwLh4_92)M0W#n5}vl{eWiobg+1atRl3y^Vns5eC?K?Gsle*Fe^Hk}HFt
z=y&%5?O3?bB>?_L!{KB#+CRq+X*5VR1!+4V#rgq5{1<Jqx=AXFnOs0NGEGF8BrZ~8
zNE^NtT@3D}p@$}j8&s0mF6{JB#79#0PKW*bgq!M{`7)kAZEq$#Gy1L9r7(-5;hW#W
z>lAT#m2{6x8YB$F5&GjW?o8P{S3Q|#TaSq})1W&)Y3#LlqTbc70ZwN9maHW$7Ff(Z
z;_c(g!iEm=&yH1(R%2`Qt{uFp%PYuqd8uVPe0#mGFuWk39UO?VL?<mtmezIMali2~
zecM%sUnD0YBIGD(Sp1V%0hP}9-~*>WFu3_2n5CokMyD16rky5l%_uDPy}{an5n)xj
z^svWo`h0UAbjKuHOyT<S*tN2{9D$DYaqA>kvcAxZh?}Y>&={mFx?2&F)4lcZk#54P
z&j})8+pvbLKkqkJl;&KJXja#|L}dNM!URkBc7J%JQeB<O$h>k_#G)d3xv@@XQpz18
zH-SC4f3>KGk|e5WWp0`nDN%S|?ok-~C^34=Ny>!$W7FNoIUR8t(5$|QrIG#8ToZQs
z<Mf&>3aQIw^@BCu;5VQfo241tbN14()K}#w!gikoET~T!rh1Fs@GRI|3(3|=CdS7v
z=-XLfGVvxXQleeH>-vYu_rr;y$R(4E*R{!?)ODMd^<~RL1T@(z?%8C^9Ku+b!m!?;
za?;Us9$J#X2-)X@xoOfI=TRYXt+?RbAGk?%nN3dvWR@{=xz{G&2MaCscFX$=*?dLv
zGc8g?zBG-70?x!xXv_8*#c0xQavG8AHtm8wdM&a(!cgC?WhqsFS`k+H@?5)n3cJV%
zbFJu*tm>vWl*@^T4GvAN6e{XTK$j<&mP?uTfGuyyvB@4{TH)=%=W8`B(5jw2*5@Cx
z+y}(J`fxFgB0zlcfg*9=chs)#$-%%36!Zh30PBm-e-$-FGaRpDv=7n7SGH|rt0}1)
zcR*}z`-9k?oUR99?3v7q{SK=#YV8+i>l+Rc-)rXEml*d!4<`>+4y&_TpO+ujm<tME
zn@v919||^SHnZ2@VL8E5_)RQWNIA11&qkn`VPUPz`~e=Gn>6rY2AlS%fxeG4!_*-B
zfrh3geupb{Eo>lt7LwY7E`1#CxxIx=UmxR3rdBy<*`h&7W>CS{-fqe8xgyLGtPe`p
ziD-UERcEW0uk6+Gz}L$_Zg&AALJnKy!*Y-O*Wwg(A~OOODeOV&)fTj%t%u}kZ+9<E
z2f30Ip5+=@<VK8WNOfB^C6##)zc4(PrH0M?Oe={ZQo*YJ?4fm04-dE(u2)X}wm&TT
zxxbt|F=JY+eYdVxXPr98llxp(2T%XT$y0y*+tEHAK$(ALLt~Z9Y<EM3)e5Z4)+B4E
zz*n$_0ov+^8cg@JZ04SBCMthE$~*tCzK4ELc?jD6JYbqKsf29{sB6!AnhBOlHhc-`
ziha3NvFoNTikilv=Jm$y*|%9`w*VJT`CydJ#oR0x=F@X($J<f6f?sR}u%`DnwSD8I
zJ>u`&&@XaL*HWfUcJ@&zz3H~fJUTLeb4KI6WLYrN+c6eEF=11VJW-?go_1EOK8^D!
zIeju0F>G;(36ZBE-A6Znc1h=~Pwc$B#}z~`=j;!$24ijIj8!lO?GY!x8E4ggKWM5g
zf~CnQDLSW8{K^T3$SY}JpRoUf1f>LLL_I@&DkA1TjDki;u%;#fjMX5)3Li5oH3EW6
z%5fO2Zi(Es!Z+=47zM*_GNKG?4)2yUE;v}#f{^FurtL(opflx&U$^;FzYdVqf}-K%
zC<ZbyN-OsN8#=<`$RO0UATkt#BaBQeSKK;Gy+W<VTP!P};$egQdxf*cf`-3875u43
z&>vrz>Ga+-4uS2gPa;Y<{QpAF{?-tpfUXMs^r{efNLw|a4FrMML5Us_2Zz()itlTk
zVLes2TufD<{2rnHZqyI&susB2sZj2TNKI@*(=g_fVuYXKttgeMze|~f>5+6RB=T%?
zdeC?IU1nk4IbRJX8WD1UTkbVM%e@K`Z81rKW9<o1f89<ryTX=*dX)?;$7JCIq3%3l
zsI%L1cWa@<3e~aU$?vWmq~=`uPl$q#BH>ohqmDMlILjK?H_ehBr8|SJHT{i%jH`+t
zf3Lvpy(BDmbL?9ApLwq(7Ut3o&ZP+Z9B;AKyh}S^*hl@E^$5o|;3^%moEe)}ZgQYd
zEkf_w-oeY1%bs)=(yE2$G#IOh7{M2UN)rOS#QqvQUux$BaWG3<g-q5~(H%epk$ZM8
zsDe~mi!_&~1}Y-kKBL?1m9D#c@a|O^^0)m_<JMa+`C%TVD<?1%*~ky>UXF6eIMq#c
z9Tv`Pj63>^h*;<-`;}Bzsu5a_KNO1P*B2?*<uv;??5T&s<)5clS5KCH!vKl&G?2sH
zKXcd;`_{KVtX3jj&@;Ynif4bkRU@}|e!K2JCYYqndD!dERd{XJw;wKBw<xDn#V2$P
z7zW&7M|tfSw$bekG#)w+SzlhF8vrH4m$f!eH&({~pzK0Mt88p?MW-Ph`TfIQRcj}N
zOj3`?*9PGBvBjxIahEy8t30cnA2&AIWx*hD+RJ{Kp4CS@EE{%C2W^F@E7dGuLnkkG
ziK}u2%(T|j1;GaYiTM9%ol@c*tmr?GH2;69|Nk`abId{gB&7QFPtoOnJ>L5FjdiNE
H9Ap0n(7!_A

literal 23828
zcmbTdbx>SS@GlA>Sa3ps4eqws;u755-C=Qpy9P;cx5X0NAuR3=fyD{#?(V@J-`{)n
z>b|P?$E|y+=1g@@caNUWoa*U`P*IY``0)7y92^{mtc-*j92^1|4(=Tl3c?$wZdJVC
zjRB{kpdtDC`g(hNdvS5Gva&KdI{Ntd$jQO6v%7ObO!NABfBIJZdU5;uy7u}iCJe1T
zdewS+6eeC@A75XO7SG;9c4iM=XYOCt--Pa$2gNgA_lsZeCj+M+@29_WaPW7YJ<D@*
za&jryh<4?&CnhR4AGZzU3nnKoJQ#3t{JDL7z8xqpyu3(Fdwtz{y*s*|eSn&AwSFl$
zy18`~<dFXHx6c+@VHI_IdwfFjr9`&VQ>NqY?^t8q@$1przq{A3{nu(Tb$3h9*UM%<
zeU+DW$m{FE4%BikPWi0MW&hzdOpY^OnT?azz@z2#X?gk{=3D6~a!}mHA@KEnz(FNv
zy;h*)f|dI9VkRUmL%rns=jwCQ=)<iBorKokr-S;JgK>W6em%AN*WCp%iDpid`Vu|P
z`x%IeQNZRe<;#&RNJi%C`JQRb-c8Hzj`^F7%=&W4ltC-$o9pFa7yf@oy}4<b=lz#=
zwp_ZJ4rOL_L#xk0u(QE{*7Ju~4r$-=oo5cEsF6@*doAq~TICb(@yeinaZdSbF*?hi
zGf!)M|1xFn4qumV4~~u646K`08`A%3B;*bssCkv04;FTHcaL4XZr)y*Cm(NYY``iq
z)dTiqby|+)^G?{^^2~HbPtPx2Pc(x13e5$JJXWMxD(l?U)<QFHI}NnV(znf%TTRtD
zwemLtI1=0pGRwzI6$BQ4f=_5os<Lwo-QqG@m&cPjuAzB#u;t*?`t=cLiEEB&?QO;E
zDIa)4)nd8CaM}V4KG-`6t{Cf0x1p67hvYP#Ll62(0)9tWS`{BJEOa+2r?zv2tiN71
zSy=uH;-7_GUkwd{BcMA?!H<=ym38y6#@Z)$d)@P^3rF*?#=T>a2pTr1xQeAg%v4=N
za~4zK`F_?}=4ygq-M<>Y^$^JdgMl;e>do4r37^Kv;r#N%qu3w*C(>>={+1Vy_oFMj
zMe?O_ttZXdONW(n{k9o_nl6q`PWj?LyLGa5nmW^Q!|XJ|$~rT0GlE%<PafgmbjM{S
zL^Zq?kF&mne-y<V<j7Z&6F4zQ8ct3`(bow_@!9{^ZAyX9m`59~16-t+aF8JC$Li5T
zL@URpWGrzSUHRmBL$a8C>k>9Uf8fi(GM@dO%V3K4buCk#wzk>eW=r@EBwJc1X_W~F
zH%&3EF9U^Y!3Ejm!t~{?;?aX_^7iB*nG+(pDj!)$8JQf^KG-02;Vnp{;(RWHXG5&h
z88Cr`iz%A6mQ;enL2yMM^pW7w1#vBBLD{SrDbY2(QyAXiLbQy(q5Fcc-^TAvK3F>r
z@_Ntc4H%R#!HG%9--Q&f*882jTy~|`d#;$G-WY3*v1-nPbbDIYN`BimJ#OHy-7yvY
z^HJI{D3csj7sgD#)#T!~vP}d7?dl(p7LbC)rq6w&FY;<6?hZR8Y^c}pCd*cr<H6-z
z=HAPiaPNc1&K5wreAXk!%OtbMkx*jL9bWC$DT~WF#SV_Q%X0cEK}MAuUd?MzM272=
zhU?da+i<Hi<hpYiU~PnTV%UQ@>^hmBJElaW%<GoQxM|h|!)(^xqnL7vs#ZUxpKo4h
zhNoTWUj<HO`{UDOR^2<A;#sEJ#=kX%a>;&yAJ1Yl(&m_k4%UG3o<zevE_5BecW;d%
zy{Wp6rpVDB&C6G6CEe3JHEAv16qr;@9GCmM@1T9qZ$a?7Ntzm2Res+ef{EsR$pgEc
zi?(4=A9ViCJn)D_PFyf3W8El>H~I(nwoeZ~4YtEydMyY8UH$-WmB?c?wo<E{*`>DA
z{ghxdlLp5DC-9S+{Zot^=eiSQHLxXn$3Zp6Nwo^lXtE5ZGBB}k7Wg*R4fGzCB%i{9
ziOz1J+nU+or2L*7uZ@&SINdpMD0cFo;8y61DYNerq*<@7!ir#RSFN9JWUDwc?GJP_
zc9QzYUp%Yaycb0*2$AWQ@Miz?2fu^|J9Ep=TtC-vFtrM)k``m*(6wju!B-+Ppz{r_
z5IE#Ro>V{w@NBy?MM=mvO>mf5VEXq;OofTA&!kV%j4Q0d;{YuUJT>H*TGPkNh1w(1
zqX3*<ZJmmxtKYxb(e_`01llaRp;I?YW<MSFaJMMpH!LW<+yuBK3p|`)k+_U=&LJl>
z`)>&{Pe@Hc_1II<ZVT`Q5#TBI2mxoIDO;a^nk#6{$9lP&<kg3@I9M7uH(nDV-}?+U
zD3!GyxBi_;VNJ0%hB5y^Ee}`L*$7!EA&>mgo&Y2wVe?UAK??_AW46SDq12)rL!AX1
zgeXyF={4ZCO~aJZoRZmlICREv+%}GfRY{10R})z%00lW4>tQu+BR%0@U^3uWZYEZ6
z-g!G&5*i1D?@2^F4d+If03+eoZ}Vg(q9e`w6U-qNi8rMaFoflMbeimz(o<SI&67S-
z@GopjqdW1_F9Yrhug!zutX2}wYcw)<;?#sBmS%rC6{w+4X?<Ea?}P8Dvi*dE+)TN@
zoAw^?@DHDlJ_#?>2hMT!@WyT#lU@C-eh{U=le0##nVaG^cl>~IK-@Afxrx7@LW^w{
zDn^TF6Z}3QNbC~X&AQ9Zn!&<(B3GV(!=1@+>u<ZV;P!d5lwfrC;A7h<uRq?`Ngm^C
z%%1L6#^*KDqCyHJzr3HlfpXt`0X2%ZvyY>5ujYS|#+~vNsGL}-Lc_zo>dL3|v_ug;
z@;ucYooKD^0Te*bF1lyHt55h%oc=REyJG!uJ;^iyaMAdZsBluKDe%@RfFS3$gp>kL
zi2T1vhEBk9{Oqee)Hx=W@$MVZf~2gKpqp4n0QCcHA|4JM`1%b5++ej0o{J+twZc^Z
zUBB_svWioOVcB-R660uP4Vpd~8fc@z<~*^g01**8d;gQag^V!gi+^Dm+7Q-S@CYcV
zOK&8rSP4uQwwUYVHQNYvn`vOZ^W?aFj)`tRuA(gh0Wv?-rWe4=QqV^_bj4V+4>gN7
zLO0=gj6d0C8iDml%AgPVSI>2|k~_5r_o}ijOwSeU;!H`Ca)zPW?YJo+Qc~C=PeG8V
z?J+193rlJH88B*ON}*oDX0JQgb=mMBlnn~ip`q*rszpqaQ-%Z6X&P4Pnh&&YDN|sV
zt3GSP9%X<fmeK|mT&wAol(=MCEp8g>FbJOd`zoBL!F2rtjiQ{3gEDPhLAz2Og9FFu
z>jA!;(wsBTBJIrzAJI3JXhL1xCMv2#1WoSUpPTct1@pF3mWj%lJ9v9Wj%<58P<w~Y
z$6mjguXOFh9vu46<#ct$2r|gjje(tD!Rc^y>@YetJM63R6CKA-x7beTp^9(mcWqev
zUGI)2_v=ax5m~qp|1G$lUHT&bO4o5GMAwpX1XSC1S_IY9_^yj@<OzaVE_%0g`;ew!
zsOxTCZTTv9*qf%t)Qi;GZ;%OCz5Aq&H*j$jZPb2enK7xrP$-4AKnaEMH^i%XpwdwE
z;ef4x6m|{Ax|F`LG_6#`k4jL|#}3Cq7vi;c&k=J4_9Twze`Lj;XRXz~#>IC>uPbN%
zTQclL-blr+vrNGUlu~>QUaIeKw^5qaJ|l~N7-V>9s7YQR)q+umwc_LmK#unY$8_;|
zaG!bpNdbNiU^gqQt8}~=0ye=$tD-~_asB~fa;MoGU?m4O&0slwn}138Gg9#8xY79|
zOf)X?Q?VmN{PLdKKYbbavFRlF4_}1UX2VUTjsw`80b5f7Yp@s7H(N0oo?XVR&PZwv
zA54Z*LDly)1FEy3@qaB2;^YN@1M_?qKD&s+a*u3Oay5_rn4{^x)aV5uslu5#xAevX
zz7TB*f)VZLYF;F6ggH;~5s&w=96``}hUoH6!aS}|)5Nj4j051>L<hk#ws#jb>g%u-
zuX0<gb>5+}MOOugdD~2_&M)+QOLXuh-lkdhbboeSiE}t%S?Y}%vpX%@#ZSDaItATj
z4@~KJr)z}ajuMN#554oITtXT3oXLpd6<#dy2cxUz28Y{yxJ%aXX=uaujhzJXvsuCw
z^k*01gsSBOk#=YWK2_5(o=BV_^u}lIs$WUsNmH(LR5gV6^tBB#nP?SnBdH$+`dg#?
z`R04qE<kiYSHsH&!>`T1ool<fO{AKC6D{0ctpXq-WXh{XE1S*2Yb#fWtLh5)>1}+L
zdvA;K74c;T+HY9CWSyc3i>MKSh(Fb4cviBRbiYkU?uhJ8T*;JLt%S518iHl!^vP#Y
zgwR2bMAGtSRsWgh0+Pekl2#!5QH@V<x?)t8v;7y|Uwrc7x>!hdrC+=N(;vXB4K+aR
z<OQw!FM+n~p5?{gL88n{NA0J@z6NvJukJySFEb0n@4yQ{-72Zl<1;)jpgz(LaN1Y*
zInf<klUX1%YcLLoK}|8p`wKl-X3$LXTZh`8TpOn1qLe=CMLUO}@J6sb+V7w7HJ9;W
z>)%CmgEYNkV)(R~9MQprn&LQ`$?pb3)LgzR%FjH>>BzHhcAtHg9Fuy255#4M#uU3K
zK}4+JvN(B06LD^)3M<eErjsXg71C(X221%=v|Q)24rrGBRmtvfADe_|<nAx)md4xB
z`O*ZRT_YaT80qbq=w%dYzv@cxf541BcfY>9t=c45C)p@}P;@)yBcgw-(^hLMznOI5
zDmcdyzAPvtws$y*JBMqIrR^!Z=Q?)$QB0ZqmYwN@Jvs}yH5^46mUtN+8JdP2d&ea-
z(WpovdGXiQ-;gTF1Na89I`Eaw!eRsLPEsZW;t6VM{oRlN<kQ}OQ7%vw2^`SW7+(*H
zDG=0MW|6Rq>-|T`ifETO4wjw07+pxm5)`63{29)%`4Y5*U<<IoX7JQw73ZLsqi;23
z&k!<BhSrthq5wsm<t#ih3d0n_2}P-L-dVKZ<X1t9M~dz*x~<lokCqonU%{T%mDWo>
zpi@1q-*)n2s;uVPo>@BC1GLYdFg(P{MN{;*v<%xk?PKy@7+~K{79ENo5#mmKPhicm
z`O9*eRUfrvg2}w`8<l)y?3$S=5PE1Wy0AwzvQnW_J$@Mi(E~c&?`H%;KW#e6U$&<u
zK=j>OwrMLRk?Fzr#6Q}jmtnYq{16~1vlr9LH`@+h_x_-+0k=m`)TC%@H|$!4n^B$`
zS2J{F$~5Ijr5pBj>$PxO)WI7?#Y10wp=dEUZO7KwG68>qfJ1^|`V@9pPJ4gB@Q?ng
z@jb-6j~9)6dMF@X*fs_r6?iLtO_a=+r6WWP5B9?mP|(;rBsR7{J;irBn>#BKlLurp
z`z?Je$gK!SAy8C%#{iDYL_}i2H{*X9tIwsk?+>op7_Rjd8ZF!)`r7R}?pu%qUoaCE
z^~FzDFhyj6*;I%b%#GFR@r6F(>!+2Ry7$8}1Xud|E))@%;0eUx{dS9uNtngO89kM-
zL30YvCwgRv&>(N(HepgGyr!aoKzulR(aN!7axST;oY;yW1Enf8&Pp1hwtUb4UiCc#
zw$D3ocY5V?PshkTu{mgfSwtKXZ|?Tply%?4HAH#dhQW1MFmgn**5^5Es5n`ac1?y=
z+|RAkIV}=<4Yv)wB~8{(hM`yS#|N?>D9NUS@WW+V?ViWUU(f^gzz1A{4tDfIReySu
z)V}LupL{EhhZl{to2CDwA)+g7@30Q;@zWb7->ze(MlC<0Eo8nv=gO)!Ku%y$u|$(9
zM5jW~8(!xpl-f4RJKwi}Hi-a){ZTfx1H3qhcj%`(S(hj?cBjM0u#*Ft3d-q*0)Rv`
z7m;0$pkKhhbQe3Z#cDr_oX$(BXDx*1=fike{3hsTZ9lZARPHS3l-JbDH#UD*h*SI$
zGsYeaku0M7GS=1|7pw5IIQ9r`Aue~@f5GFj1E`U$dLC}iK#<Q*mw`kz=7hM$JCmar
zwUo%o+DqI%64W@w56oWM&!C?a5KB56qFY|^XHfMrrlm8qW-i{d*J{ze69=C;y#o(@
zN%cN!<5C(~CBi*!w<n(ZKzhc;Lt~$|8>M<Jidu-?*tKmUEr}djJ{Kb{m=h`9Jz92;
zC@pck9r{q0T+`cg<dOJ0<u1IQ8M7O+T-|h;CRtWbPoIn$fqF1qVQfJ#SpH9<Me^GM
z>1xCgpx>kGzBWP<ATN1NU;iPN2}&@Ibl)ORI!i3O4)*U})c>s>B9Rm6te7>Lv-YQn
zn$en;4DUC?H;Z$!Pz@&5K+Hkzc>7~a$dl~(0*Oq-cG+z&R)0mB!%Lo#9u~L3i`WS=
z_3;)+ZBx)$*d>Y=?v^tr9}NB5<MQOd@q#t%##j*@V2p&5|MkvFw9EPzxe1<L5E)30
zJMQq;n%byRmT3M`YQFJM^dTY9_iYu)9YI=(&e%x{u7?5j+1q=6VT<@YTQ^2vU36hI
z36e)>0CRwomzQdY)WQMDWVMA7*Ta}ORMCkyY<a92|Ai{~nW~<Q7PoaRD-UPdujBY@
zfTGZTwKwD`=(wdg!dBQVlH<kV)1SNG_jzNVRY~0=4i2<DpU^n<b^{})(ZMBYH+aYj
zI|}NLx$0RL$mJq#3?FBI#Tfn(b95gRCck>BaDz{7)a+to&DWeMDt*Rg<V1mY!J>f^
zjbw4rM^uAVel^TuN55;#1&F7M{2EDZJ03WCsX2@lJO!u9(mTSYZ~;XVcT|d+j}#i#
z)(rWQ1kIK2P?sJ{{dWLgw|!GgGh|W0!OaQouLkbYIkzCR1hA1u?YmS;wBEK623LBJ
z>IpI~BPTt0%JEIL;$3dCKlm)#0sDO(+tYj2zM#)G2H$ja^sL1kHqbFQkz%_@Gj!(v
zkdKLEbrBP|B<xaSZ$G>^!WlU?`+f6UHYAvK@~x#AEQXCNj~!wzu=Jv?{*8X-%gb5B
zf`)yM<>=8)%-GM9L{w|8pl3`QE&p){D2R$1I#8;CENQM#EQ5nSH{1Jr39-Ne{?m{G
zSnU(8N)R|OV%Ng*yps0sKu|1Zt7Hb5mP``uon@bui^u9{K|>^4^xEf=jS`WTzFCqG
zIF?6qpRdBp^B)S36;{|$)<Fp-{)S!PbpgL)s3}}p#QUQoI2Hsa0PFvhkaU6JShf%i
zXmKm55t`7$Qi5SR-BaM<aTjG4?_H**=#M}{d<!ZXh$?;;ILqV!T0{3}WwW!vqNMfP
z4WLBb)jn0RegHZKy^9WltsZA9TL0%EDqgIhUDymV#;~2<PNtWTa6Qk<Lsi{MO{Hg~
zw87!h^MQ(nNKR`^(3f!PU6^r-v60V+h3Wh$yosmxioo3zMeLTppz<PE8}ZOy;MAou
z^KgFo26W^WILYm7<8ds0oXgi?nNE94|8HKFvp(tqDVjNVy<8hhpD><cITsPX*=}*u
zyWxAF)u=q7S*B@0DJvI`)3WS5kh<s$1lUMncz0aCC#~OgI2G=<uQxU&L?yMBNxU<g
zaUE#{e)02pp4Cfkp30?bES57VDEK?&R<zvl^qkJPlUt8+)3FxgR&1P=GPc)Y)M4DK
zYlCAp6^xGd`4>76_D5j-T$x_K{k7+XBTBgBAx%@=QdSNhe3*QoE^PR&d&M`S@ffm3
zgUu|W@HxKwd?{IWAh7C-zftA{>TLrysf_Dwo=yEvd9ShJ?q#o(Bq|R%HnxL}clqP?
z9%?ZAJ0~#$GB+0|kRYa}6{gDmMyRp~gi!rzR>%nbPaOGiMPU2=n5-tN(yG@=2%T#Q
zk#vWzlQT^dq8V`pNw;`-2Fnfyt2yr&vNR>8JSxE;2UEnIn)3+nH+IrR$q~mrOE`{Y
zKofZ+XZ6q6##u8g9+M}X-iwC*uE>ER7+k4B3JA;y!W6RiC3mCb5ZUAB&T2m;7Qydi
zQ)hSdn*Z+iZitNy`_Ik4$_RZS8PyHNh)a%EgxJ&S_46G6Nrbg7_Kl(BwE&q|0ekVi
zf-dw`2>ILwYo>DfqnzhDzzhv#a1#p4;f=#g-Xtou`o#oc8214&G0PI(#PX8EBuO<#
z4$OA?y&`!Y#XKXTtcnH2=acw+88HHU^wN%3m6B~6j}umwQqt=c8i8ijMIA*LC|qy+
zk=G#5ualC}t-KIY*~{EdG^3uJ8_nL^wbC1T{#|2wU0;8OAEbUNP>b&2tt2c=*NS!_
zAAW4!8J^Kg9$LY01%kywsSa&guRf#ZOkceR9`2n|CRYZFC<N)7&~PW%(0!sh3HZaI
z;?l~cC%xLI@hms|Pr+1Jn%;aWO~Vx;v49Y3-4r7h)@8+y5}{axE`;dToZCR{a^?0;
zzI(vg!zo}*iXq8N4nj<?!IK70N@<(t>b6M-revE*4ns>%cNtC|;;_yD;nW65_M@D{
zDZ>Fu2_e@s_3i*xb=qhagC?g8G0BH3ME-Jm4s^`44Rs=hqIJn^DmU-9-vzLm0z(o)
z*kKjSRvn2jW>g%)1kw3Udo2y<t}vc_j$dAC4$=y=q0N)>iGKI+c!orL`$4sZ<p@s8
zq6CF2?omNA`GQX38U>uJs-}19gc)CoDa&4{fQeVOWZp(M<105@VlGe^-Ln%Do;=AR
z^-IC(6V;pLg`qV15GA3cu>zk?;J!kgP%TDWbF64<-Z_R&o?G*i+J+diU)Lso=Mvs>
za*2Y!-n(M5)H4husCV&-DJgpVsdO4%Z*pm*F2A?GMhb|WHo5%^+8wNKRWca_js-8u
zzBzAKu;gtLJ%pn_<{tm$CEa8m@)V4w)w?2`GVt6U0(t2N7XW%h0EOB|8D~|TNn;w0
zXFpQp>nW~>Z4|G&)XI5=bj4MzmtE^Jmie@sCg624mufR*LSCAF)Y1*CGiG9A{A-@@
zTvDBG!KEXv)18y4;X`tGX<jECvdPo93pg(Fzn`E#93T`hBeYueVhcbv^j;48WhJZ|
zRN?+BNPJD2atC*SfpjJ4dmt%Rmjep%TA1_p#4`GJMiUki=Nhbf)wI7rU}k9$B}nzx
zL>m*&FLB9NA2O=H-xtqDYFBz*a}U+N`;5=>I#XlCJb$*u1h=2*TS;$@dd`Z5s}5Xi
z-)<9`*QX&X=AknV{81-4RkuifMmBqW$a7{N4*2ak$c-J!2KPQrI{Za^v3b}it>qoj
z`1SALuJ33C)fHFOQ$KT||Df6p<I_tG(`v@I!?VlT%C$9CcT86pQ0-mBvmcjJ+DJR4
z=<iZQ`4~e)z$&rB<SKyVd=QET(oP9_a6k&=cV63ZN~QOzvGEx@v){MlloJjutF3vz
zQdVDZ;O!Xq-@-RfU*G_wmV|G|v~{JmroF}0?oXSFn8MNsCj~WyX>yxM4UmIs91HR{
zxww>w21}B{h|MM~G1}7TS2E9Ka<ca2<Ih@OtSzr)x0lyvp_OHhSnbxI&jGJ%&p%%q
zmCtrxfM-iYKQ}GNdi7_Q6E`B>8w8~Si0h&eCO{z}sm^eEmQE8OOc5A5=G#^QFWg`c
z{qa2>F@@^l74za{wnbOU)#2xbr!8BNDklbiy?rj$rjg#4hf<%(<4zO7g?i0pLpqp9
zEeS4`>~Wxs161wsUu!lUyKaQ28!(@tKAzfvD$okz{ycx8ze`-224>zlC>UZ6NfwdW
z?qH}_=uH;6ooG}r$fZV%#2WAYQQ`?<`|BJjMJJja;sJWbZW-b-?S5XMG7x0#L&8R3
z#a*U}BGmYcm;F2tkfkCG>-**Rc{Ak${zWhlCTkW(KtrUOm(+5Zh_Yrngqv3?H;t-n
z9nmN*&k?G*DX-85#`1@ywkFT^9V|L~O4)j4I!0TBrQ5vg^EY2j5V3%Ryo;Dj@;{il
zIETEbJg}Vo_&XZ12ONeTkKPg#<kX`dGe!%g!*{-`*}@HlCvEcOW@WxkTP5J|w(%}9
z$vXh6Hvb(|@)DDGM83RDwMkzcBW+3T26fG5v#lIDhi6W6huhEbOMPDLFxz!%a{V;g
z>F|S<Ca~q-Yh$!qN2wJv6(?zi<xClgxxOxH#TUFFJ3G3Vp2CEzlfEI5mfbb|hIw2p
zG;4tGzFz9uS)OZq>TeMU3MHS|sbU8B*V>j%6muCk>B~oD)MPhB1><by{DtGcAT~K9
z+}_|vAZN7yC_7~H*W_d<s}H{v`OW85jAO!uExPHrAc4{Xsm!3#J|a>CjC@M~s-%!F
zalsPQu9y;ey(9DeJ!`5W55gI@r~WMvnU&|#PWMN(Ks!k>+403k7}8ieAG(~DCbKnG
zuJm7E$ep3mLRl)GA?7{h=a`0yO26U^XFGNN0j1v<>vJI7620$9en^uw6$sx+3aHxt
zDv61ErW=jce$>h8ZWHe&5>l_}qEra-?SlT@Dc`6YZ3VhNe4Rt)e=cbSlkAY-qPU~X
zf+zZ=KBgf>5a6b{{6JH)Ln<P3F>)CWTb@}`mrb5?2fS!yfCC!t%<)f@_Nmtt*L2UI
zIWgo|hB!L8l8t=HWu6JK|4?<^6bL^r6)0S49=6_5(Zg!!z)kVH9>W3!-`IxYNaFjR
z*@lEOwL9oOq+(ah?0&53BGDpNv4aRD`Eykce>j&tA#}f2ek5$q7If_FS?M7@UrLQR
zQt+=mBz+p@&|it-(x>#{T!qSxKjQ6p^srh3ke{l+t2PPL4;i{*vr07TKJt0uTg<*F
z*_+6*@&LU7g!}ENZc!L-*Lf<JYZ+%Df!=_SRO3h%)o2VrIntagUw?Ma;_I&MEG)V>
z$0cZzx%XomYLDReMs(41WM=XB&l8~crkHi;qP8M1>`=ZpsuZ4C9DDbTDnNQ;rz89)
zlraukHVtEon`Az+8(ukHX<HuI_-cyD)&kx&sO}h5s3sPA=y|KR;{O0XMh<YT;Y<k4
zzWQSFxsTHYMtMHAeAnS#U{!v7@lj0!8Vh#8WJ}Vey5afjfd$lxKu?=*Zj%mgxbtE8
z!L{!KfuMqjJr}3GVzfF;##5tuYbPpYc|#%b{99_pdkEQ?M^2vw4bp>b06-Pjq1E}N
z9_--SFrvG5CMnK5T~(Q>ytMzb?Fneh@U3)XQ?V5Mx#&IgM=Y3fbmf{QL}H1z0x8Ex
z8)$bWI&T})0Yh+X{+=WBH@mNe7e#lll9Yl3d%4<Byp~%>{66fD+mL_*yTh`S;Gw76
zj<8#FCBQq!xZ}P94Vs*MgBnUN+ov}|y$kNDXf`}w%+o1KsY5P5@OL@DMjfgdcOokP
z{~`|wOr%UPo|7Ht5D@x0DubbItIKpkju3ByA8pX`83|ETx|peXQVu}?0h^N09ehW8
z9i6b*VZW2wy3+Cr2j7Oa8f$oml`7id;4wiRn0J1!b&MVHD1UY*E>4(&=Gkh(%>Dby
zZReA<M!mh9*+<pi9;vHqVb&Z?HE)=tBl*8zl22-Vn6~Oq6o9aVs#5((zFv_<YIbno
z$tj`u=|lYP*Ke7NrD+YHeu2@pH_H^fr|sre18eME21Er!1t5tK<##pW=ajyVuB&2b
z1lC+X!@JuGZ6UREF>gn-pOft9l!~tSIveb3Xh`uVK(`aM=z0Sm^R4S`4_r{$jGqRe
zT4`_+NY^^E8=39cYQ>r`OG)m^PeDG^EH<eT-4%$DdaW`DOS1I0p;8L!wRxZ&x7S5I
zsZGP{Glys^#XGlXkwY$-?<7>d`?^wD36YMJ^9|m+?(xO5P)*t-2CIV_^eprdVy$}M
z;c6YX;Wr|wBq=?lhm$gJ#F1s~!>;EM=&a_L+~peqjNfnN;Dq<t1j`V%*u%5GREo|f
zHlxQ40Op5C(;ISeNbNOT{fDU6-#`5#o<M>5@DGf&70}lUP8)HU*D-MnU<M|JFY8FX
z(}X!Cj+3{fM%W=9#<-u1z$D{Zvp>8aE@b*KE?w1ooo~V@Hdqwuyt^jfN2K+Su=q#;
z+-X$<&SiakIXr@AdoK9_Cf8Q0FaE^x_h8WNl1;RjOrXfe^q(mssY*1FpBF+zT5XB}
zdy+ka6$Ls$a`feImfkhF+UP2<$%sUM-f!&3W4+J8{g4m^e%6Ah9>~k8KHKUmd1<})
zFNRZ5y?c><<-_p6W@9HSEQT<ZEi&4N%j?nK13riZvJQ!?bXCba0&a<{#s(O^!rKBW
zoptSG$j>gN^wl4RKqJ!Wh@LvmBH6d>n8b)er6lIeO>sD$XgmTpHTJTqGfZ{N5vjRg
z87kmMp7QcAUAK|wHxlIyLGa%2j&K!fsgzQ5Ee4w(+)stL7BsVO$vPjKyTSftjEowL
z`amxJs{e;PUco$~>4WE~6ew4}z$j`6df&-MzNJ#gU|0i!*^u8Tt~YiMvGoFh$0;FV
zFgDUJ>k;hR7jV_qkcSEXBVLa$yyH14{s-igZg&l&|NnQ6b+xmpn)@-jiPEV~b^}16
z^!SFfmCQuyOcSR*i0|fv(|ly2DGvTE+Ub9J?#ixPF%=Ff!vF0A3kzstFG^rbW@`@S
z^L#mI;p38)Mesc*&z(qa(!{~)7r*%x6Ydr6qZ)owM9{A#_9g5CBE?@BCTntnqwY;r
z9N*IRG~&M2e;$^#YnH?g+z};DchjSmmM>4JBS)`SR@~8|-3zt8gG{u55O$e$ZRNdd
z3?m>Ah=9%BXkLtcJW3c1Z$_ts#H+xC6vBp`3Qr7z{zez!gw3b06C#A>zr}p8A))?O
z@CNB;s7@T+O)1Jxf7d;?+v@baq3_ukVUjm4IDzq+S+F4?=%wAn*c6%<W4c#zGVWXl
z{PS|qDf$|T(u@OG#$5?|wPSk$1%b1t6QLAyA3zdi_adJL2=ZxCQkC$_rLwQo5zFWu
ziUKVlYhJ{)$M)TZ0lJfs_e+c7Uni5kI@3@EE#RPE8-7~+!M+`Th<A_kr%u;8fNZte
z>cH@P>Ih1RF|zd6C8?-2OSfh9a#;YtF4E=u4gCJqV+6`@);4o3%3AMNs+qRO)D!sK
zv~m6R{B`}0ovv@f!A#}DowOjJt(Af>`sG9p;l-|0q@ZP=R8t6}R)``iM+b>EySApn
zje@ISby$;qcc#qrh%<}O4LWgvdOoB^aZgDYO}ZW)dpt-lR=Il;Z<!VW_NnSM4j%}2
z!x8+aU6^Hjv;8lmHEJrm^t<zlt)XVbM@Ai(y!WZ1ArgP;V}#f;0pnW!hI~KhHNdb)
zMG+`|jT~gv(yRwXH12)~&zSkG*kQm-f;L7UW_9V-2a0?zU;Rb*DPzox@KiM(R~s4@
zcR3a;h#_8@JLKm3Z9l|<!;Xx{^lNyCRO+@^M>G^OeNxC8sm$I~w=^iY)QxRY&ip%B
zAoI#jj3%UP+&(LL0sYT9K5M<;$n{jdX`SH9fNK<yd@qi+0^%W>5xtJl_1|sMl${7k
z-EwT0gt?}YV}+_~-|t^b5?q(yEitOOSSGq3c8>ynke_Ks`&yR$D}x)2Hlf<k8xPTz
zqe{#C_0`h*vvzeXF-^B`ymg#(E;emwNK|}Tl{j1J?{`0qE<2k+E*ZL+Nq+702=?O{
z=<nw`ebMLD{Pfb(F30Pvd_<gYtBFCP&$)zwcqqT7xPeK{sx8WPSA@f^=Bm*x7rJ2D
zcwd(%ol}Tsg$$#e-p3CSj}q*N=}2VWH`AEhpHwb7@mFU>D@7<Y|KsE`kT}`t8tpC1
zDd6xMm^NVF_tYEuv*)jLc3XvAXmG|dJ8WG)kI95y#kkq661NgIu>G{WOl;ZO4YF0z
zgnKu`HaP!YAnzhLuG@$U+w6N4(NYfx`sv7w{Lys7@^8$0H73A4=#G}`(n1zk$I^@O
z7JU;2o?PxHj*;OglTG6P$l`y9`@3dD<yQw@6dp`OET-bkbX7S@fccDk%<FJk?vis7
z*h0QmA;wOo@;V%x*8f<Ct~+3I-JiyHPehg2yfx4=G6rSSX1j8l%d`N(aAB6*Vg?H=
zu-lSVSCp5<hX6N?x4FvNXCeer>2D)nRDG7jp;EE`W)V)vXJSDe>S1|%{3F4<;P)u`
zD0NTA@5u&TDgSR|I57I-C8Iu+Li=s2@jku*)nRs9>+%fnJ@-}y*dj^?DL5zeTZ3+d
z2DKI#x12kGC7dQV{Rosj6qw8Lup0nK6-<8Kr^e9H(wpyK&F>h)54O~aGosr_RuQ&>
zMrSyJ0VqK{_qr;QCuyT)3ewwT(z_gs+(<w00=3jJsNCWbnauz$6u8V>Pe~4`h~j=m
zQ3=FKgSAt;Rds73^i6et<?=<W);ed{hn$<t{z=kQ=wI=mox;Y`WBRpLQl@$0hm<Ri
z#c4aAa`UR+vEh}ls$j|bVCh@(bcJ5_H72a5##ubJj#JK#sJM~=?mX=-1}^?dc-<m|
z^e)7SWItQmX3NXVuXT-wv-Qg}CSe(E2Fa`>DOdG1NSKR`bp=y$^6{$r06+b;{MIXI
zPW?lC0c$HX#W_}WWU8tTeYU4%(Nd4*iwhO6c+)|%v~o#>sF22IHSPB<!s^y{VRH5U
zBws(3W7vB7(}eutLN_!YZKwQ7syRtPk0)I08l59xJSnh{xR}y|ap7n?R5-vFi8Iik
zk&|yQ6E#k6cqoxA@7@zhMq_or+<V{o<-z5oMnhK&Avvth92ckLpy>^>|J5+9I4{2#
z-`9{F-9Pa#K?;B{b1pgjbo1z(jfs|#Tf(Smjtd!Q?2KImrd#e9N!@@CFAMCg`-<9%
zYJ5*{=<;e0E3Ex|_?zy}a%K1ONu5P$HaR(bwOAaiuu+{awtvQ=CqG8lk|CAy@1Ct~
z4_o_Lew$A6H=D&Dcot^T9xyQ-XMc$qL;r1?u~l21PVBNz+rlFY_BeJ*O(@i7_b6ok
zSY8~as|=rJK2sMru8|A6`fhsyPuU8Ra&#icW9O%3;(y0pcCw4)giF83H5z-6Ur41^
z0L5go%szb(UfeZ#(w%t9J^lgF;H+=iK{rf_AFYxNdp~G4LLLj=*ik`Kj|<F7DD<TW
z0?%TP8?>lgaDR>d+M=TIKP@UD94*ao!`OW0So$8_Nff((pZyg3x|eD*$EBL6qCTNx
zKf@hpp#kZuE73#CQ&!-UGIoi+wsf{U^jP94m};kFSXf%f2F$3mb6InuDU1(V0Xl14
zk(tnYM6g9@o?in`YrrPI^e=v&=fLNCh5s;>edjz{y__P{t`RyYuX}cTQLmP&Bys*o
z%4&u)Fw6N$zqdW-Jpvf3+Fjc-pl=2H`RB9h`Qa$(ySY*G@@NwZGuYMBji#}lro8zK
za;5-JwvbfYMONnVXSf{qc9wNnH6+nt6B33Ha`5liWYKt#j`T3O44{Gvj9q{}n*c&l
zLpqXK$TSo{v0;ow;C)%L4GeTtdWUAq+i<{;GGZcjR-jPX^+6si8eJI@*{;x?6(WL4
z%bdc#i%q<r`;{b-yAZoA-TnuQC?m>BUhHRyyBA-_I4dHVD**emf6mhl-Dda2&7Lb2
zHI}B7%uw#|A5_kma5S;_pEwV2RUdHrd0mW4L1J#M$f{3>9(^l6oLS9Wq!^TmNf*TZ
zR!~GzaPrcaqmzF3U#Z<L5{ihY&&{IWkP2<r6ct<(zgbLJRQ02q>t{kUrGAVk(9#vP
zR9Xaw;yIw<(1}*rQ)FdroUxGTK!hn|m$vfZ529Jg`BEYn1#A?gsIu~oSXtpRk;OUW
zzzH@Z255AT^~0Bk>nWO2+5^v}6ydCpFzb;5g@?8JXhwlm1u1e#?6(aiaS41vSaa2g
z;0ft|w-Z5lufA7N%I7|8#i<4!O#ep76id=SNIF8FVtD&Kn0MCO1WXkXMBVSDZ+A*l
z%m0ZDQUVftxhFF(QLTy&42>Gq1B<EHhZ4W~axBY_rqF2(?40Zv$yI!N69tq&x1vb<
z^P1Y}Eq|?rL_wMe#6A7PlRC&Fd*eDNPESv~>xQKe#EZOIf_cDCgyG;jy?=rw>bW<)
z+Ov8vi}u~wMf^VpmRxt=?DoyEfFf?KWAo(O&Yh)}TrhEw*u-;G9D?mD`FOISazAee
zhU#aPw-A`Bx1m)JDow7-meCHvHmtE$hyT)Au{y(P%>7z0*Y*kt8QW_EogRNGJ4Qj_
zA+&>b1?)INMlO03S}Qn{4@yRM4>$I9k<pILi-%vF!G5iZ70>BD{lp<6a>1gGU;|$+
zJ9wwY8vRCQIdEou><*?8q&kK#HCrV1H`Iq9v*hC}^K52Cd8~)7>_jK5i7~*K`!Hi=
zp`<yzGJ3qWnCISKSA4BDTk}P7d*aT5(ZRu;WB1<g2gZjrNIs$r1O3-D8&(<>MstOL
z0)1~V!bf+Lll=rHqJ5X)K=wuc`|6E-<v;(<l#kP93s2co8pe^fA9fHsU^>dDbauZC
zZ!_C#gJLMdd(N3!TIsr0;F<e_k&9&#dNy2myK|ekn~>??gFn6<9dW^nIbq=w2irM$
zHiq(?Su`Ou&>_YyJtT2+2(&6rpUeeX-kz-eO1(3Ziud+f*zAk2S_lU<$8Dv-GbBA$
zaAAWlzbj;87Q*L-UW4)~>ZclgKzY%4sn?zJ`)PM}Zl$MDX<;<~MxztxiJSqi7rLh8
z!;4H30FiR4PqpYsx$h0qW~`Did<V$4I0l|&w+3aZK-#E5pcMhcHr0_V3P4xg0&k=4
zVMTM02KgftJ%}-BxJ!95@tuB}^RRVlIOTBJ=JDimZvCpD0OHj5Kd3bOr^uvok%NKD
z&obNn+HwGFaA|%|H_=Q^GamsSeabtLj>9%FmfeErpLsM7o8TPeM|_4dUJv0GF&L$~
zu|r_$I|f)-W~}|4xkE<yp;v{BYK+pymvto})!l}qpY$(8shlp*4`6n2ONAjwC8rxK
z)wOzx*=lTx-?!1Kq0P;HGP#<M%pek45DQhA>29GRX%KsEA(4~DXVjoy)GAi_F(q<1
zCHS@F>-vp{FH1tmj<va<Tr%Vk5QB_yGn|qsTU-#}hs2q`4L=hw!3rnVuZ2RwL<<AC
z^N(3UE#lvTffFrGjC1sV0;e_5P!M$<abDFAwy`^jlTXZDibs;S?z=d4-E8!@(8tpV
z$n$awoqHoHG6fkG&G{|wK;G_G&NK&68mHjWIt2ZcJjXaA?~RuluGA03iFbBW7W#T3
zxoJB{X5}KYc!tnj%gVpH9F2<ZE+)DJvuDP}GsA0ZYu>sl6{%rO8jKYhr(4zZXpi?7
z=IM`4eG!G?@C?(1^1b1<nrf4U`hQwvp*P3gu+2R5BIcD#JmO@{@Bm#0Z$kmFvhCl>
z)}V2EKCnaDsKYc1N4*-rX+THdaoKRpRcm=_m~4GJPmh;+J0wkXzJpB}#r=y<Y)5je
zUz9Jvqk7_LLKK21ohO8~nY<&4EIu4b&>48F!HenR1u50hl`&-G;MLPrpJ=Jz4iG!g
z^79eE<D}dG=Tfk?9WVa_s$)D^#F?ru=Y3poqL7lPafd)!=JjlaE%g#A3otHMG<m^M
z{HzRhTesHjet{~_V)9ZUKhnB&&7<o0h5>u~6v%dfm{w&>&eY6O<m6>zrfJCyFh&0M
zH(}OZz}8ud0k0_@u2GR{sh93Mwb}c71R@kv*6EVOSFC!njFfZegvyhVrBM8(6pOQP
zd&)V8BCrjpWNZbcq`%VYfrUQlrD^I{+#LiApvx|ESdAxg1L11Yg00I;#JwQ#WJl0c
zvhTis14<$eMF&n+5<R(Kjr7gojN)|~zbV=j{&=#=2RgHP{oew|$uj~0Pp6b+=#y&Q
z-`}cQYp*!e`SSmin%#a6tLz{KN0=DRHf^9Yf3inm{dl<myPRvPaa7lfTf}mQYz~~v
zWNLXDC)DmCyjjus&q{9RbK6LkQrk64o5I$=8-KiH#2=w)Vc&y`ZpDQJG(rPWSkV+q
zy#UJ8O_L*!kC47<AU5Odt0F#1e~=0g-hCydLpkQOvje@w!T~8~)09#eR_DFoUiN4O
zK|xA!!VS9goe3u}qTQ}dqq#XOt^27<eweH}f!rU24gPx*-&XZl&6<C-+}|q^Q0b%i
zfX2(XL2c5b?GwPd0Tz3yfR!!(MLvG}MD2lSp>m=lKr{){QgT7;_YC^Y;->86_pw(0
zxHhVQ*w>_1_sB=@)Gg{NU8*^OUqV&C&wZDcVmig4bZQudW(PTzp4ewH8l?I2+P%Sz
z2+xFd1>*4ySHC2HB>x@<fhKOEq_+M?2ZEkmnMUn@W48huf!QJHF)Q4V<EYx~E7~f+
zQLu`OJE6Vj`)zJbijM#?H@uP)@VUAxHryaXdU4AgCztjK-^>HXNr$4O`><88aTHk8
zHk9Jcf}Dn}$$cXinOP>w=d(Z_3Z_O<LsV!B`x3+C2a|YNJ&wA}0TTbM2%}l&6pFXF
z7oT{ow+`=oavBame{YIbArlF&%jdJBzrMLa_B8*1mMsr^>!Z3yrgV4{J;xW@>D^zs
z(jlaZrTq-~r$fi#6Db-<Si`?z^!eKfF#GEd;CQFua{=u*GE^4UBx7PE3gPRtJ3{>c
z`J+$Z!yEGUd?4pSq)D>wzUK@x_O?#7sJ9Zk!ezw)N4V9urm#TXPtBGwVGx4+3cgzI
zYY-{L8xCxds279knA5nz+7aVc=Q~M{^3pbnBlZ(H*YnWFdrf^WWeB+W6Y7K1i!wnn
z%mLKtZ3<QG;CC9Ib+KhW6}(*M$EMIWL!I{Y;V<J73cSg;JKt45>*RpIm0fp)iq1Z%
zh-SsnP?2yb8G<OKa@dl&$bj?~dsS`hITnJG?OY5Ji9D9-uYee9{bx)P4|=!C*0dq<
z#S}Z!_^6ZFR=p1@=_p6IV5El4eE3Ck1BpR}xa=V+PB~^JfRuabK-~Mdv;#95R8%5i
zW#yG{zmGvf#76}RJSKQ6Tl3ypPNixQ!Q|W37EC8U5$CR;#)!ccZE&dM(_7H>6f-?~
zp_s3_9}Yi|vWCdX0K5qaiqWNyMGnq`egQ?3g;Ga}`Q@%EzRd)@e`!<rf?Xh1O~Q?$
zbBSF?6>i+}7JUgmx6#O%BL0gfVI-BzUM97$6y4w(Q!rzmrIP&<S!q~Hj%s1zy}is7
zeSFOoakT@0=?huh^zZ;e0WKDWKh=RL@HSNA<@w@IFv&t3TYB5b(ju1775bc#x6C_l
zh~K}A9IaZlz$Y(44x@~1&Uyis;a?rsPGS#If3&4aJ3k%D+zeo(J5~zrdXKbAT~$7q
z{~TONIu-fHPIc7(ZNf1uD3{Py$VXS=2huC_w%4%f3%tk;d5$I#g$`+=h!N5rk6v~w
z*%pt|I>*4Dgpa&ncc(mhtaND(DSpHVV0waKoKfl|dwSpqJ+@kZwD@jR+5x79J>2W>
z+byeMFGJ+&MU<gS?7%jKY>?E4VtYZ4Aa(9<>fhDHi(d8;#HALz4~?eLhBrSwmta2r
zYXfeLGD>f(%><}$A<fvTJRp8W4_M$xfQ-f!65V8cxA2zfGAdHi-C_R`sJzg-NYB)o
zNKXX#u@c+^dx=lmi5rVfey&}_@=<!qZm<1TS{B`V4}-d&w)>&t;`Y(wrT1xbVMscP
zT-RSxnyY?ZoZ4yOh`i~>__r8J*}V96ag#EkJleK=DN>Kj!nmS>bBeQfS1p>EuF8@R
zHuL)2scppP(z3VAT6OpIv$>b=j+n(hZE!E^PH^wea=hzsfCTMf*VkWt5{X-PB}n+C
zq~gbWCfIq4LX-Ux(qjn>^yMo?S~msduB~ZyYsKfa9|<rqH^bBpmn$MHS_O?e1Vbdg
zkQofOg_kdP>>D6Qckw15;}OL-D?qfxpKt%%3dxKJS%_5mS9nQjjVzbdJvfs`xvcjP
zsFk$J|MJe>&T|}MVw_LL!A!67?F%BnUFXyc)aKFg;E!|9a8aTrChp=E3$3n~zdKC@
zR98gOB*9;(yMVo|Qxk_J6?Q!aAu;~=LI%cItRN@!j^mSnb9g^G;?C}`OGW6556+cb
z(aet8n9by`mxdprkFQ`{P+gSMv|Jc%#pAb)PpoppMrky_(Xmbc_!(%+H}ODK9V}Ee
zx=!&dwz3ps^SOuDE0g3E3V`zNX+Th=pB7JEJt=OMQh<*goM9{tB7*n_!zf9@JE}h`
zSZ<K&L#e;B;*`IVjAt;ju^4^R!yZkgCa=Jg6ePM@&oY~Lv*8B;C}9no@QuU6KciAk
zFs(BP?K~xv;u7#srxL*RY_~XkD_FS`T7veq1VbJ_-imEJRZ<hstrk8h*ZmIP4;X>5
zuo`DYGDp4107s816zT-l>(w)8g%(MY!PHy2S{G;FxB(^WyWSc?Ux3y1bGy^-);tu&
zx!+v!3M!cn_Cixw;k=kR{RO0YU?0%&7Ic5tN~Tl4E{X28u=FPA-XDUYprqNXK)PAJ
zBC0EOX&%k~oYI7P9ANVNE|M@F`~z#;k60g{qRML?uuXG}!=lck=vOB|50e?#O9^1O
z*b}COK7P|gGwS2bXXUoBBy9kS>0i8s8eJIoo3$bs7TIRp+y2)n%Hse?^{MoG9Fc8`
z|7m`~bj2eM#d~b;@No}hFo2;48ZDuVfAcmlsV7Vraogrx!I8^vz0B72m|;=|yc&Nj
z$DtkuC3dh6RJn3_+T97>$3H8^G{@9>Z<(`j3wfyr;}rm@)Lss#*K|{zoi{DBsUDzv
z-)6!!$?l9MrhBwoyItlKcN9_iqtNYAKgE)_;>kW_lxfOE)sxLt?9i99yZK81unmPv
zbW<<1%MwnJZv`c|#F{F0kE-uNVD@`i%o(@^Y+%4*cT(*Koi3T-IlM$dRlB(8Mgm3J
zA?<Jwc}sxP&lHyBjCjXB@cfk*Tp7+fG(A0sj5R9}Nf+tUKyfaE=y-e>7YuY5M1tO>
zCObUm21z}C)MN>XFbl6Dvp5~ROljHkOZb9SA{7KPOY->|$k%u=cj{QAm}zH|H{H)>
zrRx%sX<iYj9Dm8fP)H@x(pZakKjjm8<?-Qg)+w!!DUz;;^mkEylV9qFnjTsg|2{}|
zZZN{^dg_MhFl9F2!h8gN2bGM@*f6H+mXb<wgpdlCP}#s)lc_fisJ*D;ZWo)6u$)vy
zJj}4m0<%7=b5$~^=-Y9$`!c1%YCwDN&n1uWi`z4edvCt*0JZEML7eK@sTJ)PKB*Yv
z=!L8X0{o;L(}Le|?3Rqri7Z(MVd?nYX@2wHtP+>OEO5W_2&A1K9-i!L^_uK~hJhc6
z3*)M=Uhj38UI=StpkX1IU~4Ght~$F#?YR}84^_RnqG84%ghN2G1${0ICWX_h>8D-f
z1lbOViZFAVx-mCsTCikHdl(#IJ(g2w<Cb^jDcih@@(UK4u#CoA1;V<VMN(jLEA_((
z?7FGy!q&q5;Qp;<bh^k%@j2-<j7!DFQiEO3RKuHKZbRv7LDE^|e_n|RJsIGkDvYlU
zWvH?bmv0`sS`5*ZDKOi=$s!lsLdtPM8bf0Ry!^DPH42d@OBcw-5eQ<5WpjrAC9Zw`
z)P#7)G;HwCZlogSzpl9!zI_BjGGA7~s1M{DZN{^zhgpqv9T*rCEK@#NGi;GH`E^B&
zB=+x>#ML=#NPZ*bli{X0x_Z0JHuNGbvkE|}jOSOYGpl}9UGp+ZE&m<|?tFY5B;+8g
zEi)`S^%jIgpJ1*UDztw6u>R$KR~PQBMm8Al*4Dy;7GqH8O#q~cO-!xh4-`k}G&cVl
z!#!swD{EW{`|^>G#(4Ac>g{&rl-Y4Wmc(53Wz)^{O`5A^84>*R2=QfRQs9xp=T`7I
zg2BKN$+D_b@rcLRm7gmvvcCer9l$2E%KxVX4tD^ko^*K;t=j4IE3{O@ITXiG_&)1t
zyXpt00j)1-(rKv2U^*~lbmdRbRg3)xD?iSL`8Tb9z-ihY?cU~Gqry+<oOPbKhQjDy
zq=2BYiFsyU(sBEZ6U+_z0K?SiOyX+_Bk>98k)My18gP&<C7~Rm!-j<$7D<5HM?+z;
zoRr-!aU3ppWsc%M^JK?KEvS7uM*KcBJ$^$$cmE;xwG2Q;?W48mM|mnu{eM+*-v4ZV
z|Nlo2RE?@VVpWY;QHml~QM1)ju~&?uMX4Qo$Eu21iqaZ2LQ$he?Zm8V?Y)Yib`yNk
z*Zcbqe1EuZ*STG{a~|h;p0{(|uXA0`=eZKmV8!<Wn1<Uz!yiT-R}&G2aD>@C?Rx!m
zxgQ>Qp9sH5{K4-KPm3HCqqjvwVD{7mt|}=yOh^)`a{$)Ei&ZT9i-zPY601V;;Qo&}
zF4Hha323CF5(fN2NJxc4ui%5>BUIWZV#@W_AbNB3tU%xk7(%2PcfumLF4+FBy)+!=
zqoa*7xjmn|akUZOPp>Q*gu8qR(`_I;l8UTb_Y!)Pir)Fdm~K1GZ=%=?y2n&b*hc8A
z#_}jkwx#!hp$5lm5T#*38hOmP*3C<oBZg5Z@>#rp*kRzXHaDI@&+eYG0?Q5<5PUaz
z)y*54KN(F}9Px^vv03oPv{b(86o2_$PgmE;%F4<cV;N)aReV@hv?ZC3aUVK3I867X
z+{nvaX&24Oe>xfKR$1B_%+2GB+H#2K9H{=dM_$zCWwWz+6HL%h&<on0q+)AI{q%D{
zr!&M<(PKlS3!eL>cL}knf#~3pntw1|F(3Q69M-PXx)lqsuzqX&<+w8FdDBwclLN6~
z@=OCGPw;%k+s{6Ke(yvvPZ9TYOeH3GemXZ^&v>xO*SWYVVDfI#K^Qw~eNl7siPmJN
zl@p71U$-Z8vPwT#JQ);tv^kdL)dc#R3X<2m*Iem6xm00o&TDxxBYTa-rxWqUXR39e
z<#u1^a?kNyNer3`uk>7`N};s+D%C4Hn&(E;Hl$mUOqi{#R07Um$QR3(9CP*!3kGB>
z@HodBH`5m+uS?PpLycN$1<eE0Tp`hq^?_C;a9`cr;M?KfKV%D&Ihjq(=SgyXQE*bi
z^+{Our#Og@!_QM~7twkW3N`TUC%;7Mz^>~hvv3gx2iwm-UdnsOeyBc=ylJ-L9p@%n
zH>wdK-J7-i`(~!`@{4z+E>z1Rpt$E69T|fzr#Bbo==KSG`_y^disy;ENJ)uYJ$5bN
zoK@qV{;X)^i)ZP3f}Vd@(If##)QDW@vkYLt;Kk--IKh?z|7jdP*Z3S9aFrSBAPzKt
zrYD!oS4!6{X+31clF0<RdSq!BKzJ#(?l^=QUUrMkUBUcZ3MhhY5*Kl>TZs8$B`wk_
zV7ffpFyN(v-vS+NO9=RTVk@vSStv%A@+9{I$Vi1ElSW)@;lbk30K9N*6(7U+E|B_M
znh)mWwXa|n6%d7qs)LZ0eyn8N&NZKN$>OHT^z&v5;BzkX6uA)19Pqm#AL0#D%?mDQ
zp*}A(cKjFCI_1Z_$H-It%D1e31*>5Cc+#^xfgZvS{8)5#*tlw{%(9fTM!gl9qc>i3
zK<-|2!3Sp9#^J%5<X_7zO|Jt3Gi)}jhLogNUZXIg%!&@Q`^{MIo<K2d&w1SvPJ6I6
z^!cg-kN9n7Rh^}R#5zPI(=b~PTXB{&!*z;b&gY&YZl1gV2oglX>MHg3k=(f8Mm+`T
z);#r!Ac3apd%S8eZcg^kTXMq!%k?X^;<dr<<H@e7`v@QC>%a~;>Jn%u8kX7>YDHNY
zloxJl=>)UIy!7L?Or27+#eV(j6ISgftm{f~35zRU%WJUod=#@9B=^3)nK7=Qv)%RR
zE!#5DlcYr4%UnoTXq{?U$kWsMWmDD9&-O+Rn2|_7uoH+t>pSJt;<W9;XQT`ZR{c%^
zmZkv&4C^bWvKG*EmkN~=yM%j(W)E1#z)s*bTED-5r}$Bx%D@$mfYTIiE)`ymi*2Z(
z=eO$R>O)*5w_xu?C-q~{sjvPpelU7eis}TX|AeFQg|`&wu^%z2%ypk0l1LM!$m)=e
zmP;7)?u)!D%{(p#<k)e*vXWFKDF-O{z{lu0P7NGp>G--!YZcBuAep@kbG-qFy@_iA
zr4jQqHy_H^VcGx6^u9N$ByCqCXVewO6lr{I0X2<Y1sK|MbVU?5Kyc0LuX@QSLOLGH
zI5gbS)_iV`Ncet5sj#+o*yp=)PZ9@X6`P`V2(=o7+>n{L2#aRSl!jXMf~yy*u9F}9
zgufQxzni#@9mP|6x_7ni9MD(8J_^0*x5-2!yJ>x)o{avejJfvhT-Q8`abXr~6{_-b
z#}&LO4niJLW5CHu2)jm29<C3QE9jJQ4mn%Q{*fzl0OA!S|7kWgbP%$D+sLJ&UQ$n{
zT3I*{hYlw)Pt{>Dj7wcF62Aum`s^!86kY_foW-@rw0Hhipte=<oH~oTA(<IPd@d{^
zvdum=gR}(CdCwSR!MqPmhj1zzJC9g&60zuV*qmQyLXzaF#)Qa}(qo)ioEt>((qZpe
zWKnt7KItkX$nUOa8Y)Q=8p97}!_+?z<;7SI4!#%h>PRt=aU`q*v+ReuZC5lM;r+<h
z@Em4YO_1ojGJS#M;fJux(VX70fl~JGIfy$3ZF@nIryc0}(}tN>d$@qCD!&nh)7}~-
zrPI8_okI!sAW_zPn>*?}k7YE31vT~L--#L8#UdpiU}!$`5x=!@?o!@Yrkkn1-T@ip
z?c5{TgS9$VGMTYR6*}rj#E(KqmCbu$y=@?^&&Z9)ebv-2w~ee_fa4<ImhJkd@a*dV
zwN<3=hi!K5I~~=ZFB)P@4!88-+1q_Ri79}foiEq@8+<Esbs%i4MYYJPFfY&SUi7W$
zRA&EdvC->Ir~^3y-7nP<(9A)+w5|F;s^yJBI9YE!b_8!nr37;1ql|cXGY&}1nyUdn
z{(+`IlBvBDw@|3O{thnoP%&?(PfXrf7jSZ?GOPaDNJF{to6_FsMTSAf!hd6AvRU>8
z8*m!OWiltf0_0@wHs{^VQj1t1y|cc~hanSsBPE`xiN%l~1wl7EfSf-d`SgFfY<c5$
zhqg_xI$*shgDK@X%=+Bj3uan!*DJ*78yO0JTP}IY;svi9%5dT1{)P|1w=zCR<hA@}
zc&h-^t;aG@QG5OC-)n`L2D4eR&Chkl(27<5eVuaR+%{!g3+zyle9twhR{5cNe2yOf
zI-Cr3g1Biih;WN&u9iv;uf3)vMajeF%iCpsBFZEs)`h!Z`*vyZYdiQWxIM!U@RY4D
z{qC8xPY+V}?lrPn8=;OgwijyRiQH^s=q7m~TSGx1%45WP9zE&Tcxf#gi?i*X%^lwK
z2OVA~7?p**t_LbM6b8*ZnRN-Gc4YpRr49U+9#KJK9BZ#@&%Y0693_uUXStb_D4m6h
zDeuI!qQ-3WNtyT==d<3;)7OAh&RV%##bvA|wVZ)>v(HlM7Gk#J(`Nz75HK%U_ImZ7
zwQ9C&n3P|W`4-`Bx!q{1FEEK&CY>MCmaH?P@aAiI3;r6bpOFlpOu?`Er^h0~FwZ@p
zL!HgbC<{!Njbu!VDlY4ZvGJDHT}odm&0e29Ww)wcw|Mwz<%5wd3Ho-@8nA3*&}%#$
z9HZc&^#X?^gr8R=eE;ZyOQxqUtE!mYrL$)*mQ!B_v_a;g^BlYIe6}Rr=<)^5E)~gZ
zp>1ps`~D`6x1;zzf~)u}i8`&-(`sumH*l^JfwBW~$E&H@q<$xP-c{Lmfb}rp)vT^Z
zbYvn=H0$tMr&ivJ18OY~8WaiUW``fA++M6L?8v_9RN%1UI(X+0z5g?@HoXO4z}1bG
zcuv9>9N=`m>nOei#t-jNW`=kKxW2ITm0)ko3zvIv?0cOx_*P$|G3V~2i22ED$?SS$
zqZi8VI^EI*gH$>ER8FC*t(WaGP&&sAKu6t{DPiEocb8JT_wr(8-`sJE?2X(g<e1VD
zGwQgk6-rf0ds1Q%Xctm!;ql!J1#Q$eHIwrb#Oz2;RLs*8Zzhfie`v$J-(^1#=jjF`
zP6%S+i_5akX0+>Lo&ebxpY}s=-O6W3j(+PQN<vxqkkb>j#Dz_AL*DO47vJNmN|c?D
zdjmh21)DjzYdCyeH&FPw-OY3pPBfN^qR<UC@iXy!)3;RBB|A)wBiaB1wvt?*zR{5v
z>R#7Yqv{VO*DrzpwJj2)Q)Vz1T8!nLr`Hvfx3Qzx14ak#BS{b{D@iXs2KchnA6C7R
zUwkL8@djUb>aa}<n)Q!b(IBcCdhTiS<8&=L@SQR$@)&<jZJp9o8d=h)dm1jcKn8pn
zf2jImiJXFN9w4xPd&%bMX^*M$y44UOwA*YcZP*%-Zqr69Qx(~{B1H{(jPc8KZF!X_
z6gAttq~6@i(0qzkP?idVd^`ZAX2*T=1{zZ%67}ihp@}dxZ;nh<E2sm~|6u=RFqJry
z6z(Ey1s!5|7z9m)lcvkZ!Ea95^`)k!T+dbCtjlxI!Squse}D^cfz&bNK9O~mzl|Et
z+dx?dj8gHDW<>Q@G}O}WG<JTFR$AXwOJ2kw1p3eP@X)1;U+J$QvDpiFHFl5%;FcG!
zm~iE{WIjG@lgRsFzaP}*4o~v<x*<mhpAnaDJ$j-4jrj3y=KWFpLElN*0mZscaFhWC
zQdQlVy%93b4{X`~YcIs`9Q*n`$5q3)-)$Wxs2O`A(Kp*vPz;%ZivD{zrlj%h*<&jj
z@S#t#Gnrk?uJwcF9_e77qpzC%uCEP>;l=7rs_GMv8@0DNI8)(|Szu?*woXV}_Oh8$
zoSwxiJq1~<RuD1(g|g_xFa6ywZ=ivM?Z*XA7p!Yk#*EhQ;Cidj-WhN|qtpc0bnSKt
zej}`EH|_nQIQFC`XrHsw?1#R{2B6T#&TeY8Y`ir141@b(6ezzCclr(C?xau`b_$S*
z93bgpJz+*2@R@43m^TF1;kO#!mZRVut>Bhgv87+#N;7TrjYoS3%i;XdiL234LPSaK
z2v3&zFgbE~;Ev~14sax)04n(7*Vjz%gd~3`x;@)B;-8~GTYlA|5by)m>U)J_#YLuu
zDfMd2XnO0x(kl74_Hhwt3$Cdo=57f5xOm?LNT^kIv;oeFDNn1cd9n9rW>Bw~9Wo!U
zFYEPXmOF4X<%@s#SzQu|da(A}=#F6@0@itSK!Zu<)!(`35M}BXFL`$@zvPi7!b;kq
z^=4v5sjyd}V2^liY7a!MSRS)e(54;gpJ^3(nVx}tEX0@KvA3Z#<|oTU^#~)6VHw!Q
z2->srNk|icvyv^a=b_6}d2VmN7uyRdd6#I4>p53_ZgDzl44w>U6i-Y9Njs~y?yiD!
z-^ys`D}9cst)KKOc`=C+*Cp{^a=x0v#&c$=vaQ1p1v^R$(i{o#2j4Z7=jq2G<>&TJ
zd%zkbE$@@j=;3;MXZj8`u^xNAD2U`CL%fO`_Pd!=i(g%?Nhgo_%WDo8(7Y@K8<mcS
zfm7MpkuU@52fd4K)Jb6l=%CZ>D+7kgtMmlHfcaRSxQuIJ8F>TAyEVhDE5ahCEWt4W
zx$1RaWOc}fuW9m47v{?zY#PCh1D2bIobi4yUJVbs>Yi=Y`WQ_mwsU0#CiWoil&UUf
z1lv#5WDw4>4iILHSJs#?lN*EI!M}K-<Ih$W48j5;Hu$zKi{bPeFLFg57~L7wz3diZ
zzZwV_)1@C@Ck=o`?CIX_A<nKGof{@(pJGYqDw<qQ5oR1+=^$<$8r+3{jr7_}g;sMd
z4>EO(J-JtB8|ct5?({MAA1)>!e-PNf+WG!gup?Bb>-V<l!a4;Y&Yq$+&+Z?mti@OG
z@RHCWckj&;L=+A<<$YGnIbYr<nDoj!#%!pDuU=oZ9d%%})pF^xeTs<M0X{O5J6#>P
zw!2DOoAaN9o%pvEl@HJNv^No{Mc=%<Uc7B{%>&lcR`OQ=PoN_Xm3F!^k^koKDXl+L
z{Up*|VaK!W^Phh!7S&+}47(iBQW9{T4Jyz8Y0(pnx(~d(yVu9o2`Plvl0A0Iw*Eya
zr{#PWJBr>6quYz!O?1xTN*jedlgX!0ce)tc?)X?s9+S}eFpA2Eerfda!hPbvy!QHb
zLL_syE}o!OBkGVp!}(BK;(xjXH2njqVypg25MhT5gc|!fvA@lXUZ7v@0W+W@-rKXp
zA^!`Wnl+sAGCvC+@pr-*fvkaA)K^l*tI_<Lj%RPC`J&-Bxzz3W6@Rz<<Lg%&l!EM9
zf5z3pg&;I;qqJfxeIk`a-JH@M0%lnK)tU6<<64Fv%WR0J_!Xg^<#K7lYBf5P_f03<
zAHYc&D{RN#{6C@J7yX2}PySH;YWOTYnxRfrsyeLB-G!IRum&Pb>9|)H6bk23B7D<H
zW3~U^T9paI_MbrD{{$-G6d%%Wb5@_<aDw(v=k|_8<^W-G$%v6yiio4?vJdLw?td<Q
zk9j0?i7Ki@6?Q^wwD>+{7;u6E^8Yk;E<7cIdR>Mx_wj06SrkPz|Kf{q4S*WN@Pn8%
z?}$OHTr#OF;hXFG{%f_^lQ-4x#~*C*{pXMW6!5oySybT1vn#$dH;=G4wIsIdMm|mU
z;O3li5fCTS+smY&A)gA0Ng_-C=@-u|4tP~B<upTS$=SVXo)3ENwm$3Sv5jAIYFK3j
za*}emvZJyG@uVKb8c%uqt=#AWr(C4`LNeLLAWi`GAa?52>%-`Ex@^PBVTc~_sV>0r
z89O;UHoeA+nxdaD^+P(8{P^+v+X8C%_X~@qi5PC{ATggicb@vWgFoT6_o7gpJihPY
zW-PP#M>R6H)-Dgyj-^pN4W$A!eDz>q%;e8xZPXIL&RC4l^7Yl)+asUx0HG~a57uJ3
zcl@+%Q7d=R<<k;-UBfJ&wAqQKZtWwa<nXczulH^Th>;gP#Yo8d`G><PR5?#vJkTfI
zaW7>#g`cQ7-*qSn+1l^kB(WzY_VHDJBqdV$4#d13=?yjO9FEahNBFZ(eq>kzja9(u
zjGXV*xbra}OGtm*L@_40G<>dU;27QD89;aP$R`yAu-Ae)=|%Zd@oMuljF*f(<}r2X
z0x~qV5M6nXXOXHKQUX-0?1m_R0b$(y|K+WEj=-WqH9w=4w6adRf4!?knrfQ2#0A8q
zfg^cT#ubf_iuGXMet?6ZH&m{u(LRytD$ZEEZsi$erI=_1$j$B;V5)4c63w%CqsrB1
zc4yi&<YfCY#dQflCsjn_PgULeEJf}$44~Kw%5LA#R>w4r0NR-O2?DyGX)`zOIiAH<
zuYUWZ^l4b;sR;OIqp6>KP4SD8>#>1jx5nwyYcu?pO#<uFSk6C5tsthfs}s#4Hh-Ya
zqRLu9IgRQy;Afw~LUvl>Obq5MR-<Y7gZqOY^9gZhN;~a6?$V}(@Uig4`ILya119*q
z%RT4SV4j;tjK##t+{DVYp#XQiSR0~6k+a;i#Y&D~iBGNF@93Yo^}VAXJkv=~0L1-c
zO!5VKDv61Z&-QKSQ0JB(@LA4Anis=fxMaH2X5_RE<_$6>OD2pkH$N5Kf3w$)pe!?d
zZzxtDWrmUt`c@<YyIWP?FC|n_7}wSFxBS7NXxiM8`bw|1?w3x{2QylXB0W{(tGW}D
z)=fT<(@hVH2n&9_JTIm8^60jeH<a4z-k#)rnUNU@m``!O3iZ#1K6HkfLnSwjp9p$p
zAG)M%(fz$oVD%0jmi1D&fAepabRqg#Qc4Gf>LUD7br)VQVgOCp*b?1P6P+>Lfcq#2
zxpn+_B3Ep_BEZu`Le+`|-5EPvdk!i4D82*P7~>!3Hn|@k06R9m`S;^|dbEmWQp)0g
zT8p83nU98vSvY~5Ih4rGZ}FMgw3`VtJF)n@SY5a&vD#8MhM#tG@O{u!=ewl8LLSyE
z_NqoP2EHf7_X6D0)Zs%LN)dUwghuOLTBb-XalW*&I#BWV$>t%1nF9UCorO=*9KAJ+
zorN=e)`-C!V8)(l<?6kGfH;cok{`C(qltlc)X2ro)l_$MQ%+_~)+IL7a_u0$9*O;C
z?A8aKV8TA(`bxeJsP)8+sSSUldDUOUgHF<0gmCXeY^&{<+ubnI&6t3lD$09KlL>*N
zBA(fF{_HhyXudM{qbCV3QxD^8&KP&%5{h_?;Va=7pAJk|Ld{8=F@dZV+t)OR*c=t>
zk?g=kT;uT;ScG?^A<gq|Er9LuhTjLi3^wOu34eS)oE2oO2rrUK0<Npa&TUlL%n{$7
z?0y_(A#$IvgdlEEC@sR@b*6e=gz@9YcsKY=Ol9kV*XY|{q){lP9)@Lx2fhly*<5^r
zNIl!W@yl{pqg+3K*?UPNklmYSAM)^aal;ss`#@u@^lfXWH6?IHt$~fo*Egcqtaph+
zanWzcvt^Gxgotg}l_59H*_kMmBA&JYtSLDHVNNT#A5NVVEFJ2-d7kt}ioS~bzBY4>
z^=l@D5=vl$Xs`i;*6_;6<}$qEp1e((Ez8*B);o#e?SqQLiCC?l4|5t9?j4<?LHz-Q
z2s!J?nJ@gkzvW3XF5pG5nF@W)?Jt5R$$a>w>$6zQv|=OVBbE<mqJ^70^(X9H&lgo3
zK=dzzjzPsQ^Z-5w5M$U8m88jdSvZ?=p=|IKLmBw*U%HTMW~jsE={q2#N6WyBP0!}_
zy!CTNC4rx!7UKhW7VTK3Pn-dEwUIl0{&KEQ+5a{W*gv&`a||*h+YPJjKYbbhcF6r6
zeZKW4ap*|7DR=J$=GeV|hxs(I5|7nsQn$Ib=x8Vx4!5w7H)@;4CXO{KeI8Uh5m4)a
zx{GQouuxp}VXzN%5Nr)vNZ?<mw62ENP3zJq@u6q1NhXJv_ttB%>o?+)hM*b_83M|+
zzR?u!Gh<WwH+l=@pQ*@&uQ>yoY;ENN*cZ(4ZX5=VP&WL1_~uNJ)1Njh?NnQ(A~R;y
zRPgy4Yk%S-7i;B>M1kEzww4}>lCrihgtl0JFs@C%+})`1ZePiuU`*LCf}r)Uxg)VT
zRV}o_-OA3(03OuUO&J*as-b{4V@+~=P0HawTVqQN`P*s))8JXO7s-$<YtJQTGL^DW
zMjK+mQ9RV@t`48;YFdq}H5C?l=}=>wC8Yg2qAHC%iDyH45pf*(Jlp4L<3~!WBAzcH
zbmDg9Xyg5XtG#n|JUre7wQlY4g}GMF{Gw_6k$;|9foSZ4M*;T;OX+9aZuZ-$nbZCI
zYfJ&t7oN8kM#AtJgq%U+`+hw?zHQEolNhw>J3*5@t=YEU$E(|e#224#9*dg(nZc|a
zb(ULk2Ydd;RVaOKFF2<2x?E=hJV%30tXwmdZAKeQNSpoHv%nM7Vv(dF#NT;=H22LX
z*ofQ<%iN+R;;IAVLutKEzX(?ZNFD<BA#cMo&jEnl6WM=P`CS^TC7r0O*RRpm&-`Tl
z9+i0>K{dU*o$$bvQu2_3l2{Ck((he_e<gV=H9{(R$n}eO>_h%5{(qmA_6m1ka~wT<
Rng2HbJ-BbES*30p@qZrhv6uh=

diff --git a/modules/web-console/frontend/public/images/igfs.png b/modules/web-console/frontend/public/images/igfs.png
index 47c659eff5fcc457c106f534a03b7f4e727f6c38..b62c27b60cd93babdaf7fb3912e760f0d111aec7 100644
GIT binary patch
literal 14139
zcmaKT1yEc~*X9rc1cJLI1a}(<9^4_gyA3W0?hxDwHn_XP;O;s|aF^ijuFL!GSG)V~
z{<XJmSDn*+s&7~K>F1p4`*f&+oCFFIArb%pK#`IZRRREDK>)xz@(-|YHMOfEd2dR%
zf~<<z>+35U8{084#qI6w$jHd`_4UikGY1>T>+Ai_&DGiK>+_q%?TkDZN6*1+a&pSu
zoBDcDv;QK@#(sT!!p1IqbG^sGDSx-uHFNvS#;*N(w^^pgb$WBtclG+bn}0RxtIf{#
z`nsQ+oVNb@der&Dq-wWf^=7jg^t1Q;^|~{8>0$Qp^}g3)Jy%IcsP=hl@n#kBuwU;W
z@Kq^uO+luXjmKv7@?rb__2z0p`qx}OZ@j9iN{bmsV@C4K%&eg7kA|j3tJcHgPn`La
zXWtz9A7)2i@Ae<AUq?Iy6`k{z>JkS+RIcVS*yX~(Db<dm9S2SQigM$=Vr*<8?qwFg
zuO%twljYcSYhPcNW98Xr4lWH6$Bs$lqaAgwF8^|Bm*n&vyq=7l)gE0Rjt`oPJuUw>
z=U~fIW|57X9bVZxGVGQI1s+aRSbf(h%RBEANxGW8RWCm&`XPL4%>$C-71CV#HFeLf
zoN=1DJRc%?4y3q9OqBEQI>>9kX(_34RZK1FJC-ly6xH$26Z4HsJ!ZFSoxHo*dTGgW
zEbp3Zwri~8D-4@B_0&||oNAlyDf8(&1SRc4eVceJoAjI_FQ8ioYoj&OM<>2Rj@cav
z#tNst+?EkFsgQ-Cl2k~T`D3{}NM`8uY){E#El#?5zc=evlcsrMHG5*Mv2m}?P<J%l
zNLkB0xOzfYtNtFWeBhZsTInVIO{PG5tWdQsIq2s^Lfg1nLP35(U`FO+l}Cx+R2p5v
zu~4wOGU$X~MHW~=Mpc=W8ZtZIf8}L;Xc+u@cj@W>i%xXZRG{iyf~}1|+fTD$d@cu4
zv@itOo*lm2*w{EaI(&S5JU>6*-r2stzh7Bd+3((01OV8rq(p^OT=k3$8tv~VQ5Jm_
zS&+oL@}cVZ_!assNCs<Bx{DurxS}4j&?JKL{w!1O8w%1=(ZY36P~_FJ2RRn0;la{|
z773e|%rjAb@%wF*h$~H`fQ(^LIO)ej^P#!kUpZJ(={0TDBXK7RDF~T%`^l?QB=c`t
z`*P~->W@l*VD5eCX=`E(jNoQ#<yhN$z;*k_6G}ki9Wy<wAPC?)3{W5d(7^&!;C+W7
zqVk`X5dn1cp3ayy(-+Hi-PxD+AVASkUs_zNw(5&<2~ByLw~t8j7Z)hWXAXL{Q!8~q
z;Cb8}^b`H%krahwO*vmTMqPJ`;`TR}(Xx@6-4$J1>kg-|$3?WxY41M`xpQ$kM+%yf
zIr9zFWHo@pS(@SQ8uD_W8$@(iG$VMq_MuvIRTm+(=UymBOEc_1M%F^WraVGq=wN<%
zLfVM|(}3t5zP%ln7}ti`tdA<F{li2-4>>TrNn^0jR*TS)T7EZg=I+Tvbg?M%RACyp
zg$9a^HM1CpwFz}QQt<~5dot6v6KuC@qo7jiT1)s2hqTslc~Ft(<yDkei^exrm{H|x
z&{Xs@qL}J1RMO(eZB4T~G&Z_Jfkz}U9G^z)HFZRkrj-^3g>^D?wiw_r{b}o|1wqwW
zt3$}&cPon+WTwZgLj(W9O@tOMSU=Rk&mCSj?@`CR4Y0*jXGejZu<mKx!@CAjc7s4Y
z97`CV$x!_bhAwwsyT4L8-xyRQ4%(@a)g#Zrj*c}PDw6F;>2U^yHc(FLlrV@+sn(l;
z02k4}e*FUBBC%l~*BYhORp4Kzxq3V$=q)@nR#}&i5ZBXP@@0j(zy@z<n(DFwI~o;1
z^@m{?Srg_}Ofk70FQqO5pZ8-F`}?{H;WVZMq-o1@C^?c6G=s{AyEo2n=fYT@Al-cS
z@kaHJCkVcSG>bEG?aVm^n&fmJ@x<KiSF|)e<}0jaJ>yfE!&Az6?}gfckEdI(0fUe2
zr45sOsw(wzw&P7=)}*w>pI{aJL*>Ozh4flV5bkdjj<)1IuN^kH%hM9DQ=D;7)017I
zX1&_JYIh{S(;VTS4z(js3Acu0sqJw0uXrB9)yDIk9WK$rY%YEtL#v-nHnx*YzS~zh
zF$o>?xYbIldiR?ItGw0w?tK2c<wIuL(CMxfd}ktgPe0jh9uROw3?9_Dy)FU*_RN}A
zkFFcIV6sYw0e}RUEFj>`^NWz*#5d3XpW?rpz;~^=>xX~60d|C+BEX;$RS_F0izPzO
zmeAF&(cf^m)4B)IPXo?>#t4%k2OoI@_k9<mdA)|7JuH(1YZzHky+b!jgU{2?2>Ev^
z@N}>VFmL78FyO5Sf_(oiC-45P5XvVuJ1LB8yr>)Y&W6kh<)>fUXL5!uCXgqz!FZav
zU`#+4*u-X1W)3QcKMRBWL*%#@O~!&X^>KZYxe3M>lb<s$7xXeP%0<RF)md@tt~(e#
z0CHjPP4kR@eMUycr0leIjt?zXvT*r|#bP`Y%nEV<AnqD>TB{9y8SiymXUEcFG6rk|
z_wR?FB5^T?pM=GSpFC+V@9$e0gT6Dhc#md)w%$Ej%@KsWSJ$WnE_cF6kiw#|n6dV$
z*|Y5d>)covOcr-xMqJsNNOMVuDFxqY<f_{HQ~0)01;~9dfZYc6h=1I_zidn#-Nd|-
zGl5m2bv`h!*J<Jqby#qElu4|N()n>8`z>bSXcz?FYJlB1m&gKhiNTea&)Vl_uL*=z
zV#0dA17v}v3CL&SWrB8ATW1JP%1LFU+anLa`TB(-@2wxqkS0SbbdB1)L0f)@Wr={J
zv}biJ_0Rnsio`+5A#5?aA=xk@f&l^V*=4}*_aPU07z)j*<xboQlb=<lakU%ZmTN3=
z^64zgomF8DS}G(o_KQEbFQ3p#+vp-caeJJ~b_??M3XRFKHjSeVrH`b0FqvKaZMx+h
z4@HYNKjHg8ma`~iVtm!Hv%Kr4?2LH7WHS6tPtEaaDKF1~Sic^4Net`5ml<EXJ{R9L
zxK}Evx$1z_tvP+k2T~IkLdRHu`_WI^5~8<7WBe1%k^L7?Ie;i^tTL}-;jmvXNG<9E
z_sbKv_!!hT<(5tfmlf%6`NRi>95ZxT(Ym6NbCREZAzDS}IIo{df;~~S*V1ZUu}i#{
z*Cw7CxYthaoUVInH!5{>l-P3^4xl(8z~`_TalwFZuVhkDkljiO5IOoPp-}v&j;N;a
zjYyGqYh+1iL-_Tc#(W5A0n=Y9@B1*Zgn{QoDkb=z^tfgGqh!svN8g&i5bgm32(z>{
zvoA|mfHJqg^CJ{)a#|!s6~9Q9u}YIbYZ+V}ZgTztwp#AGGk_Z)(Vv5ZDSj-DKYyID
zdi#~jX77LkAS2%|ovNwj$9th6G`!x{YJAuCr++G}Ur}+f%?tFv^%@NeNDtRME8563
zzn}RF$v|Ii^iIW5pEl6kWg?Xj7Tu<CQRC0&jK{6<-EP;PB9U8LLuf#4?B|e1P>c-B
zl@m2S9NPhdT<f?K#8NScHLhJt<+yVc1#{|&X!Y0Ck1U33Cz0g>-s{2|aDLG#NL0&}
zU@vihIMN`LSz$vF@Ac{Kc;GR7CRf&ngQP#fwFen1p^A~1MrNtL#dP$v^&=huraq^?
zu*EpZ#Il8=TC;VN<if0vsQ9eJR8#ZC_m9TP?e{d8m2+2&G1adod*42Y<&ALevg7y<
z$M@M>WbkR&jhwl+4K$V8to~v<Im?cyv#Xy3(UXcK+J)*9Mo{uxzrVbQuYRINrcpGM
zd$bLtrty+mfm!gLUO_;Mb}%YzL7fYDlw7khc_#w8`9XRJf-1inbjE9L$a|jkrC=fJ
zV|pB=W9qtHmE5v;Zzn==3p;1qkiRS-2t4tY0zqK#15{$^AoBHlJBDuBtn-d0K@PoV
zD+75jEK8jz6Y!4*xMw<N?QARn$nwdVKKJNu#j%fiuM$mmqSAsogsz0em!f24X7H}%
zPo^!JF0*!pekQqV@)XK%McX_X{(9l=C+u~HH6zQ@Blu<t1%kf<PZTz14hgiSeJuI2
zL=#~XA%i^cp6>TpawF<;eeAy|Qb8vryUV}C<J3!ByYZ=KFKq4`rYMG^5!;%bo3VO5
zXJoi!%*)C{U@UZ{JfBpnauWlV@9r#JjBKA&ajSUH(tww3r~^txj@(LVB24ND7!3hf
z8(ziEl}koFb4c;!OR?z+qq3g`^U5xM<+7ICX6J@m*f&M_XDOPIZ=M=iRN>UU5|`+~
zhgPezfPR|c9XP@a<1&jm+SD&PX>6<EtFso#5UfT?#cMJlAMtJlvE(3^<HG>sz<_le
zW0vpV33M0!JhDAZ&JsHt`}`phFQ@U`<F%gJ9DmXi_wo5sA%XwiP)U2#z(}o5kFR!j
z;;Z)JSN+L$uhK{W_Ss61@OxOU(*9;TmwLTo7Dwl#U-<wkeQ1O-m<q-B5ivfQ4tb2u
zN->aNPj8>OCq7j7Oa<xD-Dg9168h(OxwYL_TV37OKSE!res6ewejZ*y7ZWt}fl2a4
z8UAcLtlr(hRaF2Gd&&U!yV5&?HcR%jX~4j0oya<=C`B*mbjcTu<l2Mpd0{hN$U&aX
z7B*0FsKhi@+nIFuNdERoqrqyf&KjUrMJ}spmQZ&wTTUW6{JBvF)xvhVuaH39!n0gd
zvpWw)bQs;pd$^NZr*6`RwLlg-16SLzWeqn)55De_{9|Kn-~ER&eFe3aADrgo7JXbS
zhJ8h^vnUVsPj7RnegQZ9gYuQK4f&30ye$bn)f<wyACm}r>=;wH>V-FOpLF@ME@3fq
znu8~*DT{mooF1ppijhn%rYq?-IKV%2Kdzf4$H#W=7Ol{Q%8AT{#cS4!8Z*bie^8j!
zG(=T^w>C^ry#NY%W}@6JWfaPOIDu|}+r|O<IrQsSRvZ#K8jZ6f<dX(7Po7m<-bZ@s
z4L29L9|wM1i+mimT#RyfsL2<;hD}Z(fYh&u&P0>@)*-dU6;5utu-@m+D;;9TD+*?Y
zS?t|f<DtAeg)(%LTCNzw4eu>Ix7ps+reZvNjPEnaV$>9-mpBa-Zz=OA+X5v4ty+3!
zxMuql%lFOo!97BPW!r}y4AGUKPCfPG`nefZwm1ZItBCKY185sz)iZxFWujE_4-FGl
z+3WP+M^j)U^A&GI5_^PZJ>ZBbn4%ckDzr07r^=r(j{hiaoHLd0FwmVcW6<B{sEm7?
z<Ve4C97Y!Fo%&|)t#h1qei!$oJHGbr5Af=i#w5COBhX`&bE3)roYElA507inOp6H4
z(fI2%Tnrb)bVY<mc`gB`HTo`rRMWWEDBopYUV^#9Fe;{9&r<bAp%j%5d_3tsk^6F1
z@|{|+@xe1f<ZMNaYIf34<-M{fffjeHEWhvY0vc`;It|wGM{f}t64<a2mz`LuJil(L
zxXdQ02#>*V*W~y^z8u|~p8ZO-KLVOUD!Wys6Ep;>4KSj~L6i5kHSR(f3{8l%-+!Ys
zE58UtyDqRXRr^lRJd%19|2!8)Uunh?7X(B4lm3^b<eMcl72$8Wo7<Ew9+gkOMci{%
zz`++e&oi!O(puS1rhraH8e)4#F{e$}5LY8tsRqrqtPkMj0Bj1$ki`28Ay??G|2}Q$
zXqMtl4+Y=;yAOOP-Hap;g4SYAh5q<x_avx+)_Wm?`y8eC*f77-e_!_@Vs3!Q-P)Vb
z+fxpqMCloc`{SQsBOxSQb)=>O0nA?mwG}^ns@^_idDtX1=EA-Zl9>Jx{Pxulhe5;N
zssHI<3$NAC{Q)YD%0BFEV2n{iqSmTRhpAP5FNnx^1p4I~LI6l_eq_;_$qIQ2Fu|mF
z3#6thl?~Hw2y%pVL*7-caTF#3WhyPX!!)&NJyCQuf9QyXGZD|xiIMKTf@khK%S;tS
zy@etF47N**UAu<l$kiW32UgHOLgi!FpuD8?@2W4%Z_%6d_wyebnjAI109S&Tn+3G(
z9M6bmx$?G<L=3R`Q>tFd%gKQCu(#xf)Q!kDjEUbmofh{kBsFx8zr-LU2jOKOmDlFp
z7AT0z^vzd^lT8Gsabxm9ip}B+Xoc7tNjM8=$%rvW!8~Mo^#@?p+@s*PO3DQ^3zOJq
z1yxZ^&WJ@5fwWCh)Fin7SPI7k%&^yxFs}YG&ia1SLmS9tDs*Ff5wKN5@@jS(;hn<M
z!}5}`$>q3PJ3osK?-cLMQ;YhpkHsiTs7Sj?V}y)U?N@~DFHIs$_yCrHGC^YU1sKVs
zFUAle=*WQ}R_sup>`wj16TspNw?{{%M7Xbve8l9Z_QGEUg>CCQiMw8zmkGhgpcZ<)
zw)*xapSpInPYC6ht6K|bgS9mzonzkSYSxVO?bSXw<67a>Z}WOIV5N3l0C?2wHvQUa
zF;PR}@x}SZxbjMk_@-fl=QzRi8B}%-zLsXI*0T^NxCHKiUL`Ak0mTWhE+LMK%QB`E
z;tTZ82zJk|Z;eZ>p+wE(&7YPNvoxLM@$$CpHY%m!S=vFpjf>0{g(ElVWD+DHOk(_~
z`$GyGFJ0c2xpyQ%q9f3!V0sdYe)=knVq<Dhe>nBu4wF#y5+!?%;z}7U@vIL%wMsk4
z5)yO$38oH$3sT8HofAt)Ki;P#tiHV#Uf3z#QZ-7lzR2itRTaM=#&CUr?rNDF(_t0*
zr34@avi^vq7A-Gfo!@&21X1JW_z5kx^t$)(OgqEDNM$Vz@4Kl?e3LeW7vm8%Xc*u5
z9`c_9p%y143%0|@u-bR%E(N2<Usj|Q_(xE_|2~ZN5WqaWws&*5A7kFb`{2{AiU3Yf
zU9F*jDbv|~rHUu;xTE&LAkpr1sZ-oRo}#6#G7umBlTS5zBIiiA)XE^A-bbr+qxN@_
zOk!#9dSIAV(I;MN-qor|;A6hdu?(SI4ad#$Xi~r(Ro*G6Kg<kOymiF&zJ#EvYUNCI
zo7s3JQM3%*6VKy)+f2t!I-n)@=T%sX@t5;I(`LnXumF+qtoKYWsSuoYeaNK@3G;@Y
z>7K>?mN=G@c)xgL{KV{cx;*4It4ZYcF|44;y{qFh-@E&xEtn4@W0ap?6p?Ky+1<s0
z^~9H!x^Tyss!kSHqr%L}NkWeI{lCq^NoE4OW)xcjfdbhhFGb__g!H{oyD&#11{(&7
zI~a@JKxbtVu2ZdsGEfjOR4y!>(dlZzsZC${#N;@GDKh@sTK{|DWD;6(WX-WxD!)hS
z7}<zgq&W^=L?)0h0+w;4tkVx7kP_nDZ`ijq95>FHSw<0_e;~QZF3uzQFR_Fpl1Cv1
zjG4>H^k@m*Y*mhG#%?8selFe-A4D}r=>|y;oP+Esy(i4llt?fJeLE3LoGgSL&Z0t0
z{+)wNE$(}98wNAjTZF(mZ0*R5_3n0*SpmMjrF>y@hXRDsB}#TqiaxAh3~h_lZW|dJ
zpC;7}3XvuNSrsN)tU;CHf7y5vxYSaBRt*?%J&3M^2>OoJZ?0g?J$Qd#kvsI|(Hc_>
z77>sK5jrx%LYx!SaLGVd1aF!DuaBzhC7q>NC!p#2C&`8AhUw6l(?Y$1m&puPzZ6!?
zosspPtIbvRbnG}R<M@}a&DqG7e}?L*@8f~7TZbYMz}>&Y6QXVI9mvYj6X$=A79$-{
zKFYTcMcpy`G=FL4ZSATC(a(Q6g`NNI!t%@|H;MMu*m^h7_Ug=EiTAqzyaWSAthXzF
z++fS%X<}f;3Ueu(>^Izgt?f{htI8$AQ;gjsK?O%zhps}O9pA@$y`94+y#|+jHVxUu
zac`BF=@$z8mZt2rPjG5$KREE{#MN1|XYPZVbnMz!0!Xql$E<K?DAu;`VHm<&&8Rl^
zfxjsP{2JlNp5iLq<kb(nlXq%JR4M8*-*2`Omnh~0;;%xLxdCG7iuA`l=};G!Vf%7-
zbuBPLug?PSW+KNVdz65`6Oc2`B=-G8FR#bfQp=WM%kbs_+Z`=FQ3opkoUVtXVskS*
zoH;GnUw8#+07*h=USuIdQ3D+h{EiXZy+bei(ZMM8iTknDF1A^#p(SvHgZJ_o5SKiX
z3RdE8J>_$|@cy)5xQ%YGV1W#bz)DnUQQ3E@ef6fbD=maC@~mKLb2<&)icf|SDQ`2z
zH(O3D+QYTQH_%+se7ZO7YdAF)ZD8>Qnf-i`x!O|q1pzuX0zd$;=IEAf3L^v$01e}|
z=j<a}_~;9gP^%7ImQkLd!|(-h_<w4JVepUu0H6Rs6*52r9>4$#=mY``2>wyuYTvp7
zIBI0~Aaz1w6QFp@GW=!>l?0v4`YAFJC;8K9^s#LK4O1Kydp651vQ(J)m@Ltg0>>x-
zLr*)&{4H!J<N+26VDNt%OBe8H9R<pyhdyEASXJ!GC2_L>Axoa}mp`FljPrKK()~Lt
zb6qz=#F@NH>kofFBX7exbws*BFQ9tDgzxBg-exV>Zfay|Z{-*xfIf*?ko+m$xQ+EZ
zC;vW9J6AH-m9rDLp`e}a-V;}h_xRTd^}E&(jsup%gZ0%<?q_o<VY`j->-rJ%XlCq-
z<atypTr4BT{s5CM(Y^|_&5tlVFdj+7800sx#ZS(BL>dv11j65RmqV+PJ<C5O1>YaH
zMfT4eIYN|{A`44-$E4K!KzgX;L|S@?=o0U5hUjzr2S~yi1^tZ=ytSc4-}uGzK2eY}
z3_Vg%Ach)0H7SU$enUiyylo|P`Hf<Z_<5l}=%Q-`{*L^clBx(*<yZCr+F#*Qj3j}%
zq!Rh>yCJK8(J=_DUP?f8MAT$FON{&qm6%>Tm$C(%3ACSQn@%sw9zt6izM*-egHdeh
z&_2V9PAj!B7@N7vX>cShN8OLtma*}J4H8|KQ8B!D$;>(o$Vxv#$NHYSfyaV$50J9g
z&kSr_=V~$$6TbGNQlSl1vZ;-atLHdU?%zs-c~tk5PM)(lpVrqncXu6+GbgOwz3Fnk
zi`FDt)R!G!<Xw9mQeqHka6<3yl%ew}>#Kf0gFt+Az@&%}(w*55Avz3sQza?Z>}nfp
zO$FEa^)zX_!kfno^Fhu1<0@-8AE$hputIr;zLOAkLCh-&k*)NdrV^J%5lh$AIcmM_
zu%8A6{;ikv*E*~h$aO!gleq@(_mH7?Hh*KdWE`%^L4y)a8fuA^-Y;2I*34E4{zl2M
zrG*S^4+*pa++)BI&|<*G{!XdKw!Llqd!6lvRK1#`DYuvLQ<4*|Q*COs6Qpn|#?5HN
zJE{ea#B$2*+*tx}T}@P7vGgN@@rjBwC27eBh~cjp3V))e>QEHpCJy;TR?!2ObzjIX
z1U2XU>VnaYa<7$;REc}xd6!YUQi~&jE)h!JJAL0&CL$+Eigc3f2e;`24U!cC3LmQp
zGT}Or5ypd1ofP>KY`#7?B=^OpS9t%HwWF3$6);%n+op&4sa7MKR{60_PTRTFXshh@
zAS?!@+$Hj7WSiks9xDci4Rz+Y+59Ir&ZN>svO2n*%O#_o&7TxbvC8^ROq&i<@n6M%
zu4H``KRh^0@U%$IrMtgtXAso=yP|SmdiM2az6^B7tlmBu+UjcPPCpK6<|<Yd@FPL9
z0~Kh0J`jP%6<^gTtdx+{okIOWP3SVC%wQ)8xxWsz1g3hD#mGs|L7F-1e}2o1V#J@E
zh*1Yeo>qe*FT4oa6U0x0FXjf^ufxty9vg4T$tacMXHOgHF0L6e19)zG!iu(-`ISCD
zjn6^kGOfdYT^DaPbR0Z?aZe6S-T^Z;3-$*{>nw+)#-@PE&_B|2;Ir}1ZypR7`fAWN
z1;mFBWgNiD`Cofy^&ojgz$2<8c>9&i*b>g<ik7>O<Z3-`jtj3QlqLINIm8*Zo@W|o
zY2TFt_Hey5f6o`7GQd_vsIbm)B$-qpuO~8BA$I~xlb65Oh&jVnDHAuP8pRyJbwbEX
z=S5HQi$iqf4U?Y@5bOZZRg8Zi@E;@u!~8eE2>7J7SM{!{%w+ol?lyK@^TrDP44Swz
z8RJ3wbvMj}`3-jA5|$&hss>E~&J;8;Ewt(60BF@I!p$?FY6RYZz!&t-P&9%*wHhWb
zV{|xi3U_=S^R0$L>NO~7Ej%B2V!+<ERIi{vr^I=Roj^I4Au!ui+Q~(!#S(rU6q0RH
z^0eV*GRlq@U>|%RvG0-!N-LXo-rPTkGLB6!NamoATY5Zg4+Q1K>Vi>_e#yW`4&aFz
zDG3HqHwA3QclParqq`@5D=WP-Om=an6wGsPxYDD1@L6LdTvpL)xu(>w6?xQ2o}!LV
zCi-jHSxW~ga3SYo5rswSA^o-bm@%t6=9n2a-3J^Cr4lNvI`TgR{u$8)6QsG8Qudn*
zHtodsYHM&fW=EGe(#C0%lU<!k-4R|(@Y39!r&XZTKINWKFd(2XuF(semn$b$S<}^x
z8vaW6N#l{#*(<0+Y5NSeKp0ADVimS$BF{~^-hC&R-=O^HCC|PUf90$}b(tL<P6+Je
zxVvAu4FC~pi^H;EL}9#YN!Ion^WF;E8!Q<bd~pv2XH>*-boC4tg@cGYn?9KtFF>^T
zJURox1kc0qnQ0~jJ3xLKWenPAjxG>-%#L?AP!Ij0W!kM}TYxB6_0#@SAiZbFhpiQN
z4e_-XDHEv<{Tha(^^S26_&ltdkn|u6hdcusQoFHc=G_JPfH*mRG7>-suJa+45bvN*
zgw7S~fmLC%^Fj771f=%**ZwB%yxjQ@Fi>-FVhDvU&)exi2##@x9}bkKNI4CikxF8J
zW1Al5lUXh)<FP(c<y=ia{xT7kgJz!PLe@|_4(v(kmJF0wb!gxgq}_y!2P`VB3zk$9
z@Lb86{CQH!(!smF!01DAQo+~b)iV>#&;4VlA$BRF{dymdEtvU+WVZStrkP;;V8d->
zUCJg~LBpXx1!GM*_cj+aDc=@036Z-(R6p#P2yJF^rLv><K}1Gg`_Q2B(y`=5fp@*#
zsVVj&9GAq6uk@Q!=~lFvXry9)4YdSTL1#7iEjjRt5Z|Vmi_IhKP$bDM$%&qD%F^cI
z;ysdRsET~^^n&j+(zUSQ1%$Um0JDeSEfw%xaY`9lVb?a{1n~B~?GW4{JpcnQB<%Z^
zUP@T~sXLMZf%Dl*j290D06*jxz}e)b?OO17q;m?=D{>J4@&y?XXuxa0gMa-i{-c0l
z-V_6Zf5p)q)T%fR7hjGwM+eyX%uifgFn}N-06u!ltF>kJreoL>UwE8V+b2~@HivgV
zBvfJUjf@Qp401o=!{r)a1Lo_Dac8P&P}hT%Lw+#orEGU$V?YsLD^_M<I7hLkKHyql
z7gQtVA!5LV|CUGpFh7WEQHR^xpKLU|^33lswoPb=Z&;YLvU~Q*x=F!#yKMD0(*rB`
zh5C(p@&A9;8FvfaEcP4#f*zUG!^RKDfc5eP#DbvdZ*1~IW;@f%5^#tDap2oX2|l)~
z%C922oWD`^5j%s3QfOxC+%CDlvTFq7dF#Ax-HMY)E?j4GxJh1N_lmk)`n6;FMlh=d
z#e2^(TijJD-)sX|3cU*uAIo=WC?=ZotaA@j8@AW;CXy(KRe}N<e6|g!gC8R@C2CeB
zA8e9JTa^0;rtd~B)je|OR+r$_7m@Q%Fn3zn7s;cS?7Nj$O2;y`1m(~s>+IPm*fwpB
z1s(kXcj%K}HdQ705zd$DWKW?d{(USlae7`fw`qhPHWPCqnJzty<0H`wU-clsmA#*h
zZ*&#OA5_=F=H=|YyOWm+Ic;N5eJk3D0zS_LQJ++2PMT{k*=;&?W)G&J(Ju-6KRyM%
zeI<!r#9vo3E=ET4FqAP7l2IT=K9>Ns35^xcvpbUeNfdZf$>9Q_eg3>uag)fQXl}(j
zFrUC7f+!#T9B6al-?+fqTu7qLeQc7D-W5k+!jf`i3j_hIjl~Ips7MXqZ#R$62#LVw
zvX<jq9#8TKFVIeM5VRnoS&>v=>zFN7t^qZuUafJ`lxAjolL>4MNoh}noEJgZT<glf
zxtk6ad$=%FMG+_jyNQoLDUmqHbUp(erDq^1S!Sui<V?jlZJb=_%9nzjR&X5?twVld
zc3F#Cw<`70wb&gMsN7-LG7)$Ire2v{{o<P<1`_EncH3>2Rh~Y=57iQ_PwtO})R}bY
z6*^_?uS|EXsx@gf7$2n@X-jL_OD(id;~e50XLC$=SZT8Lh*%yGH3|XvIVs50IB8aH
zL9y{k95*pyI8hRA+<f0{P~942T|pT~HKD56jF<?=&BRhGf>xX-Lqj}tIu>l$7<F+X
z++}EMCvFq^C&K&p8heDAgrUXN`>_dsMzd}~(L4%fdajqn1S2BNp8#kJlqPnsx5hoh
zy={yLEfLCWQc0P{m>KM<PUsES+#aL5UgS=g{7f}ZT{Gf=&V%{AZ;P_uS_uTnE|P2E
zA57~IF#q*ul*_yG=;y0p8=POu-{zV*E*i_T+DC!G&tx~d&&{USn$y-+f2`HI>1%JN
zefYA%ol@{34%7N%xJO+#&AcrAzyL7dj4#fXhXpV`K9c_bH{ZXlKF{~jOe<L^{{n$d
zvuEJ1zYle#fSb{?{74F<b^fC&hAQnJHBwCYMc7}(J|v1`s#PV1y!$=_OXf;pr1ar?
zf<_i5x=<0tJER;xqw3mSi9j{MFTMbT59@`nWakgN%N{jELsE+eLgvzIX!z=8E~dWE
z=c}!nVx<Q71<u#<10Z@z<2PwR5vM%BFv)VjY(g48=ziH5%!;ht&+<>PCg`aGA^%&z
znzX+B=QcXyVFuoWI2*rUFm#_HSgbX7YKsK-pIaM=>Yh0ps>^ET&W?0j7kl06dt7Qq
zJYIyJLRhR9y361*+8X9!8HJCI{+8`5V7pMb5<zw3M?V=Fx3%Vi&>~nn?l@*G6cvpS
z6em8jSSLT`z6${6B1u(Nmkx72)HM>NH`G|qvnkv*eFX=`21b^SIJ=eT*TNxd<0^cy
z=aWmhex@e-Hntf((hQ#u8#~5pWq_M3E+ZFiK=mie!WXBNDg94)rZ)RXFX<Kcy(#kR
z-l#vb9*2B4K-s<W-4~SPKb@w7`q7||ATiojoB2B0YOJpx0RFQw0DXZKL5G%Lm@Zm-
zJLEK&T(XLF1^;=6Q{J?1jMnx;$<QgI{dG%EB|IBI57<r7Pj}{fikKs{fyq>N|1&ny
zEtDmAm}0c;638GnF+9Ie_Uk8EgKbi#T4n%kO>J}b4@EH2d)75R<`nYox*3$6!mrPo
z(Wmovkv)^Gf@QA5d?5p9306=lcTX!E#>rDjyWTOnZ##0f7=Ka&tK4#eF52rZ$worU
zMT~(J;W|ijO1-TPJ}U~C=OR7Ivb}hQeTWWInkuz<T^a=*3bioE+Tud+YI&x^Az#iQ
zS1;w{%N5d0j!j@3B;^(5ht0EiIVJ-Gq-)uN{S2IrbWn5f5JXTm<uX{h+Vpon5r@lJ
zig=}2#8td80qOx!KD*Oc#`4L`=TJN?wLV&~ceQ^hX9U_j!6th-W&FSiA|ria-;jfp
z4xVBDXT;OfiGlkeM`r$~XTy1jg&?+eO_Tei8vHO;u>z}7xmspQs+u%>vzgU|Uf@W~
zsbh#rx3hYFn0QZnM4Cp}<YsF9ApMIDFo&})LNujP4R1@sFJu4m<82$evrz@UpnSDv
zSNpS@nZ9Yh=fKPXf4R9Z`Jvxg%2n?Y;%O<9mIX%t^dJ3Zb1eLNWK%*0a(f>zsSFWm
z=)I&CIiiwm4q$p)L%jSlOa`h`2ttZR0msB{Nhm|lkSG?4l$(7gSA#60gQnISXUUQR
zRSyaj{}}wfYbJ)i`$HXUQ#I0EtAb-=LeN97DQn!H)D#Ph5Fy!dNBBwh%n036rv}5t
z4@G(`eM12&=4tmE+)_fp8@(~c*&ZHh{!;1iW44M~A=av43WRO-3=l#Ij%gw=ijG+X
z*`)~%iFIZ~%-@57q<G}C$nOd}&R6hr718x&@kh%Yg{vd@pzzAO0xT%aL`w|o1}?qO
zt<T$0TpaOH@1JLk9TU_>xtEY-J35;gh~+oqGV09lj963nIOQ%hNi+2`XhHP%^;$wn
z4pC#v7Js!0(8bjZRE6;7S$FKJBI@P5W3mr|9XoLp2pMvpP!viEz=q@~<t1{=`;r`r
zNnZywFtQL@-L7^5Yb^-|GxT#u8FQuErNK>0^|9k;#aE)ynw@q6@wDl!w9A#)=g&7(
z-&BS&&!N)%IIG1`{B_xjHcS@k7avJ1q3A6%sp4DnZRtvB<F$T*RNRP$lYB{QJiV<K
zmb^#2Qc&rap10py)yPO@yKc?t-3-!lxP7xfPK0=RE_Kodl>>uuYzCtp<ZX+z!+&!@
zf^lLM8mln+l=n3{%;oW4x=X;k4DWC+oeVRjQv-LemG`ka9O!iS8V;#4CP0=5qI>B^
zRB|X9VOn;``9I<VNbNoYoM{6V;N1L}U<|DBQ$MVto@sFfk13L#tFbE>v2QwEuW2za
zewA1JXb!QAwmPS9<Vk>EQ$$+I@35FB75Y=+bHrSR|3cL<W!oNhxRa<j>MZGf?gU)H
zXfs?Kdl0)6d!A-_dIdqgFkAFBbRqYECrEFT=uW~KCO3)D-_#YlSRt}!A2Lk~i8m;|
zA@BY#VEkVZTJ;~`E_T)lQ|o3K0R@{gBKJ5xK~ag*004hMbY$X!*Tt*%&}t+f-jm@b
zm@o8C?fH!`f)`<tZ*;Xt=l@Wu5KztE;u1Rif<PU2dsoi?6~QX#6Pf>;V0BXTwU`BY
zy%Fv3?-v&T;sHu5OohlVEM11|Ke_={&beRm=Ka3J*ApQ+lfTD6NB>F_`^)iLh%bDX
zvv%fs$l>AiIhP_k_EOX1tL=8v)27GjWv5~TU{C+)=4v?tS+HZbdipz{%*QkG9bnB9
z=3iQv3jDu(up;CvNNmUvj(VTzR=b$W%+t%{@41x{Cb%rf^Fv7hzmwyhm(1cQHHmcB
zWlar<B6&rMjnv8LcymWu5@9l07Vxr^lp}XXRidvk-_kZ<PUY)$>k-?>$$Fpg`^!Y~
zY}=aG`{R;tKr}&+ln9Wz#o6?nAR$@%uPCd#tH`Ug*kw<Q4z;R)*MwC5CeNGEp+!uz
z-qaYwW@^RJ!e+#26tZ#i84o#v#9{urmk-xZw%sXi09`l%+2m&v(=BRx-s&=a#)WfS
zzf(sYYdf~Hvtl)hjFrJdVnBkbsCe>MDmtvx^DM%7q?Os92_6_TNA%=rAC5R2NdXBn
zrYg8RtE=s5swnfXAms$gZAA&Za^&@1jZ`-y9p!<&rZT)HKGu>G{nH{QSEKih&+`PL
zRG_tyVYRS{p3!sO{uTs@lIp3}gV$AW!15OFJGf%8{<TC(hfpDq%*qyQYl5@007QSX
z%U0QD>|~&+=8=^c@e}i_8lgcC-M2+!>muXY0j{qQcHi`;5IK#BQkefN$s23$e=iU4
zjkl^mAP53CBl>Rd7pDTQ_zsu=83IS0U5Egd5AVA=4Ie7M%U2PXo;7S%q!9ri-KW}N
zL;#A)d3=2Qd2<BuE*g9|fb+U9{F^fx{(e?Y-6Uf8hiCqOJnPN>Ep_)aN{_H9hbwCL
z&u1nWs!mfXDSsG|34Gl5<un)!p)~7_512x%MhpQ3VYnI?e&`?YVFGp4CHEm@N6OWI
zbhEns9vAU4aYN=$GN&BOfaKZq*v+&oFZ`XoTN&ra9}{_hy|H&Yk28G&ZQTh70L!ZZ
zZ+`;;e^9+l&;aY4`v1^&--wL=(02djG(sk>q?3373YsxIvC($|;iDwrB<_hafvq<J
z;&B-f_Uk^SOWIMa7iCyxKBU+r=#|mo#VsCnjGUOkci5<dc5+%PNw<fNUftpKwSu7`
z)knLT+oD&VTLF-OLQD919Xe)RRSIPCVYtO;vn*n*XyuS>OP(P4Oc2kEt=-@t>U7Wk
zs=L8pxj>1_`#ry2-#$lrXp;@JfU)yJgTa^p4pr;E=wAwP&t>>nPHx<X0;yJfW5MUQ
zef`1R1M{wk;T%Pgo_H!bQlgm-iyB4g{xEqye$o4NOwx2-r{L+@JzlFgGdgFOsWqzq
z_Nbd_MtS7p{?=b!V?N`IvdL|?>}-lM>1^{h9#m+Ra`KU=OYu#Dz%b4|Gs2vm=Ha-&
z0q2HCt%la>2ZtX3IwA#86&IlLRQj-a6=V09rHtN4P&;Zd>*sPNu`B&@Y-apPVe2rH
zVx@R_sMiod6KV6%P8y-fGymoZej8iAw{Z%VSn`F$wcm2dkZx+)BZtySxWMOU)}Ahy
zBJ1P;d$y%3x~VedLI~^4l?1gith6ZwOCe+-KErEK%7Q|7$+E$Pg4$usy`VHy;IKkg
zG68darKlsTbUG%Q-J&|4mv`G2Mn&9eB_SP8biV?!Jn=bnXqx4_i#kVMDQn>dLFjK{
zA}l@7tN=Yx*um)fin$BufeIo++&K*3?XNrelsgRBQKc&W?e(;Ms#q)w63|^RmMT>H
z!=!Yp&Y|>a?({@_1<q$+kk1F-5Op;Q<HrN*p{Te_j0YW2`7G7)r%m*4ZIzIiS{Kuw
zcAzGHnc$=7hmWiK2&wX@R~X?82qfziYn3CVIm=l?(|(Hd``Y}9nkT`HDt*U4TtJYm
z0d`p@ugRoqi8i=binU^fGwOTa!b$2Dm9-1`i{tS4_o7R49=MF+Mh@`;S8^A4(LF#b
z{a(GnpU+?izSnDaN1hua)pWZi^&PK8{)oY|$)B#IpL~3rW)w7i;#c6DRj-uy=V4vx
zQ0{_MV?vf0wVNtyqKPeHK|U;H47*oxdjiP2#q5;KYTH_mzn$g8uX0U!xH2VGLW{O_
z%#_o(a{ZG*WMKlI_qNhNc5fY?u<WLyT@b1Ho^JU}xW&vxoq}rSzn;0RT=cwg-(TL3
zcoA-Wh7kni&T!6sZwo)d0H~;0;F~Ob3wfgu!y)`n4)K2>(!XuPR1ZYJVm8q~RC~mq
zb})i_cFybT?*aJl{S4Mwp63dvHKJ7SmVT;_p#U`h-ZJ)~0Akgxk&%(DYhm9GZ6e+Q
zcz+1K-EQ$^8GI=oJx9TX`#%8bP5hzT<hc)T)A@#8x$BeL^vK9_mAY_ME(n$1trdv}
zJkCD7VG>%BY{4f?Nmq6$+7H#y!y&?i*eo`XQ1s$_tq&<+wN6ZQQU!Ck198}d_hNex
zKO~h8N#&^(@0ChkqjpWnjLT>JOg5kRJ6Wzc(q|G^n{I_&H2~{-mx(bwhT`x5iD2??
zpQ!+Q&vk3!FutdrZ!`3t>jeM7r5x6r*Lzu@Xg=%98_}9R&a}LJN8kK7qYmhDlkpF5
z&Eld!G8+~gt{l^mNL~){ElI}&I4j092-Kn8AZikPbl>3|J8TO}(%y`)l;4MYtuw^p
zpbNprnPEk%rQZnf@3J5fIs0=CJT=%JW4H9-)DgVdhbG^vw5QZ7P6+c>^{(kR2c2?)
zp4c1MA0}&&8Z^pmXs?~6&`#zoYIyVM!$)SN&^&-m&d%r0mb|<P8(#n_96VyRSYZbv
z1wG!N?(|I5hFK?l`RlLlV&%@fL2)<qo3r)BNNg<yy38YG%H(4@vOLkeQZ2L{COin?
z@cY9+(~6d&&w5v%T|2n7>X}mIcw#;}0Nx-+U37ZStl(mRF1$alNTL~7G58fyjTLJ^
z!O{U_8h+*7k2WdNYRA}FVtu5`q^FWGw*7vhu_iSL#B^g%@{L++!cW%SK+0d%$bfzG
zuE5?eHQQ5kmlPv1=f(-T;ZTG<7#XXJ!-cKU-|cnOHB$TI9?V4Qk$RPdkJEG)2=gln
zay-C>kza2B^%pszX~_vwpqN2~c4rWWml1<wWA#BwSR6hfoq8OiuNfq5gQ^7bCCJ4~
z9?o=SzNDYJ?Cy^9xfxZaOMPkwUdHNc8jEY+a~PZGJvNLIXAzJaTyJU#0Goe$UT;>k
zIt+PvD=Y+hxZjigbO*geRGS~&vGs#)bQ>!Z#aDNe{o5a4G=?e<o`IF<4O9u&`n9%p
ze~J^DSj}R&>8bfkK-A>}GRBjIC!CYU@>Gc@>bT1Zo}CqruSWyHauA;z5!{tbmjig5
zN!v&@v;*9P?s9Irqdo-s`+REB=U;7mHIQqg!0@5b0g^*!<+&EOKdgiMLE9X2&yAEb
zD4)$cKAcpruLsNwIYd}#n{*7L=X14)sr`*J{A#><G`W;OWoD;WfhIuNmVnVlpRPU0
zdQ4P%Um<Rp(?MTuf4b2jU<l{ScE5W!{!jRQbNRP~@!!nke|i6ZF4MfNQgkpPz@0pp
zPZscltGfpdpx^gn`(Rs!(;W8Sv?GxTcK(+!A>bcSpvb$!3(hJrcx<7ovxD_rksL<#
uwL?IC|F+f9gSO?<yJ6}n|0@FgzZ$}nxtl&P?2z-HCMhvF(F$P$zyATid04do

literal 14307
zcmb7rbyOT*)8>$1A-DyH00DxAK@%WofB?aQ4NhQi4ekVj6MTjYPH>08JwUJ-+}$+{
z?!NqX-`(@Q@3(*K_BlQGRGr(m?(M01>UP)D;i}4VcsNuz0002*qr8kd0PqwH06byB
ze)_1XUX;#xQ~*?!G(J2$Je-}KEi5eD+}!r}_w(`b9UUF*(J*gpY+hbooSpCT@rgV<
zTwOgpY@eNJYHB?^96el3@$rMC9=r0%ADo{*Tx_ncu4?e}Rqi|(@bgWaUGs@L_MSg1
zpPjAUJ`Vp^C1tMd@0o1K=)-mqpMJ$jm;G{yNx6~0#ooZfY5&P%my?#IY&At;p|idA
zkdJ(O_YVgMe;1k7`J;!q`-g$O)AN;LK51VR3KctZV^zBHaJPK1Rhz8BceglwH3<tE
zI&gZ&H~IHId-1BFvFYu5)$7R!KJ`qy&La~A;q$W@b>*6!W*aYYzFM)o607Jzf2lp%
zcV|kkH1jqdcIPK1CPbWja>XGz))Jmw`*}?hYR<n9c9IpNE9y$4H!CT|ZjtXkSy&43
z8P@Ea?ag#a|IAE==xJKF9Xv?9hw~{$+G^@HjGjL(_iVNgW!e#(m^v0A_eBx0KUNK2
zJ(rfQyGi|I)%y3m7L=#Svq!^qx%Z%|>&Yhs-lJ7K;$#M=3}$ELP9`g6_wFD=oA!hp
zhaz<^-9`6CGLMsz1Wg+iEraS^lwCmLC!-godB-tE8a+Xu;dWx)L)Wm0{fF%~Nu4oJ
z)Nrv4{H(F4&P2D>lK)toZE3i1kHzq)>*x^v_q}sdONL8*!?uxj?d^aoII_C8)VDOs
zZYAKI#@G6p`l#q&=N~QxmnQ6oOw3(Vt7>M;YTvsK{HxR@Dpdn2&Xym*WhbYDvwNCl
z>A;_vPFns+@X6L3Aq{UokY<-dBQoVn^AG*SpPxPJSNFY!QH_W=`TBNvs+PfzuEnFp
zYt-TDm`{H1YE@^Ba>tMsBo-32pS={@HGkCF`S*P9;SjY}<MG*lv%qrHIV9=y;XD#P
zygu8HQpnts2|}#g9VCr#$P8rE)|w@vRu2|evzrGClSA}GCzsZ48iTKLJcF!Ng&509
zA~&-Az~`Q(d2t;B(^V*o2mzg>mcY->3Vas}7lEL<)xcB_E9GNNsXW&3KKRBY;)F1(
zYXAV?PyZ+*so^n;nzIR0GNI`?+Cm-mQ#wO`w!=7?+-(-(-7LOr*fSzV*_n84L_AFl
z;CqdH-@DXhh$p#{=9v^czK!Rdg}hKmQ8cWFdPlrS-+C)D&=1oU;oWfZ%P4g|SAq#$
zpZgCw4-)?hiJ}4Y;xpyId*`_^0PwC2t&iAoXl4l<GPsXoAH!Q5fZ1n!_NVf}2tZB{
z0${638Q}7G;;}!CRX%5LQnWg##R1%w!9UQms8O}R^36(a#~Cy*V-08ECqdiaJ)INl
zlYzcB#3xvITI`e@`>6%CBRe0fJO9edVf^+~XFr+G3v6wD9ka^2D0eGb(YY;Ngeu9v
zsX8GA^n5HV;mI}FoL_<AX@Adk2@zJI`NMva3O=!P%L0zeb=J8J<Fh{fa@~@DeI6VE
z|2CT4Gfl;t@rld#tUD25eLFt4ZEw8fLuw=Mn7)p{j9H=d5d2K8M-U7X{rW>#kM8D|
z;^3HL=ISJcn3wad&1@Rh_{`0Es{g0lQAb3++?IeBe#NWJw{7nF>R_@vwv=Ezv>|>!
zOvmJPM4sv<yjQ85d<GKy^dlbTbFv|IU-C#584dD5Q#8+u3jgihzHA&gvsjJAliQR=
zR!yyO*yvvVPHn9(TUxjtpJVDeLU1^*-%ZUheEYlhrc=`bE$D3$Ugx%AQx)Q}AVri}
zr}qg?8*dbq_-<bW0ysWZm3{Wj^#iP2Y??D0^&E|cr3LP*1r7&of)MXZwJ_BcRf>U5
z>uAqbBt(mjO%vD>v_KUV&Cp*!N5;7bUOJdznu}4brB7)A>aWMXpEcO1D>P|U&C_jc
zW=${cOBJVUmwb!L;^zo5CSf(ievd}wO(k;nZW%hdvML&@73t0OvH{NM?;{*uaLG$k
zQBJ{e9d0!hs*dstAjv&g_a_m#I@+H^g==f+v$zRGB_deL50CAsW?9%xXEevv)j(fZ
zmx^#ju9oEo-b2U_{Q4tdTfU1<&F$O7&OOP9<7?nSQ048kFcf3m_e<-ACI?=RPrG~o
zSgtN3C1<SsVuo6)U4HCBwEMG_<C`05Eg@nDoo83KV%^VJpOfttR@2#`*aZ1D_2iB;
zl4Vse|4cqxn&9#JB%*(ZnzC5fc>YK1ql3;)G1E`8WtmIl3jnnSCIhVortzaw|EFU=
z`g?^W7%=*H3Krl>yeq){jR`@M$AX}lB^BTT;F0TufL@mW-)pBcfq<P%pkpTcr?K=+
zL*t$<y~}TJTBdnk2~eF7LeOq2SD)+lNgtp8I!7HA>?CW|A=t~;6|O`+-5jS8YG5Ab
z&#FxA$FW0`3T!q^q+}0gHAekun<^;P9J-HHDJ4QStzX^nL}V6KXUu-Nmo)eV^iHV6
z_z4D3>Xlj9riLTZ=}Ppvl=V_dIolTbnJ~ORRl%pD&MM08QxF>NRvcUGuEjaQ8cC7s
z%lX0L-%rI0N)5o8ZY>HTz*XY~Ha+mKXb*5h9V5n7rGh%iYs(AkY-@v=wh1mPuyrBP
z*SFfeY<4E*)?pHuUyCmnzfc#$8)nbKXIH}q2H}$!-#0#7_Z-jmG<KJx^toSiI8#Ub
zHH%nb;ToC!z(7JW<2%y`At)s?&EK)9D-reiHG^Q8>iM!MJ2~0$t2^*gld7gB3Ib`=
zST!puPW-*ZG8uW%`_*pGT@6I-mQefglLoXYcb&>A-oH7ZVrOC>qQg$CehT+{8E%xT
zMn=hDvw`ZxR?~|kzwCK<@$eQoeFA4P8-HgvMs#ko5V;yT+tQ-<sEcqfsf(eXdc{sn
z)3nsM`B`W(ahEW|aBpDx4RM>o3n?)17fAaO6lru>&mFCDm8IOE3a>(VU3A75aJs{+
zT78sz8ONfUug8BJ+Qt6Sq<XC#Re)Qi#BQ&_^0FV;U<~cUu2SNr?1!VJ`B%DgtEy=@
zQ9N`C!n>9DY^u>Y_To8)xxd&bKC@j_i#+{l*~dXBIY7v(NJ{dmG&_uvvTqFQToqeD
zDN;S_&$z~F;+H&1t8>+=r1RQ~7I#Twy&oUlW@cK%hvH^v&1nd3)|#BR=4kWEsYr$R
zl#HIb1V*R`TXy0@KO9-T2Q)hS6w;+qCxb_&l%Qk;M>P1E6xG#y@vpZuQ<tM9sNH(u
z?Y%(3H>{p)!f*MJIl(_qfL(0nML7JDPHBAwGtLh`V^WN{rib8N)VRwi{Xt;<aSX(H
z8U2(L3>uBCG)J0G@};XNn`rZX-Yq+epm@{4_Yotr*tD4)Pj$qGg2CQM?BX*~e0Z&r
zHl#gY`RTTQ^p<!dk&>p5^5^6jmW-_{DaWsW%qgksHT*H4&q7a!Df9g63o5EOT-M<E
zcNZ63Q3PNWiyt(T@GqOZRtl`#$lIshH^xYQ$sq^swkyw;H{CWvLE?NO_WHmP`W+zw
z&uj;DQgM59nXcsKJ(_9sFCgR2&9x>fjpW;$stpVx)N1OV<9CVR4nAu8*v*&@H&8eC
zW_d3VTut>(di_OYhe%7Hf%w4R3y^MJGgx64=R7=LGCm9muZ)oNm4`0G$ov)3?22yS
zAXS%Q`Bq_F`2O#pqj695U=+4zXyAhft?iRaR9GA?5g7&%4JRn96(BQ0l<k^iR#(~-
z1)g*Bb9~e4##Is)Bc3PcUrN^7*?Cl>;(XaqOmI)>k7~Ef2F_w*k6mY?Y6vStElUO2
z*et6}0J^IHXR}#B@hna;Utt<LSgXJ$G&$~7QHy1G2_VX?T9VhEZ4s)IM`1CpyB}ct
zJl{svuRUE^<)Ne!HCMugd4kWM5X$a;k2fmKJqV<5v>>W2sbAI&v8q(s)x$>BxBh*L
zYi`Y^FN0b8O()OiWsqD#1WN7wnKRRTI+5}alKaPbJhU>DHR<k$VXM%GVkdRBUq2eR
z2@X3yk4T&LI&X5C`lo971fbBZhy|#)a-)=ArzF(hheEoSA^CJ3mn%XV0fgsQGSNZ+
znP~T=<HgCe4|SJpe0P~-WAzEI!Df1$ACTAX^JvR<6X0L~TbDf4V4jBlcWuy!|3~{0
zMj%q6>Na1}Db;FNMLH`Z7u*s*!d19o>>LJ0?WcldIO_fVU3KH2y5|n_A||laAyZ5m
z;*RyK>4Fq|itj;ntD)L=l=Mb6&6boU!nLa+mX(|yF3uA7$K*&j3ACL5NxLmb$$Z;;
zchWb6le+2T&Ot7^Kl#ZESi-aeS7e=C6{<E`#Q3N-EAE9tNLQ4$dvz2|f83Jw4+h>x
z@6FL4`s38z8UqmN?tmA>N>x1n1N`Lo8tdt(&v(Ah5|>wy`!nzDFg05!(f6^V9#xO7
zY9w6t_K=j4KAEdCGH*MGb&j*?83rL7^|#mwnzc1+0RvVMvHrY)?v_b|1asHEmFQcI
zuh@lylbu-Apsw@Zh#jbL-uQ~nDq<ngT>u>pM@t_Mwm9~^K=fkRrm^4%HE{z?*F6bw
zG+0QpH#&duzIqBAi-o?tF{a;dn)C~-qW-0)zwhju&ypK6$Je8q7GL9$#bb+8*)s$J
z!XjU)P}|t$IjS}`*r+OPm9kRbSzFTgJY&nD9y=n@qCNLo-V%5!D89#KEIgC!1x6eA
z0~y#>yvq!rCLhVcKRBJ1Ix0mvFDA`=duyD%?k-XT>L#Qm&MjRf+}vQTYxp|NrH!1$
zo6qeXK6!;VP5|Spwsl?!>60$p8DBeQrcs1S{h8w1;p+A$>%^T88ysc~JOs^ta<%mG
zCx`Xbz5WAaE8F~hVPR{zcquZZO(p3RJ>)4G#DmznV$Po`i7ZcAA<Bk?l4qis3Q6Hh
zM1+jcWor<{fepZ*elt>w1srcjDtXe`z9h!zM~fTt{v|?yZp??WpmSME=trVBP~J?}
z$%pe@zs%Rqe^>a|<SoTL7kE{4o^Nj~3nJmBIo$pG3!9;YE~FRp*#K}#))v~=fJS}l
z_SdhvrI}|?uR^3Q5Vu6PHkT4KS@VG?5YkDwz~FSm8R`0Ts+&s-RsUw0Z1DX8f%-Id
z*Bu^OonUYLRU^M)pxrLKmVaMSrH$Y`p6o>k?Z52MVusnp*1JLbTH0&kkdicAl<yx+
z{d`WH(<Ui}&Xd-m5}VuYyTWxAOK=FT)Z;6^cFf4Yi(u3ukmhQYTt$FbA|V&(Q@Qz~
zS}fd1QjCjlX^r$b6_Sng>#v4Zb7f2LLGK`i)`g*|9l5d8eAGN<*RHN=%$rb7dYe!p
z_FBo+S&*!FJN2K+8(w_W_?_x%2j-4nYy$Hu5c?bsC&;=F*!*4ViLUGQoSvO3X{8^<
zDiw2CI+&DDsORTEer^$J><*wl&2We3rsfqM!P%uR&vV42I5(_IRv5ah-WvqViUaux
zvPI~K<2vCHCk+83INZc@<`}X-n}3A0FQhy0h?BwG;Ud%Ia@&5wg#5@<ii=tm=AUo+
z%Q9|Ouono>_xw~*p}0nn+N~`=VX2F!l1=oadS7-%7jpmkQxb(6n{Gzn;<I;|(|Di!
z${j^q#Tn-mXci{!Dui+W(PAV2P|XusXY9+F>4j&DG|}(j2f2x}xk=4s!c{Jq;H7^?
z3>F9?J2MvuW-H(yI}z|3wVo62-|rV<dTWX0gj~fr2_7%FchH*_DGA*r-lXNoe(ZWz
z!TZ%g2*$Ct$(8@V#7Z&!n}Ja{U8O`EVSGREIvoA<9vm+f50d4|zF18peaynx13)X2
zDNa-HIPh`RVPaKEM?F=>v}0FBh2RGy95(T?Kdo@g68e_^7rb91*e~aRs=BqGnUG-a
zXfYLy?!<^yq-ARD<<A6>0|Zy(>d_CV%hsj3n6OYO^qB)M`cO~FeT>0fQd}nk3sOn4
zMg0BECXbW;;h&FM4oxnem?Xus1(u#A4HqwSX6FoCFV26bu4gT;SL>4Zz7dHpPf7pn
zA*=I1rr?BbX2Qt%oLR^jR;5u+Vg~~!f^;#-Y&H!F!Da4A5(|Xpne^1%FihT-yWcos
z0jq#sv$Q%T64HFEfYllrxROrS>56*U1rFBdg_{28U|l00hgbD`ugyE?PMfznnO08~
zW#1S=*gv_^R4UdjzpZv!MA9`N4__zuB%e&f%gOuioCM&U6s2Squrl_b9#-bNp`%+l
zp}@k3AuENGk@XW%BWz{mNHCgZi4N>{6xiZTdS^S1J@B0+%FPT;H|&XYk=7-_wScW|
zzNe81{$9oB#lxwxZL=+oczDM~cdU!j>lH0Mi2maq!P@F}-W4mgG7uavGWfx5U~oZD
zj7PhBb6}}y)zAz!X?k-yv)znKK>0zJe7fT5vJ4Z#BEDnmDT&@B%v#xu52kEXT*iyO
zJ|Zx%1BqjvBvXMY*9a!JcgQC{D?hmGX1t*ctKjrlJ#>S@Fkd3or8<H&eJ32}n!JEt
zP(s55J6i<pt(|D4oah6T^p-FG(vceQp7hGA)h&NMCNnRGS<vy~q|`=M|J%+(_=?uy
zVENf6D70%QvdydlOlsxSz*$v|i+Q8$u||91NhuQDe!3C{PTnvq47aw-eWI5q-1htF
z09RhWFI1=7{#`qa%T&m2>ksJPw*>pQ?hdTU0<JW?jBfm5U2i=8$j6bV3G>)g;U-$N
znkc-j0)G~2YHBi)hU4;9S<4)_fogH5n+1@WPxNY|bV3%cGeF(@gRl?Pfj1Z0`D9jU
zOTZFP!UhVf^VBN6{f)<Y5F=NK9;W+62Le5Li=r<_mK1KgCKce<az~nddo=Yr;3agY
zRXcmCUB^-~@)-F&IGOL^%s58c969_pc?S4n`sBYS;8dSKVFEVS7fSCdafjAFGh^2F
zTmL*$c3F}yOUjy0KC)a*G4k5ClD%>jHL)(RQe7JG+WV6;u)%W`FzHS_%D!&Ef5Y4C
zdfr)mO{h>Z-)Y2CZvm^Y!b`rZ-+CK%>_Il*m--XyU8QqW$7;y>-I-OV5(dNqxvQ9x
z(f5(NT-n<w##}sg)mN3VTtVc<)q{1N82at&i+#U>qmmZ!ZQ`HLGH%X~L4nH8(^D}V
z`PbFxB?y}1HC|8g_3Nmv=JlFl5_|n@CZVy^_Fnhfs7xEng58#jZE~Bo$?{#L43n&u
zCy{BN6yER&@;lt7ZE^g_6z<s)O>cj(apDs{W@A265s$JPFrb_}rKs8dgHpE?Z29CJ
zRn3J`pY8_!yz6_lhuxbd$CV9jAQglw39E=)jRSFfU*z*=Lf@iA+k;A|>jmvk$6Rw_
zvh`RlFsEddm&O=<X%>K>l4jO=Zn2HZviD|gn2(C<XNwYXqWBJ!!Hk{-VA2ST$ALb4
zRYkIqKL1Bt&xES6k^dw1NpfYYa{eR62kFXY=`4FeM?%Ox91Ays;W`HubY+S^h3uY-
zJX(j%&wbiN)O(J(x60GY)uHIm2brtG*`n4mkO}fE%1YT*N;a!h1LMi0U0j$+mIeh3
z@-oz2d&U*)VOV5T1vre{nX*KQG#dJj-x$}K1?y=_wzEh{#{-4Vm?ON-ym`WFcnc%4
z2UC37I+=O+9Ju#*PBv4LE@JQsQ3uD8(%agp$EGew%MVX<<0SlgfkDzx7y#bC91CU%
z;N%lFgFRzF;^LPs+dV<#phvHcnEAZ+vDuYW$}tXPO1@)fnd>%^_E#$TDsxuD8LJuh
z1IunWpuEcDX&$T;<2PV&jw)bPdtws(G}pw2R5;YZq-PKDQnO`$W1+F??-`++#+BkS
zm3VRcnCR;t9q5U*xF@zfmcts5_MDj_(h!y(&LI7s=w_c~k2CH&w>+%tY^TaWsN}Lt
z%0{2&nMr#(CD(~lWbLyi)cs`T@Qa&|uK0BphR88@u@`l+yaIVp_xA%ox>{FE>!elP
z85j2(N8seR1MYnaw7YUBed#_0V)a=dQJtsx2?cd)BqteHKuA8)^EW^L<oLZQQl`7~
zm?n)F^i8$L`H7^bBR>bJl~nMxdQ!{GPO$!O$_TRHBesMCirSD2RgwH4T9Oe|=;~NQ
z4CX`r-lu1&pyGabFjFEi8~vTmrb;8)M^>3glIX**!8qRe--es@AiIH_B{;s2wlU?L
z3*pUzNngr5oU&&oFI14OCMOA8FnIeOx{@DiAF7}`Ieb<53c<Cz&Uog5200$=G@Ra1
zIM1pm;m+S4>@l|VB5^3yo~+PBa4R%JVveD&Ig*qP?7n4&(;AdZuH5aJ3>C*OBrwTI
z4XaL&-=M<SIT99jGMOTn29;3h&Vn&`;{(9?^Ge($p=uTdPzdp!@Ra$QG%|bO<#Lq6
z&b^Nqv+{8>Upv=aK#<C_?N1JJz(vRJqjMYC^vKCi*uFL!(w$^<c+=1uk6)cXL=JT~
z$_vlZdQ3yHdN9bN`hHO4_m^}D3NDv}@<i+R@yw>66}T5eSmJF_?XU|*?bsOHx}vM+
z#+S;}bx{_ZREZyoGK;6-6SJ_oD9(f!zjCmlsupmU+;+iFXc&^P+t+nHq1z6O+iews
z5lYPr(Zkfzh@r1{DX`q{1&}f%d=ApDIK89kZGW2zGaZ<(jdgWJ(X)FA*c&(;+Q5cO
ziVwnwcDCOQQv9IU$ZiG?Cn^C%=UO}=mQFsAqV;DY^BbHZ`bHJDdN;qnJq<<sb>ItA
zHRTp6P}`_x@vH84SK2UMzuBMaDA!n_@e~rb*iWs&i1INCNp>aE6vx@w)3}R`@#8Ee
z5$OIMXo!5mp{`#}&vo(%@pvD8#XG`cC~SzmrYOEmQS8dBw3rm9O!29%ySP~`!dX4_
zzH1Sv9eJmXd7`OB(ckEX)Y%jj@Ou5wGu0p0)bAV8lZE&^d;QSdkNHVcd)Av<Yp;D$
z7%z4uF4}x*frq3yfIkcTjbZ<{jb7HZiFk(XJ%!8r@otCcx_pPm%ImDH6Cict#H?*Y
zO4sNYfMPI1NL0bn8VD*~C(yfgaEd);&f!sAyya`b=#J0z5NIKZ{5pgzLmW3pr7I}*
z4{iEYd@8T@dH$@q++r{aOXgL^+onqPuj>HQ=z@ZE_Y?mwP*hQEo`<%@d?&}1*Wew1
z!n5+j!u2>Z-@!3Sa9Ojj90vz_^o)_9!QZ-g?jEzLmE!p-jiuH~E_Wn#!=~RbE=M@;
zh^Va3azF27O}P1UZ3Y&&QYE&OF?%nN#F!D;{|sdQ<X<qAtw&A`fD8b14dncy>|tFP
zkRQQX5ef9!dV*}`lsFKi>hn@B>nBn6l%>mf-%lwPfq}7f@F<1|1|+uByIx=*TOcPB
z_Rgd93;R)>6U50B#`vg?#s8;Mfhf{4gzTl5Y@MxYzkrSW1`zq7D)>{t2v=1-CT@`#
zX8m>&&p+7tMVkGl$!D1Gnk@g8VgaZd7of!)OaI?6^k4Jl@JI>x3dL&+@mKGkx^ew0
z%Z2gn9{O5^oa;MI@F<&-M1xtb@uw4!xc$fMPyb57eHci@nKc-Kv(aRh8-2#)R+eS<
z>e*{M*w@VN0Vc#7(6x9FG(0g!p$_-+6N^dqD%%z8Ty%t-6*2_tL%jNO{H8^;&6%EY
zw^OZ0w4hJ00EXW!pNd&@%EV+nmT+hi?-tomh;=Q0k{>D+uhsi((t}IiO{Nbno+ETL
z=>^8q%SHYAqj$w-H*>k@UBW_c#J|_SsdS?m^_<2i?MeD%bYTE&Bvj}*nbT5J@!9JU
z3q<XC`Z8<GF~dw6zTxm7Dnk~!V>(0H-RIFsq~MSg1B7M?cygck%)MrV`YsS3T&i7O
z8QSu5r~k+pTz^r8=TUO-Lr+88t#<A<x<I}!IJ{4niubS0jP+i>o-DLa?#hPJep)c5
zLJ1_)&Vl-C10y8#$xo$DY5C1XBBs5Pm;4f62$Z+>Um_gkOR2IjCvdU_ec4pbRqV9I
z`TU@hX$1b3B4dBF7f4%7AAIUo<k$On&z2=WJ^2%R*hI93cx7lZn%E3p#gkYL(#9pa
zdf)ugnPX?_8-{5?cY*&aBc1;li5JbBqa!GVwk{%5l5&!WzgSQz-<$V%t(CaqRwWU2
zOB6M_Bo~e<mDiP|JrR1whG!sc6~{<8^2mC5LNLXCBF2;JBugqQf0QM&t$}{f<H_wW
zAVFwl{Da0*!;ypfUZ|2o$%9rOeV;{mb#O)ekd~M+P4$tV*xa~Xr$VctO76ri3o<f~
zQi}||4s}7q)P0}n86{B_94=m6(hhRNmV?HDzvDy0Vc4(P@IbOqqEqCHLI$MLqZ>!!
z1t81;rDaUwvUUBdfrxG?j;_9#5iV-dD%@ZMxFA?dOaM)`E=ubVQ_2h1fl^YQ03=KJ
zAr_4OlGcn;l2r2XrH{*_q=x>e9+iapg14eSF@%}0;UH{;RWXV_+zGka{?-3B12gLE
zjKjc!w7Y6x{@;Jo3wRzbLp?zww7*mLbtVk?e#RScoR{ySnQN5OR`R`g!{pp!lQJ_r
zfBjF<{l6Z|LxYr6Z1^&0my!`Zl20ldW~K0z%i6kMxt;gX*Kb;X=0yL<P$*SVIVXkP
zZLaqj`<FPwJ>YE_e@=rGb-PqF*Zu&rp|MjW$N4_InSj#5(9oPpd{Gz^<{$cAEWL-s
zm|_#68iR%-bslA6h$Ys6<BT6qFs_m*ZHGqhLlCAM;9UP{NQ&BUbLiwX)6)=7#i&D}
zdmQE%d&d3_+TWc780Z<JZ<{6N8tn$?H!lW0PHc~%Q*3e(84t)lZU3E$U0HN|=FYWd
z$8mxwS$D~)H0N)y-0y8ZgY8S;!YVK%)v!;JtoZtuUlelxrQ=ZLLTv$lG+`C{qNF=b
zlF{ikW2VGSVxz@DGC`o$cM29j23Kl0gwSrr<!S#owsWI&iKB(d_IDhbKQg!CrAkcg
z`rn!W7dS4%siNgzCb;c7d1n}=7>5!4r~zmeN7_ZCp7Nb#XTCAk^_Ox@r$(V>pXzfe
zm(g78g6OuE7`we>m4Xpl-?nK>g7#6z{>XQZBp(EaBlsHsk{{sPOGSq4N)xhw`IU`?
zL|ZY3Y4GkBD7bLBa3Cwp@K;Z>?d<p~O%sm`iSXo!vgfbf@~Jc<9ff`xtpRCeUDPyQ
zip{zPrO_CjJbU@dhuch+gZnRSw&-0!2PN)&I8ChoFkN|o2^F@&fP{R=W>Izj0#(RJ
zIVVYmYvBMkMg!)bmw9ES@vn0QVp`?O+5KwzFU+1CVPLen54^w~b@W7Y{T7IVf?I@1
z5_%<hfJ1tVZ-4{nx#sFXT*#J)EWCu86av0aUT*?O-U#K|#eq{0$y&|XU$wd%bT;>K
ze+|B9rWZHK<Ng|SXjpBVm-w_^yekU}cT9p@y()&B7-PPo#A0S+c_g>udk4qY-&)zK
zg(cy~#(qDhpdZD@*KSE*3gcIQUcU4-kC>Jr<^i2smaS(tX{^_YhgdPBk(RF`PL`0p
ziO+>MV;qRg(}lw*(uoVvx$}watI@ba@%P1uV4kkGWzFsw8(CfYKW<B%`ZEWy*uU&%
zL2S}oOy@J$NRkrZR$aA+BSGAtNxZL~*GFAkmcL}^S>G&JmmXnI<h?x&DmF8G1-e7^
zV2`9$t61JgRT8pZ+54eMxex1e_lJAo&YuKUKPmEX^<m$4CHz?B7sUrGb64O^l}=OH
zegAUQHc@iSJ5$cA!SX$3SPHfi%e&fo8ue#B{y4)t{JOF~zJ5s53b7(A%|7Iu?|8uR
z%th<ur@0fkEf<YyqgD5-h%+&a<GJm}>U@LSdHaO`AN`+54!AX<mel;W!({5-AO*V<
z5WV%uN2)#u*RBJSj$In&R`AX$CO%MH*TwRsYHR>z38qk%`pZbET?TI2cW>oGxt-M5
zf|X0rT#4Exb0Yn}1r-~#Tx?kO8X#%E1=0D4E6!C9UFP;V6953ecRnAciUzm{8@9)k
zXlMXN)j7$&(#4@!v3tFhLz7QPKym=A5T7ZBV-NxTSwM#G|JKMhLOyYZ^#%(u{hHzN
z(}bu`-nho)`f8(56u==dN#K1GzRSK@+ur^2F_lorMokSt_Ib5>X$a??-?6*LUr!Ho
zi9kfj(3k1rjb%vrxxvGRqI6^7Y<8iImVw*p!Di^(2oazXVnF(K?)Fy-ELzLFRiD0L
zjorwu+LwRS&Wi(2><L2|oI~hCEn?E<$do!pOVH_T<1&YnJzE1`_Ex+m)!Ifj1`t+g
zYv?1w!q>E51R8S^yjWjs4?^I9Pi0Zc(&VL$bZCU1XE>e#=Ljixi<S6qT5VdW8%uq%
z;tc8pvgH8GMW{Vqh^AlvV0y!b|BHJgbJ<!%jkpw8qQgB((#%cdxS{`NVh`f5s3mTH
z$dzc3_sPdv`se01l}@+kormsGTY<D|BONk>^r9ptH)za@>klF(2@X20pwjd7BpA_W
z$`l{t*!<DW^X#gHYd4*lhcbTWn-K%SI(ktECT3H`EZ>&;h)6dt7~MU}nO57$1j8|b
zAQfK&^Sdf#?A!lnLLN{Hm3RSfa6d?5DH6dLsyrUL2)V|&85iy-4Q)=cfWpy;WofV@
z4MegI$)jvpn~gbK_yy3yXtDQvyVAx?rc5A!&Ia5(+CPi=UlR-f;8^5hfHjLkgf(vg
z8v8}EWiZps8<z0~Y8^<25koI7)hG$PFGxnsXnLIH9d<d+L@SJk=ma$|UkSKdM&l!2
zeefS3$B-htw_3M#!6pw5L?@}1{En0+atXutVv`Iad<T=-ltkbY5rw5SyI?nJO!>8C
zsMvWc5d96OyKBabsoz_9+v4drbOFQHf2`vFQ;DDYU&a03uJ^uRR)TR9Lja!H1cH&b
z_VQkqLa6+-7oqFy^kiDX7UC)%{q?g&1k{Q^x4~2*><)r(bLJ_X*%)EIao%G=t2Y3F
zXPvc9{SIQ@oRS=06+bk?D;YRwC(xI_f+4{$8+ATlRW}^LR@K)$C@M<4uru(U>WeZ@
zGW<JpzR{9HSjHE7zr^*tDHx3Hg_nGokD%}!RDS=Jp~6tIhW}5PuiOO%XBqNulg{W|
z7@SGc{x3mazK<gPU8sB`rP<Od^op{wDG4xNngg6@uh4F4=agpC%Ig;gp1N!??rrXH
zyhym_9HHy1dqcM#;$dDAk)I@6bZQrztCroMYdE}<<`91ms?oN$+D{3SOR?U_?~#Hv
zaBI$-{{CGsG9^9B#obJ~>Yb;b{<k4m_}R=ma_|1oYVs?R-wO&HB3&Kd5lRO&>}%Zt
z3`49p#Z+cqE2-1_c>B0%%`8u+^(F{ksP(aTaxP}3og6SVmI)z}lBHMEl2(Dex}Qra
zN-49ky*m0AN2oUxI}MNYGp1{ThhmxC!Q4^DneH!6K>EqwkPt1mw9<ZiK8HH?Dss0X
zc-rDBsQfLnA3+?pl184Oi|D&;&DVXF>fe52jijSSIEpoR$T$)@L}`evcB&ZH^p4G@
z7zN{%VuE=@WIQ;dpDbIqX}Og*!$W(%Gj$=VD_@c)OZQ}348Rfim>VP}tqhE2xehgm
zbGN<>U0S`Da~uhZBYIJY-*+euSq@-J-ECuu4(Mn8oRCU*SEdd|nLcP>l=pS@SMZni
z8Cf{+p~_TUUsDVI#cZjA)x3nF{SJ{wpFeTB=GJ%GX$aHDJ?AW^7>fL=RiI>G{+dH8
zuVwk!XRXO!b#dytpjzcLZg*bOYqb4~UFL^kyLg9UrGeKSjNjz@-4nOXCmh(bi>;&M
zp3QJH9ah2DzOt(?Y1-`bVoLw2;Oat@d~7YkP4Lmd(rV+lTfrCi`G~j;8yTi!++$aX
zA-MXH+`PTh!|Y#H<WPBi9OhJbxA2s=%wOv4OT73Sx)On;3;6khY0isdSVL#)lpG`P
zer)!wF0a*k$8QgKgKo-ofw+$p^rD^vZ#U$QL@OaTqaxx@$xe^})zQK$gc6mPiF1_A
zr%#LFer~!85c8J053MiBJ}ZhvC{fLiAC1oL7R}S;qO_$QG=hFSzbV!!6nHHXZ`KlP
z)T$<C3H9qx7^+!Q!rW)&xFLxfOFEn}0`#yKYw%r2rrHBii#YX?_I+$G-#kTkECISo
zLXaeYTMK^<yo!;>XnrK~EdPt9Q~xjaPT3=|C$;w3QrENP3W*PafZrd(ah?DyG(Ff!
zzI_5LM@a;Vl98pC>OBLbR=O7FOC2wN!=j58`*+~_PiY<z^icM6`KLq&6XTrvN5FC-
zf-5}&o$;xoI{3*$JN0i9G#_e=fb7FG9zdG}81T;|{-Mn6{_i>ZYr(X7n$srNNl$XE
zpG*g6m2wiB_8Ik#;k@rJsd+5fykHOd(WeEvSYs1%&9j7^XF^0Uxudbp{1W^ergS+T
zS(&2<6c)Np8-jf-z3SJ6x7CdO-S_Fqo?AP74J|*3fU}|!_#5X>)*-O7N7UK&aM>z$
zE~y*ov-zqRH^RKz;PVlz`~2jOW#ZTsG9lz%<81t{&nw~lecz5n5zvma?>^e&C}6Bm
z#=TgS;kWzwK5I^&aP(Sz8;bgW!g6<aq9nDTm~aoiGkIhLFognLU4x|%kDx=wia}D!
zpP^&pCDA(>EHSghZ|La?Oh<(Aq1ex4R19&xGZaZDwYDwMEu?}XJ@BiBI0_2AZ^gbH
z+=?w(Ot?YxKm%&Oh*I2AgawyE5LD6|t^3aK^3Y?8y>IFJl<speFvv#-IlLC|Mx`2k
zhUk6Zt@6wL`n||Cx*npJ9hl-@6G~7)Ba0_dRLz@#&9R$bS@8QX&}s|k5y@d<+OQp8
zYW6Yqz1Jfg93(<uuzQBb#@Gc6@;Y>Z%y2&jvsqN&bxfF9uCGp*u8KBjP1Ensh%3ZN
zdn9al-2(VxoIWkve6L!c&^QHAieiDm7MNx)nh?N_kfnw!c$M<|fMkq1a}_5eeOhrB
z+@8xgPlHLBAp>shfb$JbVGvZ9`33rPo}ji{TL6MM+v!VN0jETgCb&oA``;^874!``
z4e&(b$`H5kmq=e`4a_;6`{<D<rGq&a189>e%`fPhNKe!gjy1VQR#c3l^9)A*5+1Qh
z5BA)2$WG5|fxG(rJo+m>*>WFSE%KAzqYsxC5U_hx()?4wDxfP@oyok^eXt>;lDIMT
zoM_)et!KPZ?tYR$pReI+QBl-Wg~i+@t!_R7=W8{xzbde?7#+Dp=TZF`3{CLQ^-j0i
z4TBw&jkx{K^1Fh!8<x~DScR{;+y_S2oYGncU58YW-#=IEt7Y$Y1d19<rQq!FY^^i5
z<@0gh;KnAe&t0XVSs(1spj1}M_;1CJjZDs@T9$XF%1xVr^3if=zmUJqnclLr)t}8M
zFh<DsH;tM(K(P3IoGe(F+zm??Aas$RZ*Db5Rf?KMy>8-3$}0jNu<kdXOVDXxsy-4?
z|KD1-<%d=Dil4<f%76gKUj=l4PH~QlcA5<qK&MlaRY6ff;Q~bjF#aLtEezC(puhY^
z554=((EoRtO)K|u$`!!5R>ZN<>vlj4Foqa3DJuRYqzr(#O{^FlVpjZo{_qF|xrhF(
z0NX$WAmS01dLQB7|FqEp5P^^|?nrUPLG-?ZC{K1<-B%;;(JE-~vf%4FsRe#c%D1^2
zO&FTwndj+xzswKAKXDG-g26wsPXy6a(T~}ijydJe>`Hgz@SHHyh62x-R(UXR3YEN<
zg&8JM4@iV{v>PN`d@Rq8LgT?w89>^PKvYlVY){!q%{WAPTqRYE*s6igP06*P!g3LJ
zEdaKk!4Qlvgr0?&IhIg0@$NDrKHI*iZ6D&$8oNZxsNXjx{8raTYMz;Dr3HLbk|3^s
z)-kMSz8=?CDf#RCRer$^!wo{NXH?CQ0iF>yh+w6|Cyo>A!qT3eWG}w9RlV^d($)&s
zW-so0tDGL<w+Ag?Im`3JCmN9%|DCTYp7HaU`n#IRBSJN#%)+=koZxxD6Oaz<_6uF5
zRb6<&JZnycni>nRx+&35NdDq2>~nPOpr-X<5wgfa)>EOGmhaSt!=Y8g!(zrmQ%p@u
z>(iK6nv<`VCBcvrRb*Vg{f)Q9myp)MXHXB4ug#+Sz9QBo+FE3zbBkTHRZ@j}6+CYE
z@kU6x{IoomExzofsk?52)=~C9SV2SIR9KG2rN=n3>3ki$HY!Jq)7_>saU6~l3eFD8
zpJcuv*P4_;H9T&FkU6d~C?3}S@+qPeOVROzF|Q5xMl&!!u?W?VULNQBX!YIwh~BNs
zG?+o2AT!5v{gY?+g#NzHGy|~bl}xuf2XhssQJ#|=O???D3Cn@G*7wXbHA^tDGTBm)
zoeE4u_DnEi)DvIheuOZyK*jRKz^<4f)c+}s|Jut{X#ma_Fj%OsAGLU`VUR+_ycpc4
zbP7sBx@!Bi7dg177<r`ty_;3}89&4OGBIs$|C}^Tf#)TIL{s_HUf!euL%G=3BJ>Q(
zHfmVW!-B|=&NFKs0u5)1sA?EQV8r=-DS*CsP&ySGf7&b^7M-9S{&v%@hHiE230v$0
zWMQaOX>ZqWcnVk=PkPr}BwILnGp-F6gXqmX^&2Zu@_r}JnGY}eJR<+I?aEG#@&E+6
zR|1E>{F;;0Odb8asnEC5)%RNM6Qy4jywlz<URO*HRE6zM$o@2vtksmCY0WQZ-JYz|
z&U%vlidZ4Swy5N*DHt+?c*5^%vX#iH2fNm^Kd3D!@!6<fC!^G+>nZdX_D(JG6Od0>
zJVUTdpgE^-d`1!}#(0NSn4)u#y}uLA&7;47kH!4;JKJ4Zp227!(xBqIY1mKGLs48M
z_m)Im_!l<QG97HQXyWPZQ}{A+IE5_H<7;_T^;jV@);$cx9kf`6Bxm$tZ|@lHn2jJr
zdGy8&9jsSNUs<ju@FCD<)HCX*hA*oZzK*})U=81B)sahjd)HeSmC?@>o0IBSU|tQ_
z(C9vJKJ2M0QaV+yDna7@>f`N)xbY{J()Vy35sNbKtbNMS7n>y7;1-x7hgeCQ|AWZl
z6{lsVv@z-XD}}z}tW6)6C}M;X`#!d&mK(VjeZ1ewOkIzm8hO6hKctz}&n)h1l9iO#
zgGDQQ#bx(5``)tSk@R^Y{#N(d+s&>1peBm~a0jR};g)*MzN(Yu{<e9v73DM^pS_1!
zzOy!XR-0r<q&l>t@@moJItkGz_oM9jbDt&SuZ#y}rzL<2jg|5WFO${|#Q4C}hrjsM
z%^b9>V!p^`j)9Bh4HxNF4hSq|I@Eh(s7J;Yath^@UhG`G8QV#Fc|7GpI1-?cKRJ*E
zJc|vx9qe!a+DQDPS8rBh324F6p{Rb-%=Ng}ES5F@<ix5x7T5pwaE3pwIa!oTT-!$e
zhyEcB$K}l8U63GA7aGkF-+e?&5Os3MbM|1JakJsDAgVtB{sj+~S<9nEj}?s|oeH*k
zHhGM=TKvFe27Fi5N>`%P+o$AMM6@z8gsJ+ZskDuk-c$Ual#f(40ts3(f|&gG#TJh@
zymDJ^RHhNh*3K+#f1Gan-qTE=5T`=%-cV6|M|Ee8le{f6cAZt>`yq^IYkM#g##=bJ
zZw(;iB=>|N1OeDmr6iOF2RIkthGhYNKR$5(QKei?1&c<K$m{%Mrv`jv&&Lfg%N^y~
zaHI?{`wmV}&K)f+#gqW*>IncL=BB2Kw*UFM_Fs8OJGY?;KlW!u>SV80H3i_K6(~*_
z5{@gct_TzHc>3(QCly94Tl&?L+kpQz3x5!7YOeKU3X4?cIImjqR2a}!T2lYf?2L+J
zCY6YCDA_`Ks-MR<bpp|x;;Vsq+YBGO5K@?04hWytk*OVOYkqkjoU}<$lF&@|{<<8}
zrHw9JoCQ39>CApM-n1i}nKBuVHvP50Ujx640LTew)Eb(d{VoZ+oO{7;lVQ_D?z=H$
zC6s$L*}U^~m7oQmzg3QMr?I6g-MKYF+zDO@Uo{ZhpZ{RmEt9uT6PvXK5;I=cQf}m)
ze*E%jR(XtT6m}-N*DC04_dx{<k$;1Fv@xNrHJi(ezWA!p;D;4B<7--ScRL~+wAbx@
zJTrOnp3lYFJa@@KL2NImr)_p@VFO}EZ~k>_Xk2@9;gD5%V14%dr?xxrzCnej^+UKa
z;!g_4SFz$G-}}r~ghi-@ol!aJ(VpvvMq-|ReC<brLKE7PTz@4KVg7De@Va0-y{MTc
zUUBkQqtp}GE8B86Eo$Bolp{>CIOmB|X*;$Bw#SB{ewIrP18Kt`L5p`)jVU<QvnMwF
zKexBPd|-`i4Lh%Cc%ikvw4WetSZ+YHoOg}r&PkrpjPt1hwzbWP7dzFYQ5uw>3fjbM
znm3vVw0`mRpRPrha$NX6>qIuxNTM1`k|(upe(d)=EuZqI3t2I{8T`9__O+l4K0EKq
z;Z<*4|3w?Uey0afCZtu{HqA$u=+(W%rDg8slTg;)`R!h!uH5wHtMKicMns4ME#ZU?
yNRrX7E!t4C)Uon$h)9lP95)|w_Mj%HFIFwa$-M+s+WbGB`0<0XOqrB%;Qs(Z>o7k6

diff --git a/modules/web-console/frontend/public/images/query-chart.png b/modules/web-console/frontend/public/images/query-chart.png
index c6e4cce08d0f008993529ac1f040cb563aad2c2a..1b7ef41b778b88b3e66cfbe5bd35044af5df3f47 100644
GIT binary patch
literal 17142
zcmch<1yGzpur3OL07-!0uxNnbzF2|=cXv&Y1%kU<2pR~!OK^8z+%4EH?u!Nq?jH1#
zocqps_r6p0Ue&9*wbg&k^mI>mf88@H|34d{sw{)`lJq4K5)zi2tfV>;5()?j=@~5s
z$`hw{St9SL6RoPKDW#L#KQuJ-$jz-9-+6Iy5z#Qq&CN~aQG0oLX`Tt++uv)L++5f>
zHB1{gIRE2ZJa%$*+rPB$T``gVbGdDHw`YFWuWB-{|JTjKL(TZ_@6Cv0B&743Tig7R
z@%7{1M^{0$)8b;{Ngazrzm8V-&!>MMht|!k>|HcZZ#_PaSmh3FZf%Y{KB7EBEgW3k
zKE8}^n?InSD<58eJn!J<wm3UGe!O2V9a_7GTRc8q!>rgI;3<`rRZ|PAkB>)Ac>@;Q
z+#C0gofnVyn?<Q9X+I~Ih-vH|xh*Ef#yPCuf#vYzp-05r+~ei=r_`VD6wt)gW4-)q
zZcs89C**3o%Y5jzYVR?(bNU53D}To3>BgMV8yq1_jFIRMA?(;G{BPdAz}DyGzPIB`
zx8mpGmD-(#nzkO^P5C_}TRD9IUbSdRadST`fvfvxamm!aPF>nqTOF2X6KekC<~De=
z7<!K{!4!&sq_iYek39}O4!3ojiKbA|R0BorFVB_+ckj3it4@ogns?3*sARB#F)P{X
zzb`NHH8~;ck7t5(JX~p7C7Xdi4~_>14}09C>(7oR_pya@=a<*U6@m`<9mcYA?^1^f
zCQnU;O;-{)>M{~IWWGeG@{A0mZb3iZTt2b|G^T>gtEj#_SXn41B93y_!TjaoL8D-u
z@usgj$wNbzYY!DG2cchrIaL#QlZJ&q+DbduO<tT0A#u;t1=)QP(+cVDv&+5IWwtTQ
z82oI6N5y~Ib&{22ZISZh&}-nrF>Q#O`y*2k=u0`RSC7)?ui3#1mYIDv-T~!>yX|cs
z^BtQvR#SfNb@#U|Rg9E<PMp3yzN*@oJv==8wYqkHfB*RS`1|+orKP2-n>&S@K4&DP
z543WUVw#@Drj70wr(m*L<lIjLQ9&$8mBwa`UKE`pfcBqGlE{WmH*@os@hPD*c=C2B
zq$w(R#NVome-UabqP}Ouc($-0s5g^U^+6tX7~$v>Ja;=~dJ-k2x}mS4Db$ht%-QjS
z{NyI<V8G<n=Hil}c5Zw|-s*jzQ<@jIOXq{9V2w}VH}5XaDDU5&C0vB*%t)l|?2E#^
zyEzr}S0joTFnfglaIl$?gtD~&=Vm>9(NdrHRDl>0%PPH&GQV00Rn;{e1*dRf8t`3{
zEWIxPNuFiB!^P}@WGga2Z7Wr#a6{S5;6+AvqMzn~3}z~k%v>nvFHXc0*-%3Hdrk6?
z3l?VwfT3(I?mT0&q7kiWpcD0%@FrxPqKIoX$afp<)s`M&as)62%8F~`2VygugZ?xn
zs|6eMidYW?UWki*wbN-u{U$Coq`UBwNhStri+mVCj5`?(WyKm?*PRjkv)#>P&{qZ8
z#nJSdys*DGC%Q=pqdS=(p9DH@XtK6gBiVlR9D#NeH9jb`u|7@zd`?D`_N|^vFQKgM
zaz@_>I9xzJABWZpe?~MhHSasM3$x9$^OiZ8qy}3l@Wf428dpTyB@eGD&Pdj4))dud
zMvUG;BwuB`QUj?m4^eSqlK@3~c$LP3d^cd?14nuS#18POPVzE;4|#9Y%TK)RBTrZJ
zj<#i$MTjf`vV3Wn9#Km=dY}(4#OSf!3~<qMyY6sN9c2PHMf1!eluQy?wG4|B`a`Es
ztITmka)W6X-^tvuiYayo4e!m~v3sv(__(XV@HLL4ed%zK-?4oz&BCMM<!P-Zqc;Ny
z?6p_mMR%Dl=fO4&X3K8+J6qR}Ke@7!99ov;c`68ucm$njlp0{qu@*4qce4Fjik&9e
zB_pe7EawjFiC^>Xbw$rKc~^njqzyD4yx{HI+uOSB-VM3yx370i06mamy()t<q^fkz
zH%#G}##8aEtZJr_;nv9o<r)sJQzlA(t8Wl`ZJ3v7aR@~S+chb<O{H9nkhBR(TZMmm
zRqs=+!S1%HJ{PvjXjPlVW$syphxTi`Rthq1d+LJo?wje)>rs-Up@rgIid=C8)j9P!
zAd{Zl*Mhs;c|mDBMW|t>VV@(rLQiNZc2|9=d$y)zpYorE=(tWG!z4n#7;}PfNg?&E
zS3X1UyXhCU2QjR=f;Yhl)cn?Lv_9ql48`B>`RaOE0`J$pztC>x;_y3gDZK?cnBWdc
z6#=GVEwTbvIat7`At7uD@n1yn)3|1=>J2mfp2B9z{SeCAY)$!TC)Th)b!SQ6w9=zG
zCY8+IaXZl2XjfmsN}-yP+@%V{%?rsF3v&Cd2hY%BD($*CG<evNVKvvwv3eHhfcWtu
zJ{FW=?s|zUFi6EP=wCStpzUd5eU+E|X7CMQKIxA~lzH-!(xS64T=rXsFay)_&9tr)
z(8j<*oqQ5nV87;T$NoyY8<I({G*lm^cd5EZZe=?pWQM3Bzj1qOuJ;DPv;_>k0s)<D
z`M17K#<0ffif406b8T(`!+g~VdR17}6VOSS-((j7trP3lNIY5LMVg6)-_%p#LExz_
zg=+`$FbXRGc<>4Cmz@9!%P1e6?>PWj_nTW_sLJU=QR9{<$L)`Yjbbt5`d!8der6Qu
z`TGN0txn>wsAqz7eY~B^ozTTqNU!%cfmC_;ydhDUt6BRB*e^_9n%j8N7oHZyD<-it
zw$neCD7c~EUSb)AX;5&ZlE32g^<4XO`QC9&m!n?n(9bKLip|PXUDR^3J+8}|*z5O~
zM-x+*&3rLWW(TZRwlvH{2M78!$1@`>NRaG?$+dRyP*v`@=@zq_o#AC4D$7z9Ec&y`
zde7HIwg!C|gT2S5$(7K&_55$S$teRGO}3$Ur`I_N7QjS!-%xSzs+{}HD4fK<2Zq`<
z#q68bisnr=SCi;Kq~y2Wj&_e*64dUamRI^ZFygtqv&gI1a}PzmniPskK<sT#r+0uN
zghY84d4Q*03sv`#HOS&wS<^B}0A1XVyqL=jL-ovZh`2@9uHkMh+TK<(qmO`>hNy-g
zv)w^`m`k!=!l3fm((6z*T7o|_CY^dAm}BQeyQdhY&nB#<x)^VJ_4Gn-gX9~zb>*<i
z$-ZGXFv<=<+jLIsOhP;U#4{>VZL#$>cdc-kUEhL63~}Rg{Hs8i21MNFsqBw5Vr`%1
zphypH6rrmhzxfPx5cF4p?n(<W4K7K$IBfw>yFBJUf;?^ReVl*kiOKo6i`IiRAEd_l
zQX07j3(3dL7KIZ7MNQKcw^Y><{bhYW-znc`Ao6$oyG;9#${H1<0t)7KU?2>MITxr3
z03&O<W2z=M7}K0@zGp@O|B3aMDFO;ffwG`<Nu}sR9D0B!?#>TuF4s!}@D~mid$rnq
z&C8Cv4<LDv&j*kunrd=8Sa?7ENQF#>wErtJE|Zk-3WZt&1B+UUq#6ez!?k_CnjQi5
zGy}X~QBaENibi4P0|v4Mks3e(QJaY0vUl}CV<~<;FY%42NP=kOwucwd81+ynVW&rX
zOFpKGjFx#2&!B}_lwO8(aJWMdf2wJR!?yZ$=MTH_@7^JbIo2j`RW}2VN`hIK*86pY
zGuE8CErpAeCU(`Ne`Qgg+lUvm2N9)S^OrD|7|Gn~jVblvenJ#Fgy)GLJ@@14_5l4F
z=Zu^c@|l#^zey(!wVJgO<brMX->h&3AG==?wL!}ql<1>7g@mvQ%-ys}#$Q%CVrdhZ
zAO@w@`YLR9yM4sAPpDbNlVUG;j`T|l1X-;!ip)N<sI55gj45Dv(9+@IYZYROUY_-~
zL}f>mmpdI2+rfXCNtK{I>Ze1;0|?McREbV)^N#%LUL+C{Pwbo<FoWpqFHA_ysSWEJ
z!6SB@JLk&v7`##S9!BslxfS$Czs^E0I_;`of<B<AqZJ8Xcsc3Fv~^;syZ`*{Y-CZQ
z>OKWFJVh3?4={TvFrRByA#)#oDkMHrF)T22W5EQ<{eo^bc=5j5gZ^uA=p?62LG~|?
zL;A*rn&z%;n`xfk&9H}MI_rNi#Ei<cnLnpYDHH(F8EpKS8<~#dEbc5L0x6c}S*1{^
z=#s0(_Y4@t5(Tuftyl8z-XxAQakJ*c10VFpOBT(k#5?fD?Ev<QQ}47@U(8uF*j`r}
zb8&n4Rx>AJJ=2m+9uD~V`upklKx_HI3w1Q|Y-XmGke+VPw%N|QUe*v3el`nT_uIoP
z7yV4Pxvh>~B!GAl8iB_;{nwiAX=FRs@5KF)IZ*Ad`oHQfs%ybkM_btLu07nQ8M}!u
zF8hJTTi7Z9rjRuyVvb8>BfliT1Qy+|zE~YX9bKLk$aFy&&*A%WjnD}@KtAU8G&d@k
zXF70kMvGC9bU#0_PO{D2k?3s1!J#s*Vm5DRr7PBV*SAV1yNoxme*f%EuUw(6@I1Ep
zr*(N)<0okb`2l-ZpMs0JU1n;~jqbes`Blq`Xl#6hUWrD*QI9HjmxHiK*ul)b+WMQs
z`ufTwQid-m4+_`J8#GLH(!W^ZfgTNC)tJ^LoZ@h(nao0(5~ri#?3bUN5D}u}p+F$`
zg5YD?8*9TE+i>_dnjEbKh#HWY>DHq6k4jclgXPcFB4WndBiTIu%03XlMho^%kPJ=H
z-h_iRsTzmbWB@houwehO%S&FGW9oxYeHf4@pgFf{p4FVcu8!0$Sp-s#cYhzueo4xu
z;RXv;1`9i`9?t(BQx5~T++GilP1udk@FDQ^y)m(G2hb~GC<p56`H9-eBVIH$Tr^@b
z<x<xtR=oj<E62Z0c2texG8Ee#WWu_6scJ&)m-Lp28t+EP36F7)N%lwcuNE+rEOk1%
z8BV!w?prn}9^7ueI!G-fO^`VP5>8co&t5~a4%|e7U8f|v&E1(~4ILbXXq&{NZ~E4F
z@v;jll~{_-@eWD-l!a@AujL$ZNQ|H~3(ns5%$LcA@@IM%tM+-gQ%C|DBkyX$wC&2#
z3<|#IYZP>3_Kf<QRrKf<+IA}tT%^C<NI*XwD<+|4v7xyj345GkpK~yXFnhx%fJZAX
zW>>ZVe9j#Lq}jkDk{;+C4eeA$cF`H(<axID9)-}C-Kn55wsT8K*E!_ciLrd_EZq$5
z<Ww@|!uoc2DtMn)Sh{3_doJ{)SZHIJYgWIkwRaRnlrTyn7Wgd4tB_n?_YB=TO2+8b
zyN?-C!x}M8Je*+(7)GD4#YjQekF%hdxCRu^w6OHBqKb_T?;OysOg;wp^F2}OkIH5?
z2^HMS0~A2##5xr6qSB`r*X_Fx&ovdNB;yi9E-P3o7t`S!NofGgorFZI$r>L?<o$Xy
zd*7s}e>)uhfl6X=A_@y*{V1Kb?!1}O?0r|Zx_i++`p39?oDl=e#P4W#0InnQz5Uih
zTXa~KHwQ>#3t(m_Em0O#I_b7vP?vV!_`xNLbIKm(dHZJAe}FLY;LCDbbF%V^!P3->
zb=6bpgAp-=9~HSGH%lp#z7)Pw5-FBO%y*BBU_XOWCZ8WZtsj4PpIf#@ghaB0`Y(Lv
z4iF47QnJtP%DA3mjjRRhtdsN#0qsmU+1L#jBg_Z{lF*}FGq~0H9!y!%4gHF|qXN)j
z^+knN0~-!NQV_cNX&Ay71%xweUf{U0J~GXeONk(KUE6Q}5eEbt#>b4j0XhC8Dj;Op
zPZMB?N0xy87z_|1dRf4}58G&&Qf$8r_-f%XRY%JGXZ~r~PF>}O##5#wS9(W}^WlV+
zD+o*fl+k3uKV<F~xBt{bl)NC3yvaq%C|Ldd7{Dmuamw*l^E=d|L2Lz|GCTrKpOG|9
zpGnHp5_UKhnNVU-*H!CXv~kLGp5x72z5u;f$ye~0^FjgTg$%du_v_D93xH}_D6>y+
zinOx#v8QN$nB-yJcO=OIuA^#p%Vg0!P#9^;aUtoP8S;CvakgrCe2Ut$`cBN{(fQ?)
zexibuX_FG%lbVH?16~gn3~5PCv{ZaTq{+7EsqUcnr~IMn?g8De7XGZff<NL>pmYT7
z%aA~+z)k=z&nTYy2}TwZZV}fs7GE?h7^T_OjD1X;`T(zPtPr<Ll;}N2ndkX3Hi220
z<89YS@*52_rUM-#8jI40g_sqtjfW5G^V5CrQwC*<I$xFrrvK<weE5k@bA|EkF(Op4
zT~T#FIT?Db_sT*vn$9wuD5>eQ{yKL>>^1ufO4>P|0sq!vMQSx_xzLP&<--NMr3t|w
z_c(@I$9Hxx(0j?~6U*G8gG`37DMoVPj-1_+$7m<Rct#7M<xlkHXn{emEOt3~O&YaC
zgnJbIxW7uN$m(kX?}@CK+hW=Z{@^S=#No!{5q&x(h-B#Sj~@4hDb`ZeY3ZZ`2|6l|
zy7#}zzF@CxK)-sdbF_o`e>#f(IRCEaU7(9<^r2DK<owJx%p#g99M&>4?VKKLYC!WG
zy>60n^_1jWQ_E-Ycg9kn<Q{r);-Elfi&!3UfqGLc&^u`~aonKeU89FI;ZdVT^FgTt
z*ns*9$EQQ*_Up;8Ex(l(A9FXH$dG9d&*fyx)71};K`iU5FzaJ~y>hS@RAtu)@VXLI
z$g+p<icm4vW<0YZyHv8jQ`{t>Jwe@H;C>cE!$stgFJFFbVMRtR`B6<E03kcs1<X$x
z$zheB1fZMrdd~K<c68`y0bc7^biA2mCf>Ky;Hv@z`QUY6YBMV!s1FW*`gahx@Kyn4
z+lAMgv0>`w>?V1vDOiSZ3Q0^(y3Y#iuCmHH(y&C_tL^ixiDmZUf~YVSZWdmRg0v<L
z!?WsE;Dbay)GySDA<Navs8Ws<5q<uPd;FIYJu^5C(}-2O!U&%n^eh`h>WNzz1K4}U
z?b+p&f6Q{m#_9YDw!%y#-#B5V3YQM0#Z=j<U+p{A$0ujQJQA)_taJt$CNE!1`Bu~S
zv8SvQ?d?cA7JUP*A%ila_P(JTA#=H~5-vQth-&B9vXe!OG$1EC@KDO{`e!CCiG!xb
z^EMQdJ70zF2wc~XJ}`eWeJkhSs_>mN*d0Ufnnqc$*^VdZ!jbDtQAc|7(7zDt!ZtKG
zQ#$k65*GX%i*ETR*a;-Soi9z%3g+MWC0zATHtlHyp$TzIF2GKt=vC(){OP8uO_~F#
z<<R3w{vgP#u-^NYd@n9*O$wL51S8=~m6Y42Y<yNNYsPpXokY*%=OPdhRVJ4+$n9!{
zu`K2g3)Z&>wMW|n`Yc}PYxHS?GEg7PLDbDq_B0apB2`5eQ<r8SlLGbUU@gO>;jvyu
z%)ocbs+;HQD8X5_DE?DPK*|ct*wy8Jxji7GNkKe-Ho-YhUC*hHr0_dEWIQo>jLNh}
zGO0ntT#Xh)pz{%J+%gK^Othi;IcmMC2h6h(xK|%N<Y=T_efy<7DlrbzZjvGLt)6Dz
z)Y9t$@9qR139WqCC*oi*g9F3vVHlq)GDBp;9u#uo2ohdsIJ~;@2yk4Zq{W-`I4xx|
z8!ubeXD3jjkb~2~gyg)rEd}3(v!vwjbRWuuRbYmaU*4b5BQ*)OehYZ`QlHQP+%#kP
zn4YTN2c~$zm+F|8seCQPD}rF0p&K7(&5X~WsAGsR#48_OnEw?}vA-^d3w*f#so!T2
znB=Z=g9=IyX_?qWk*7eXgU#VqOp#9C-~7Q*BIdiLErBYl78qwoPs@XTe)q^14NAY`
zQaB!eUkoxtRKM96-fVqCPCOD*c!w)>qI^?=p)a6?aPqvHwCAXZhT-|!fE*r!390io
z@;n{&Wx?DiY14fPr*Y|98x=e{^ca%g-wcE(uKG;Q91L`5^M$ner50Ko-8m#q1Ml8T
zSM1BkaV=zsHZXll<j_Jx;C%HtjxO{tG9(M~*IuJaSgVvsfqqf)-NL`|TWRJFa)FN;
zf2+Z@GT1ziyh0NNsB!7#8<(DCleEs4@p+`R%`=BtUfr#?v_S<PBA3syqgd8qZ+(_8
zd4EEP^MIh!Z;iM2(W0IRtnZ}RM<e=Q$pm<FfO`}3w__byZm^>S^sJaKKBo;L46Pq2
zu$O;sMmJED&kHi+cR_@FIV!I?E(A@A35XtdUB2N0`j3QsPx*bedRC=zLY=~*B|Z~c
z!Y3+KQ#NgzX@;405I8uA5_0Sl64f8VwcqQgi>=1mG0GV7s)3)qeD{H=iEPqga?8(U
zX*tshT;G3ihQFVQc<#Tslsw4&*b9veJ^q+xOwv@X%jd4<3Cy6^nMtIO4X&`6D_=#@
zt|jH@1ztFk%;IYPYH)02yGjtYn;7+51bAsPQF`Mg<*Nx#H(2qHx!?F!eTf8AMU4Sl
z#-L;)U9am{Gd}<Z<1|8WyyzgS<inbkKv+-Kr(oO?d7O^ec10%T-!S^06BSBq@SZLP
z=^y4lf#Pt;#5XP>BQEyZGS_)%T|WyaBcmeayDfX_V_27p=so)$zqYRs4TPx}bizz(
zJS^QSYo#~kq%19(wkOHJ_Nt)e6|0S%Fd<yOpD>q=NaR<ILUZjOP4GPG72j{(`O#HL
zX6Y3g5r#U<%IY^{5L|utHk4uPlUIUp_YvV_4xhyHDO#iD3iN|6Yi_{Qr{qVo^Pc(k
z89y7k{m)z#=wv5^{P(-he{lFLDbm-OnB6OxdK-5*vCYRgee76~KD`^>J}b4=m9|Pw
zck7lXU<*;G;kFU=j)z}Y*bVjAnXP%(`31(^E!{<PjUMUnww-MGTMnx<&(L;sfSMcH
z*6XN7SyOuV4(z?!l@e@El5S}@&bi_pu$L|WV$-TehYCiFb8s(2=a~wruv8SXJ-kch
z<c8pGa{2d^<hH;s`4^cEye=_&ElwLk*FOVHc8e{1e74xJclkmfxWJY?>M3gg^@cbb
zZSWXS+zLQlyPqkJ(ZuQppe`eMkqrvQ(U->fufZrBaX7{kcMN*8pHyw#!-*5xgdHRz
zk@2fDH|!G!<;y-#PE>(V@f!0XRg7$hG2wiL&<bOqr*+nIQOOIT@87}P-}8PMQtW00
zgqtfo(Cr+qV;sW9|1f(D9Ir+;xzWA*2n{eYNBEBnHuijZHJ@mrN5H>T$|(OflRBcF
z-AgUBy~iyA^o2X*@rB`gkRUr*Z?rEO?R;~B#$zJb7AAiRh_~A7tU&R5$zIKROn)L8
zxL087+A|&Q8(I*TJEV45=$!Z-Mh=Pw$|UDv{;D84cPakYj{5biQ3STO`FSYcmd?iH
zwX6`rX!<>+9U333AP15O``9hJI63LM_L&<MMQx{)_uCOo9H~-o<gBpP2{)c;++?7I
zRw-w*XtnH|A(DlFf|dPXrA(ieL=J!RGc7Fhxa+K$56ML1)tM!nsIG28tBP-p78$Md
zrqzM2#h~i6ey3IeB?SJw4T5>`cI<6#2FP;AwZghT7?C^NsSgK=CIQbie7r*y@X@<a
zHw3JmcrRT1aDZlx5^<!9J+e~*KMAd-vC6G+D~6e}C%j&yHfW;xw@1|9$m#Xqu`k%~
zVu>tLHlc~<fjzKmA`12I(GGXniz+nH3<%Mh1?q!-?n*OcXVu{c-lQ`wJhqR#PgI*1
z4U@>THU9mYy;Vc1Q&vx0ZVu@UT_zO8ykWULnF2;TR-!3gb^|G7^f{UG;)Q%@FrUB$
zxfZ=C7A)aaMMK90k+9J(cVtD~cI=0#o$1&az36^tO>|D)fLEkRUDg-UMMC_ni|H%1
zz2ST;eHuT;B5;$BJxGvWMCn~xH#LW&zVPV1#bzYmSKC3FlZva+hSH@!dNpQWfd!7G
z8NEk0lFBb1dUFC_Y-BE|m;8*rVkFPnPZ^=y$@O&IZ4w^L5TJnP*l)q0ST`cMrQ+Sb
zi9MymPG-X%@_LEc74?IcR;6cvEKD7W?il9a0?c0bm!pIASI3lZMPDdIe*-LK>?Mk2
z53VMBz&Kd>Mwh|2I;*b7^hTXH38yh)25BUphjAR#^{~m<iyoxwKPm2pC@kA07(<fC
z!dfQgLzOB2xn45N;3^?mPAZirUJ}aKL%M-rzViwrtyPC8qL8x`$_tZ}JD-A=RR9*w
z7z#3a6je7jQNiTu>FSGwivkVI>@`g#y0|m(z>?3E`ox?qR@^NK_AARhmHhcb(z&7h
zp}R`KeN1MXO!x$R{acvQM#C|75j1#t$$svVt4JVWik%>WM!X~j^Bn9w!kUs)%$t!b
zz%oR(mh8+nn!sq|_cg+qF~2M!qIq<`^ADyP8b;s%O_*D_T?WYKB$GbuGxKZ4VyoI5
z6}>uv(!Jy|q6tlp-?U<3JbE3S7^3l{L>lOF(-U)9{_o{bcx5m7GoyA9nn;Y#qa53B
zSaX9Ki^F)c4EPz6IoP9L&VA5JDC!^zbJXwU;YpL^l>|^_CSwzbyeTxCFmo-*fS$9o
z#Q>L0>O}$y_q)=Rnb^2K0v8UxEeSw=BZpHo4vCwje99;z)GM_39G4`!8-%~vE|)pW
zG(RCwwE7)6#FV-xaqyz(vdM;D`qg4(q4qa&kUo3zUgD1Ox$D$gfJOD*=**6;@bWqv
zVpW$cP~0jGsPbO9xQ?A%zD2!YQp!5S))fq`G*jXGTvf|c+gxR*Xf^)Y7nM;W19%yR
zl{}c;RL>ANnV)iGQ<0kbrQb7X<d~zx3inpV4XRaglOvPtORtKY?zVPK`-M%W?)57q
zqekkqnO9MTgWsW!uR)efS#t4EN`aGt<L+~R*nZC{T}&c^Er@dQDWNo#w$e)#$7(|$
z<6I}hBK{?m!)T2ff~}CB(%tFTex<aakEA0cwLK>9DetpV3qq~t;^<TmD&;L=dTv@W
zNZr2~BL&b%)YNhLM%-nB;9E>*iR+2B++!^=dWCRH#x0kx&^7aMPF51_=tOW{nK!yb
z2DFv<UKWSvQ`#CCXe<G5e?#D|YfGqMMNK<5Be6Ol2CCj4ljv1njN~&Ok;J9ht+rFB
zRYfl<IYm=^qR02j+b)bzqCf?)UBhbqT1~o?C`2~7?TfPBol)T)mD@Taoa;4N0=#TF
zv#BJF|8=;sl7;}pid|}{Jv%11Y|^n$HH;17wP^O10_<fG1jmB$slWflb?IuaDxEF@
znz{}S|0396KjZ_0<|Zyl0k*jJxnTA0U_)l8d=iPc%!me_u4(^5TN64v=Va4F5zuQ@
zRyn|}P5u?hU}|AmT!ChIaQJ%9C}y;^c1ncHMp%}%<k~`$tu&4_lb{bwdK9GQ69^1l
z{4H?BB+<3sahIK(WgiZ_y*x06!QOP|McuknfiTwEX+`A`4qUyL_+H$u-e#F(DuB3F
zC4}#~o7MeUdcKdkbb2b~x&v73Ojh78;<7VY{o&500(fS=;PlT=zbr>VA&gXom^9~M
zG$32JF2A);q8P#_s@3-#W%m2F%mR~@<wRd(?u5LP*`TNsM%d~6jeiW+2u4sLt3n?m
zSqzuA9sP;v3sD?;4zE$O;7#oxDeS_aWwQ<7q**JbPZWYR>EI2hMEW;Z3~sUxC01Vt
zN6l752;OF<Y5p{Mt=0W<P@oJJ(~w7M*Acu`nK<N!O<qt~Y0>2LD=l0n%_Hvgo|BK|
zrpwQHvm*F?q(w2<LB8j?b}?zA2S@Pus*OL|qjiqR#MU{4;utq{XXc$|nqlyf@ybP7
zNVg_7)lH(r6bcpw5qmweV{1K%M)!kwSj<KPKkXm8!obAw*OC(K)&b;%c&D#ZjqZ;n
zB-s)xQWpI_LTo8`n&n@h{q9%Ji=tDSi19akY#*}WPCuD=M(QRs|Ho6xV9%v+fC9wD
zw+aDFe|}cIx*evPN|Tm0pYFH<1)no1>*P{7&Aoolxp=nl+MGGE^N2*sGLhV&)UpeU
zgUD1A`~EVwvM2n;;jKzMO3l<sobS1qsH2scWz6>1AXo|O(FSFmQLk+K2p_Cb_Mswm
zw9`Sf+nL6=6?LJc3XJMi=KVb>LYs)!z$Eg{L3HZ9t@HO6nfh$8msajo$H!9mOt&pB
z`y%<qR8P8su8jh0SFRf-@FA`*@w!%W7$&L+k)unVF(qD1;f6VG0sC^$18%ctbAH0*
z4w022Cll%5mw_!#B?3}yHJ?A&e%V<P?F+IlA(f&zo1N}HR}7-2tlxJJadJ1ZNCxCv
zN5Ty3>ZL<8;H)_@-)M{oo0eZ(DG^`M97PDr9QPsl5j>2JP^2PZNT?C?I-x2E%2eT?
z+FVsPxFQ!4)XADqi-~KF*l6T6Z=f%C#0g6a_Z8T4Uy)Q|m0mu~uX~LTs(|fF{AY=z
z^)|%bSHkCVS+z#=F5ZI}U(pU6v%$Kj!k|1%R~;NM+xEG&A&SIoOwhi-z5t`yH?wc}
zCc>bn*Kp>V7MjD;>*#x%U;*;eV;CfxE4+LI6W+S-itQ0B$A6v`VQQi=qx_`;GE9D}
zSJGXKB+oJVk@lC=Y!<Xh`kb6;-{`cQdmK+k@^Ov!@r-{50AKxjer#kYw2<*7<<R0s
zV!o+ujtkct#iw7S%*_bwKwsT3INyH!l*l?BCpxn1g7=mc(H)G&v-PJR8yu6@LbWxZ
zHI2ne6I^8~i2=h20tVwGQlo+}s2L^xZ{6*Ql7q6cc*Ka$=#8wcr7D_v-O`6L=7C_%
zL0_Wbax@Q{g+WYMtQ*seCZhOYO1lrmBTI2#-<0etu$Fd4%R~ioC)Z(J*P==^_0#Va
zENci#G!yi(m}*Zs@#L5AUxvP;PGqSj>rn>dS75G!xymd0agH@UuzqgPz)M(n6-mW5
zbGG8q{mIVq<3~VZy6O}bbB{;iaJCEm`7{jeQnIY*@OYW-<1-eOUN=>X)|s`3Rw4~O
z)^5}4`?ZEOb?Oe{NZI6**#)ydR1NJyh9bESrXSzVf8w+hLqr@5uH<&xfqTW0N0f6!
z7C+{e`JL<91y@7+Se8FBPcD;}ebu~|%;4tLeca*Pf>oLFcoMCXx@1p|F<jX^xI_-5
zha-=V{V7qAw2sNzkM$G5zaT<bI{V2T;)e}f`jQR~XVbpc?vV(!xslw;n0tt)hb%+C
z5wd8sn}Wu_RgVlL0uK=appY&#{p4B?-yaSjRqYrH&h$m6h1J2GpXogMhl)S$zUb_Z
z`O?2j7)dcoA!!%%8tLar+X>;k8xJXh?E%w`X1pv|>h?b`Gzj+qc;P}zb_9ujrM<A_
z{c<&n%i3Bf9#zQ?KxM~hF|dpTC4Dmv_>ko=`ZM7bwLm??hH9%R`LTB0o||XQ&x#zv
z-1%?V7PY&-n$u{)T*^o^V9O^g#Y5NZ+K|w<UxdAx<0u|)Z^cZj&hiU>OjB8pI+tL=
zf=zK*ZhyD|3MCWOEo%2CcNEX%>a~;QT=L7N=sdPx41vg)CTWdXMvDU5)~`&gk25n0
zynXXNuk8Oe0#v|hK&{)Ud0RmHMK{Ar2bGlJ0)`nzvN1^}vv#AEjL3J42|uBUp931e
z$(ae3=H_XV5beu1TppWhfJCnE*MiddicNcki91`|pNl1|UDAQ~7QF*D3TnUA%1CO!
zy)1n{5L+)mCyJ1A#arrr*zpx;-69#Z^x>rBPAq8-Qhdp|qu2!eJikAY=8+lI;$reH
zS1x<Xxxc35rr@o>9Ay%PmyvsgatiJy^k)OrHw4Oorzo-<vW-yu02F+$KadM#j6Ayk
zku1sc+hl>8!Z~x&%CSwMPAWHf@j&xX8^=3x;3D`_cm~Nko+~dw5#6gn5%4Ss8+X*7
z>^bN{2e)RKID%W`o;9Nv3T_h`C<QK>5Tc&WhThtrzvmD<+H0eFt#Rf56ZUP;6qGaG
z@&y{8Cfm&B7w97j>0au7y4e>MvB$O8KLCra&G+Ps1UvP`IPiXVFdJ%}&>;3n#k6;O
zT6=yuaSV83dL>mP(y*LoiwUnmsoHo5v1I0-9ll?nEm$z|r_~qSgeeBt1PWcDz>YCX
zulSQz0otme>^4bV<%z3aQL3TGwySsOAjSb-r>7l4Ts8E++c5sO{R2Y`AigbnIkKG}
z$MUU*-7dYtR<E1``Po4&p}uPtP8u&4YP_uKo2ftmFuE_!j#g4GNMRpuOgr4ZNcx59
zTUMijUy*@JTH6UR7Ge-cePY@{=A_HfyWAGE>9_Q+(q50HzFKUESrOh^PW5$~#n~;)
z-FgYf2->b#{c)1v=;zJe5cMW}nKb^wZRJBvG@OkVV{MW&UM!Q^kSrGr@lyGz*!N1w
z)Dbml;qkRSanLpn$w%$3wKFog(Ap8`L(~9*3RQABH9qe*MI)L-wo~JWi)bt0v+!Fo
zm%3v!^t8g(>rKm9Tjbp@Z{Q;(+_~P&dbNy2(d1+s%+HaoXKgSPPy!p<c5kkNs$942
zszCs!r`zE&Csat+LdBzL39mKe+aMQ_M6Bdn(9}0RrzjGle;j6kL9E<2zbL6(;At%V
z1)%rYP_7Ja2=+L;wCibBJUeN5ovq|zgDo5xwQyqSdiGb!y>LFJ&CPfyik)~awx#jV
zZpZhY<Fbr-NwSMv%#m5gnF6Xe08*sbqvyOkJFB(qAq25hyVWQR`t@0K8^Xe&EFzOJ
z#uj_^QE%W;8iOSTf@n3iP}N#id{nplx%#r44Nab^I!JnWVdRR~QX>(0lD1QGEtX<#
zcQGjLu8~})XTHLWr4%!t;JtbeRO}>5kg@#ooJ*m4IzE6p_0o%oqV&&677-gq^IA(4
z>C1sQx+F+ooo*Wh*F1Urs?*k2x4MsQL8_b;&irk*9usVERo~2xFXdC2Bt~)SKg2Dk
zZlq7Jsg4|LS<eQCbROt!xSU8fZ3LTR!KJ05jgpn_1VY>ZHJBpi(DHTW<Ct7SZ*`~Z
zxp&Dc-GpD=ECFJWrRScVEs;xGHTy^nq4l7U%;9E55euqg2SK*=&BhkvnqaxJZaA?T
z8(cHc`Llwltr_#`;xvTXr0jGI=v7&x!BPISc_ogU5$KAGS2_Y_eU~7(wPvla$G*o`
zQ}@@F;T_LIo{xYKmwD<0^L1uJ5w~URk!xdi`WBl+qtJ~%tP-9Oqc*cU`!ENQTq}h2
zHaZKOUX*DoV30`l(}z$}_*2d$hCMw_hFK-jC$x?DJhhWn1HUg!-M_<i38c(#cQ4!D
zd10Y9R|Y47ejDxCgE|_5T=0{TD&ZB|{;t_k#T3U%g^A|NAKCvHoe05*9IKm$-W!>k
zUCpoH>vsz;$r?$k#+-;G?3c6ll$$hs23(RnhqLm1d7!8*Ob9TY-&|)fZ0ltGYN8}?
zL$bjjvtls@B|dBI5BKXgD%_8&>L&?&<r+a`ymx*uj*VJ3m8oz0r?Qx<rJp2PG)cI*
zy{t2Z(&v3<Xy}`wgaB$d>h#BrEzs9XF91P<PIQy?fgrdY^P@RgMGvC5_B%=}^VGw5
zC^OG9F^J!k$rs2{6lA#yHpgE^+JL!CHCVeJO87J~W;V&fW2M)!3Te~8aW!ZZi>f!c
zj6T}OSc7d_KfsBDvUJkOy?f<w&cCA`guU5baKi{<b&+`+yi$Ipv{y}ak294wMbCpD
zl*Q`e(?^aJi^;SK(8};S^dx%u0iPNJYP+fy*wU8?xr@{KRqFXH1zju_v}xyY$%pv~
zI}?I=oJ=o<W1dU_+Mj+IT~m4Le+hY>g7zdcefndauO{lv05F-*MBAudAGil)L~?x?
z#eDjliSz6Oj&Ec0&w&k;$;xhfs(OkVZeg6R2!4G$Ek0RHz2A6BGYQ+PL#Zm4@z-?(
zBYRM3b}HF%@i!!m4p5JNRxA{gC#=_0%Yt$yva^DUA`@_Gb4RcR2vF5@#9k5!e0$dv
zUi!84*V@X-ouD<rYxQRgN-gJ2bU25<U&&<9$sLmEBW{%+woWcI<__GkN0p&xvn!TS
zHgbQa=T~$UU$XXgo{lERa2BZPSKw9_8>bk=Bv?!`v-ysu!|>1KzLe{l(lU*e^>Xot
z_LlWKe-B7nP}KVkp3qV*=Et9$BwS{ZV0oo;uiRex;lvLBN>t;Ua1f=}{@arNKY0<(
zuwU5^ppdCm5qUE0J{B-A&yhW+(+vFU2*Rk6FvTRR(p3w_m>UIcyFB^bdv@@FV+8ik
zOPQsH{q|YrOK^q!UzabL;2%^XY>4cUb$5Y$ygu(7ho!1m=n=G?U|g5=ay}6MG7b70
zWtMH<^>UVOCF8smPqOGSif9cKXik^s`T?O2>JQB?pSIMm`WBL<VhuyRDtQ9i1Z9;-
zUUnm;De2@-Y3nbT8q80Cnpw{opR2qHdei7mS7GFX&hbka)gjlOe3hESS3KB3dzB95
z#bDAGoNH0aNF(9>j<=*zQ@f3N77$^XD+MD9gbvS&b|z@Jv3aH|RGWuq?;TuQN^_mL
zzVgifejDqyBcV$po+Vx3Ur_vqi8X5REJ{zE^gk=F{{aD$`u~@rot?~reR_fAtJ#%k
zFWL6>tl3EQ8TOocFnFo=#?K9uS-oC3^Vbsa|NR2BZz_I#djfe9g(vmCUs~>d>OdKv
zda-4-iuev^On;{#N_WnnU$)TrTU+J44wh{W<TGRrh}g<_$@UFLJnm<|S|joG#^2@~
zuBDm3Lf5IFMX^Awl2Ye+v18xAJ{yOOE#<G-8aYXXsBT26Kxz%jBiqu7e}{qtFzmwR
z(}|S;B$?H@I6coI70-FLWt2uZ@XZm)?yXcAiWf11`kbP;1ugoow;ZT~<aE(Rz&@+h
z&ER}?lq!!f4yB(?<o)R5&%)o93xI-fU+BpMR^eE$u}OWc!3Am?2mlDRqE%6GP4OZ=
zygZi`e-VuIY>_q->v^nfR-y5u^44(cN@u2Biy!{peXmnA#kT)`-RRQh-Q5AJ&6$JM
zv_<kMtB*k{i0uI@DFj+J*x_8a$tp;E+HYkDsrw^ZQAYz$Ot-%;)GH8C<hWk(u&UWN
z-TrlG^fCja@LVSEgb2haq3302K!txfr#jYW@b`=9&5kkeuxU=;fAqd=1v5(kV5*_O
zzhjL52bz0-?yrq^@zwrXEW4e~S+&n;YNgA4GENKU1NH{h#2L#`{;51Y(_Z@C@Z``I
z$i0Hc*qSemcGHXAVS33k@=0;XE1hqY+)JH^Vj%^(Yu76E6avwG%FFMp8^jd%L!<XU
zpu4vJxE3)Ltu7k7E*8pUmM1&riyGPpPlNn_(Q?O~x+`6gn%S;K;)}k8lP<6<z4ztB
zpEv$oH?6b4QohfVdjJ;kyH@TI^|@ck71gag_xNo;yU60FW7OGqAz%7Rw;Hyq%zGt&
zb!#(IdH~k;DAE1>scqs4+^^KD-1DG0&El7<Q^WT(qgx@fPrlh#TqC(p^R5h;iFU18
znQx=H;mvo2D>c&~fL=kFa|<6l@%g^<{5{KT5J;~E3fnc4GV0ugmT8>cLqEf-XYrS{
z1&txoD}N!V8(<NnQq&8p0$!b3f}EeAqJ^s#WUlLk0-^sBsW(Gg@efp>1)oiG5&=wA
z8ZyWjIea)ODJlL0mo%iwOw#h`>3f>rH~hcgr_JGW6OI&!<5K^v13kXXkLMzwOIhiE
zvQqsSUJFS_4lkLWO2o5+b;oYKH9iIiRa8v=?I88HcFYkoZEc7|^{2eXfjGK<d$IgS
zP8ShNxATPF9shjsM!PCpiT#^<Cu2TJ_K+ox;tn%O0BitwWaTtFlCLcFREbc<vhOsP
zZ_cVZz5b582a)Z30X2;fCuT>nlIgMW0<iyj|HA(pcU?{U`bfQE4c=b{!o~qslm6jJ
z1&WX8jri-v_%MaXMl{KjJ?M~en|g-B<329!)09svX-GC}3OuDZ$Z(zQiIeC@TD3n;
zEm8KgX3PPw4EB}@daS{xH4+C(OZuKCiov&>6U8clxKOK^W0-$R05po#r<D`%zU<^O
ziLw0za;%kOjKLW(Mdr{W*KsoQx~t1?osB*4tZ%_En4uf}{(HkJM*n8cqUT-+QoZ%N
zL&83%b+$v*KEHLl=iLH2hWyJpsBe^|8lT}<YR3Anv;BSVu%r35{w@Q*kx=So(WmUU
z5CT(;@Ck7!zCKPQ*|c5u@&~c)=eyd%fvTveqraXjwnob#tt1|YjqfN>?rfFIV`WYH
zZVEy=D~q8INs)u*9<Gf!y$h~AR!i5j5I6bHdi;KEM!xX+$=M}qc!|y*{|AT`$1mw<
zNN;3AD%?RY;Ap?9!K0mub(tuu?2bViX3wWl|DX~jn7MG^F<QgAtX=?0%;z?~R?dbo
zP69Klaunfzw|%z%NwC_dz5D;7CxO30|3SHb1^-v!ze@ip{8#D!Dg3`HlwDo8-D>pp
z8a(MiUpsJQw8LsV6_O!p5F&@sBXfh+{3i`E75cVZ;`hW3Oq0NldrzTsPI%)#p_x=#
zgW)y1cNT*09(<HJIvD3og_XC=AiWFajwkirc0jh}jDmors;)J=yPV7p1F%hk<q2PN
zpn`7ek~f}V>QH1uvH6O{08GF2K-;0WPG7qXE~483(Ly(vJ&ZV61`;{wEI<>+$@NSk
z8+uuChw#nrCq=DQMiEy}Lrl3tmZCEs0U-Z=SpZ1?>7DH_C4T;&r1pgDf0Fc5NdHOy
zjqqQk{}leK_}>WsFS9=>j2bI9Btj45HJ;?Zdo17mV1n1PC?!KaBW}MhB}08%CK5j<
zGxVJ(>h&3Y{)_LRBBHzEJb-N(J9K9%70-@mP9O@+z=myE`vLmZ@r2J)t=03^=7(}m
z@51&o<H_EI-546f*6;oVNy`NQA_tcS)@Hd42fac)c!>w13Y311?qO56Z^DvZYMb1<
zV6fBXENH8j-mtS-qEEChpH#j--pqLR2)%>Jdz#v|{&h`_C^O;mJc{)2pVPgT<i43t
z6;sx!qWdrS{|`a`7fSyr{1@8)g7kku>A#2cZ+W@fr~XIr_c9zT^6CAnvq+WsrX`1}
zf2Bo~S&QWv3!;mCS_-TOi<GSF%Ol{wS-Hn$+XJ%WkUW1na)&h6k~33Z&L~9k%7@7Z
zAQ4ESu<~*|A+(yDp8nNPYfc|_L?KPDTDT3q!9Z->ez7O^C7z?x3(jvzthqQOA?1-W
z7LIp~Pmc>6^j=-<FP1JW&P*G`bUycs={w#Lyt+MXn!v^Q-CpKrE9rXzp%x25bw~j|
zq%VYXECb)l#v<7m7YzHK5Fx+7d&`t6@x;kveUg!b;k}hrFEA4Lc6vN$Bl;117?m^t
ztzp3p+*y_}OVeM-Fpa?keyw4SrCAA49;T3bArhZ6T@E&p7oXDTr6jE$fAP84?l#;C
zAf>iX-zC1ATsl|E21K!GHIL2%^`~mSkhybhjjwLj7#qxqcgciu-$G36(DM=2BEuyT
z?~M>9Op}q6#;`KyEr~xeTly((54f@22C2?h2DRYRB@>pqH7(ldVuOGxy$z=>*OI^X
z!iarhfqXY<cyG6pf7Rb*E$XHGSt6I;$@qOPYZt>|_t)Ublfel25kJ^r;!b{nzg2!#
zmk`Hx@2e!R90ggO3-t0fFuyFoIXnsK{h=T4b$k3x#WO*>o4s^wB{22~`h*IXC6Rt8
zHl+!3j-97`g(Mn-vT?Io(#<<8rpb1P@nTVCiir4vfP&LR>TcH^3wuAa;5~FQ87sNi
zStI^0DsD4oIr#3YPmXZ4XpFrm@lpX8VT8J~zv_A4HpiseV#`AxdAMOn!wTBn8Xoc!
z+B%YNExZV<Q$6!W9a}jiB1@RIKb(4XxK=)4wrqZxB5;fyl&do}o~Tx&jGw@+pCV=j
zQ%@y)eWI&1N@#2>I^6`>39)TSkg7Z#+-#b%%++=kFd-JZ3fBP|d%P4-8}w314sgl3
zj%a5qCcg4|?=w%xnD%$cw?_Sd($wOIUeRTZ&IC<FYJ8w*{j5`mr7KUas|<EzzZ4%1
zydcevE*zR~g>dcU<l{ns_Xy!04{ucTN%wwFulp>Q2yW{y*A=GR%>Qimvj8dupHozB
z7)--wl!=L2?^8oNkOiQ@RmNzA@7O|(3(9$jTHhF~r`k2k@TKG}>E&2R3OOC8x~rly
znN5is{Q%abC8_;2MU7d3QqvTEVOAcz;u#GhR6pWRaXJS+1@>g5M(}xh(Z=0M(t!dh
z-yDO~W&4X+lXP!zXxkj758^}i04=~w-%_ykacjj#X|Al1u#TnSSz+MgMAI`~+Cx7C
z0{y@+vbHDjg_{sa`w#z%<dK+oq5{R+XAkjW3duVa8d4qg-52XjXAQRRa*gkv-`&uw
zh_I(uMP|V#<ogbk#XLG${<sU7`G$$~z7Mq67xb$5s=I#@ILCtH185YuiuB#tzxt}q
zY?gE#S}Dp_Z1z=mqzRKjT1cmCBFul!QBxC6kpaxcR_`0>nRDG?{LBjal$Hsa^osHT
zVl00R^Y32?*&{B95~U1;dr2z#he{T%yW_f+#aBO7)Kh5(z%bHf)5v3=ZVt4YUKwwb
z$SqmFTTpYSTTH|i;7DDJ*9dd}4R_uG*3w*$Q6W!4b5T9wwm=PCHkUmCOG0^yMYL|u
zV{iLcX>fOF@{`$*=VaP*=Z@Bz)Cy6`OukgE2xma;wZf3o-8a22!=1AyU$kHdfOUY{
zPxXYSeH!3?Sk(KO6|=Ld^BMdu2+5}==@qskWAKc)BZJN>g1HbqciL?d5`!GZ9B+Jr
zUBFoQG&ZUj>MPmG8}ye^^-0fy2IgoV$U2VZXe3H8_`aQrbjH3<h%HTW*^J%jFB(d{
zFS|;o5KbFzUp)|^E!wXy?f2haXhK+q1|8`DI`OrUiZHW)NKa?A03SgA+E2&7Q~}+S
zH`~s>b_!I_XJpQy!;kl=<~{)ysk<F)SG*rU7^7BN?NAy1^Ym_8nHBruBU#YI3TYs$
zXg{PjAAIk0v%K;;56ZH2A)(n(S^oDZS-RkfP8PV1ifYuH6n<&{(W5S5bVeIc6a}ka
zKTFC3di5x;HLpYOZ+-grJThfLse4}OFEyn>hvN@$8v`n1wN`fU(iDgH^z6H&FR-FA
zx7|PB@*xNB{hgANpQ4tJMwXx~AhQc%q}Sj9bUi<4S6tRs^SD6Df&x}cerUXw?<mb*
ziHU=quNUYe3K|YRfQlXj>9YQwb^8}MSnFjce+nE)JL<o|!(zDm$7xnvit{gKBvrj7
z#du?DnF<3r5E7VoLddbg;ufs+bkwbn@B&R;um48`eG;i4MQ{icy4^x$@*i~|<O`1j
zz~vW^;@iggP8JuRm$k?5ypI}Yq4qC2znugzQCUmXR3y_=$T5;9-CJ-uRg-%R|ExQZ
zeIzFWP4IYS^#~mCHrF>8C<Za|P8yFv#vz8aPdOuwaZ^71eXTmL4TSA8vWA+(ZHn=~
zP8*9R-RXzv3pjz)Q$IR~K6JMmJX1}6DJb=R%-NSU8N9(h*S>+jP_RUzk~X$V3+|Oh
z#8oQF81K!yJXJuFj6cG&N;A1MNsp2!De|j|CZA$_-q9wwx1hnf3qw_Jqp-wN4Pm;P
zgorBz{9$9uXV*%?Wh>#UEVBqzW@V1-%TKF%*-4q}_v5@TzwB=FF=^OCl!tt$-0m&S
zG_!NZEiMu0$Upm){DOi2KJktl%_M#jvgT%vvdNG}#1Kj9z^ayx!oB-|si^t3h!s%*
z`PP?}@gr+{m3wa5^*_oTBX)2kW`iGoLMc=gq{pQ_w50agZK2O;J14vS-5S+v4HQwX
zz(~xIM5R5TY9^UO1!PUMzXuvoiuQ*EbtsUImdO|1xlSWHHahJmHH2vmTOcj(ZT{Y4
z#ne%1s#wgz`u8EjaMcvK3Z)+v35gS<=iSzd!8kXeFV>fZIMmOGxbuTaF)Sn`3?%AX
r^j4vQ<5e_@q8%d6do&30tM{Xh>M9%?3QzxMfg~rTELkD`Dd>LyaA|;v

literal 16637
zcmdVCbyV9=*Dp#-kpe}F6)5h(f)puGytqryBEemYQ;K_$;BLhsI20(fNE6(pxVyWZ
z@Oz&3dEayHx$FLYSJwKjkv-c!d(TY1GfB9LlFTcNHyB7rNUvl+N~$3tA%l^Sp3|Zu
zBU<VfBnl7>NGb{%Qjd?1+Nr(g=jUwx%?AeuG@kW)<aG1%^Xu#DD)B$p4lnxq`{9fG
zPTz;Pxw(@&{_gJWPH&!=<-n_l*W0JJS$rD{dY62vM=P)io5wbb{w!;y^n^D~9bepV
ze{H#Ze7wE6dya(cS3UmW!w37qfyc)yql~`O%UhY4&djda*w&fIrs>D)vB$^l&NGBA
zi@~+ykB`S`yqG*ZCKVNxJUr^IB0xz=$;acLg9vt)-H#jJcOM_Qc|`Pq+ydO(%8A{a
z!uHoNo5{&ZwWRLl+sC(b*2ipgLZNNuH29Yq%x5C>OH0cKzwFs<dlGR1&;Q<EFV)_R
z_^sy2P2E0PbYEO;{ZvYty=n(R^?>1W!1nEnl$2B}9`3ohna9KW-8utaqq>XpJ?Q4+
zK%k@&AJ60EYX0gYmqO&#!J~S?>OQS1=}WA!_2UmUM<=_(dn^|0;ZrE+fI3+`EWVVz
zu2K$3UlUQ&eF|W%I#-_!V<ZrlP_sp=>1bqRm|G|JFn?W~hg*r5Ks;dda-{rWA^pZr
z>g4C~>SQM&lh<54kmD8Vz{-(L;o!c2_rd6%S>1+U)~b!2&G6|-YAt-@Z^;jP-pb{(
z)19^R^LaEp6)6_EoXL}Ylg^Z^x_y<(tn~c5RJr<-$IY$n$D`2$_aOk0>P5}Ru*s{t
z*~ooLiT;_z*~;Y5pf7*Fn|}9Hn2b}3Uwrt|chKD2EbI0if}3=0&o_5+^PC(ElrY#Q
zXUT@HrgqLldu9%!XUr4@PKsfZou!`*C4ks``_hRwv$I#lf$x=VTWFcj22-XSeHy|*
zs=N|-G!nnMy3+paHOJTmeD94i6>%H9%6lEy``wE~rtm~VHO*u!%E_=SBXzSQ$SfRM
zF;rOu=I<Dp{qV{2NL<A2$J%jdRp-RTfpy+8DRuvA@{_@ls|d4>!qfwaPwOd(@Rmqp
z_t0*kFG{LviR=|epM16-*GGTN|J?f3)@mI6bG-*z@X;dLAttuI>%2PaZ>>c|k4{Y5
z!W5Rq1R^nnn`9-$G(OMnWhhz_h`kQ@#iW`ia)+!M_CieMEa!Jc@c{lA6@E7hUkXo7
z*|YGEN?L6s#DvbcSklp*RDod;Z&q<a0`alraHT4ia_c5nWM4$S^Jl*?f0r<!xT@vK
z+kH8r#J6o`j(vM<zMwF*exit_NQ2%*Do)_bG-D=>8|0#J(OkqGNJVb8;;J2n_hOlj
zdWsIwo}fSjM_-ZYgNpvV)<i3c%G9iLr0;_oLH2&SX4~)-HA$P^JR6Oog8O~bBrpi8
z02i5;k%+~CJA&e@$|2X-oPkvEj5;h)ix*rX0CkVPcirnm0o97<Yi^3;jWoJc@UY{~
zX~3NNM2^Q1(=A&>{qr5Cb6ZGFW9&Y;s&c2JM`Hc&c9aDIkaQh#J_zCuQeD$E0{zgA
zjNoyuD80hmYgBFgDBuGsp5~8_Ix_>wOJZIK&*<@0U($l4bsYaf2eF5@@!o9*Km=h%
z&pWpkYs*bH7i(-y$=A|};6yi{n==5k8BhS}7s2z}_Z=N?e@EaB%KW4R8*l-xGs>Tz
zEZIM3d1)po=VW&738E~;C38J6+n&0*6~KOUYY3t?4bFad@~z$K+u3v?@nc9F`Lub9
zAs1D~^_{J=GU}lbssBeM9f^>>=Gt<zLdVbMT`Y6E+!vgWo<8ZJy~f|!P9w5@Q|W;k
zHh<#=xNf7tA(iuP{p*4&eoM4)bPE8h7&5Icc~5#W^_C(_YccY@+Tm$DVZE+LM?6c}
zHg#NR2e~ih(c<|ArGzz$#Q5cLrppMWzfv0|=Aimv#|L<PHjVS5FpeP2ROLA<4gZFy
zTL91KrmXbZw}XhgkitcjK#N1qvDjz3gwy!}XY0D}Q%p0tw&gtt(ai0ruhkN~Wgyjx
zO&Mc)xWeILG4?Y*6L4HYKC~RSfiVx8=38SZoLc9J`+!||@RM8P5!ts%=DF-3u6G1t
zY-x*K_oML2n5^<Ff#1*Zy?VquAd6u;LbW^CvheD`pI%j}V;Nlbtfy7rH*IpsP0P+9
zxw6c$DO@JPIEP9sM*+mr;Xo`Qk+u4U0N*po^VgV=P~<DNHb$2?B6yHlFRzu-V=tve
zlzz4ERjUm$NSfaSq^-cRG>Wy_lh0ooR4x&WJ71AuMY^yVDW0pMDjU{HA}X+^C@uL}
z>=(wLzWqMFAwl|I<&fZW;7B!qWP&$~5TmdLWei$vud*Y@<cdE(gJCn=ayGs}B-jjc
zyW*MH`a@nY0i-7V;Mkga&&j)b95o3JQinWy19=}&^hNZhR;S@5R@W<dJPALaHA&-d
zYt~bytQowhgE;2_Ncd<S);<zNM>kO4bbAT^P_U2I7i@!B1m3wVeU8)xd@@VnPeJ-X
z>RqJ$8WmU+ebYQjpPL=z+w(c73N9IT)d}s?%b4i`-V^#O7&9i`HM`BG-<O<_uwhm%
zv5B1Pu*q)=w@LetO|cB?Sz3WqNkyEN{66TIV?$?v>c9xFBteXYs0w#G&(~_+-jq))
zgKx;Vd#cZMde^V)T(U83&LuxXj$QWRJg^|)YQ1&+Jm|89osrMR+k?UUlNsP15_{Mh
z<{uU+x_md%k)YF59}qY9HUH_@*IC+rj&XzSsO@VXpP}(M+>gmps628iFX%g6Xzr@I
zpz3P5g<#9i&*5X>NbvdJJ_(#T7Q!ff3wd}oMSYlnR6YVzEjSUR>pC~YEYXzC*3nB_
zmyNm2==V%)w9^ki{A8!0((mIftz{47%A+`gmEOT3=$v)OWWHrt_QXkYcWbZBw;YDf
z-++Y^IL-$?FmfjNSBl06y#7nbuWJ6IN0?)b`+D<+%d?r#TAE>T_PP0qH<7mrn511R
zseY-}OF3GMPY-qZ3%59MmV?&T@ggXRzqbj0;3S6)xrtrZ5u+fmW;s&mr_UoP=cch4
za_wTM)Y~{g)Az72IRlQI%UvSh-f5<2%kX5<)hN!7UsdZ~eptwG80Rl>kWL0Yc%tMq
z4(03WF+E#~J40O?s-ku5Lxj>A6D0Lp)Sw?Owu!E}@)WL5g8*k#3!P4@L~-y-p@&Sc
zg0H{;+)vIAeO0zq0)l5+`0BAkp-ngV`t9UF95{%=&X$xt-~Q0t4oCP~Ky`jJ8yVy9
zm>arKA9T@Q?XRSl8u_t$AJA)unQQQhl8rVjTO5M-yXh-{79G=U=H12ibciu7MgWs|
z$A`RQ^gVI-!zo=oTOm5ir?z6f<t2XhuVl%cPWq_t*uSp&aL8917F;@$dOMQ7Z|Vn=
z_kfR<fXPlNAIEgJ-J%WC;L}F^%9Fi>-?Uw$7xa5BHaJ*m8#Pau*iF4plg~fyN_vpa
zr5hDA)G3wu8<>mwIj6$;qu@)w7Oq2h@Oqr?sIem+$mK>OKfEoE!Od`<?t#?eHxqxU
zpQOxpzpq(8M>DC?dYvfg%|)ZbF)13y)koY3U`-^Huoja7kv52PsJt}lTh5kNm1em=
zuy=7!Z<+!A*+jnCJMheEn+?6aZ})JN#f7Ws^n|p&wJX>@8IlP8%mQ~kf`;Y`C9o0n
z&_W<e+)!!2hs3d09;J)N{WQpEQ7$CX_CNDZ2<WE`zv6v~`vUHo)BU^|2CB_l<7p!t
zvV+UDY`@%TVrDz#Ev$~Bi1j;v|915!R23(H08;N0ylRIoz?PZyP_slGTRigICPvZL
zcdor{qeoAO26wc+yr8wKmD9E<M^moRLKj~CLrxfa;{nB|>|#QmmYzOhgpgj25F`>Q
zgSdbm>N=CS;RfmDH8=?X^<Y91Mx^9I9L28Lnw#?-u(>2vfz6EAJ>Iqd02V-E;^F=0
zp#qKAAUSRw<k2P{ayepDKC?qzNGe#ofI!&}WFl^p>h+`f6T!DAPKrb=22)nPuHC(?
zDO$R^%wq33<Gf0%{09OB-Rxl;<2v`@bx~WG1D|o~E9xG*NLg+;q(&gTJ*q1e6q11l
zpWf01G1R{rI|JvxtgnOAgev=$*|5?R(o);OC&>nBk|-FHTho=&QhpT%-=?!fxOt!1
z4Dxoy{;{;g{xP*UmRX4z^s4pSn9ajH@Mrg}J8YCw={seWm_5~rVT{#Uo3TIF^B}ye
zOH^ZA&&8Jl^#RBDwnrhJ{*)m%iYvaJB=af|unHHpC-QZQz}nkwShdNE>0Z-smYkEx
zb`ojr0Do)vup8EP=m&Q5pgK10gSlXhAbs?qGnuH6RrI7xATx0@bk^fj_DjP3c>}ZQ
z=#kz-^kE%T9Zfjr>In+h>KU8cM{1JmNhzJ}pFf&UUKVp)dgaF^Y*vLD+n!wlFn%eC
z0e*OPJE%j#_iI(TdeUo$qgbRhD8yfMDO7^R_1F}5V-wrFl9}jnY^S?3a>4AJ9)p)R
zfobguYo2d;zKz97b~AlEca*OA6Q!arZG%n;OBPLkfww_6=>!Ww9jUY(mE8{<k5PRr
zoGBrh2r%W@Kl}QjPuvJK$wYWI3kJ>qQd(NtjV@5Ycyl{@_BD^Gyrx`ijqo}}$4OwM
ze7lQt^&RgOBd@bm9)+BID(>pa)VU)QoN#pvRz@g{S;KcvLV14H@xq&d{`%yzLJ7$;
za^;h7a1g*Os?eR@^v?NB`7C}JN%Rk9d(v4|)p919na&??GD=OxbsI9CS5E?aUMhmz
zSl~X9#=aurfqH?><yujDq`X^GIz8AQcJS`QO}Ay_Ro;=el2sH@s=C}&^3+5fZ9=wN
zSTV{_UlC825mp?&+j97`9oo9egeIM;N~+`j$;joW@0?*>qX5;S7P4DWjRLMV3wtS~
zJ6ZD*5J#-DO1MhKZcl2fCMP&aq0w$;PDtz76~Wp2(d)wW6(ib3+t+$tlK8t5%5HjZ
ztCkHESL@zyyDtqEY)w!pJ8>11PM*zB{4T{`U*}fxH;nNX$l8MI$j0z${p4j`BD&|J
zzb83P1J!f1qWUj%M=rc@Q(IOHCFzPmA{tCheUK;BjQS1iBL6DREp@vWYM^-H*`XLn
z@>ar}vP({Fk+D&)CIoBnMsGmwQ-`F+W?P2R!V>Pfj0`z`ze^MuHb&C$i}ECj{sI#^
z2;K(C>8924Ylv#ClGrn;6zw}1h+>G2bczHn{5!y@CTHnOblO{!nLE=gkomr#vZ#J=
z0)R~A>upA|bIf9@yz2cb5nP9Ty-w1FS*8HYOo(RoTx3n<k3NF+LlHO$bGXA)3_J_5
zBRyfan)@s;12x*q)UzB@A?f|~KF+ePU?p8Vod1Y@SFjVlQOFOljUXdp;AgXEQDduc
zhbl5^p-2ULRgE>aDby)NBX^^bii9;Xt(fHi@v<dTrND+8M{urmgR!316gKIHXuExX
zF0NK@2a^;0=R}dq9YrsiZ|-Z^@wdZ$4?a)QzgYHFbhZNj0Dnn<0*vV13<J$l*jD+?
z*Oo@*BhqWlgB5LNw4MimwP?<M_q@oSS0n00ox5!fddNP8@Mcp7RMlKLZmS<R^O_i`
z(THFWe5l_W*H``f{!5cV{<x^ely~=NucMOy9cpE)7+j&(UtV}!x-@Ow$*L%8P`_4r
z3Q9E&mEU)G`D`L^U#-04-UbMF_u;?$aXMHZCX7pDoi;>ZP1@#SIvYBS&1w}OZ?wwZ
zX~7}<XX;nC*CG61FpiRqn}R{%C`7`H`2sBZ$q4nVKEO=|UEsUU<oszZ>-U-DYu{+U
z#@*>s6@C-6+`mhM0bq_FD$^{;n}6DVIczC=<`81&TYCI%m!~Q)?tNL;Z<}&B$Hels
z-B@i8uAAta`j?=#2gJ90FL$NzH#r&w?o(-JPwW;70Zaf{NJ@|<d>o6HdtM&JJNNJ7
zKO>RM1DpSHh0N#2Z@WfOX&dSHUQJF}ys{ROMuvQYZE!^Uor@wh!j+LS{8DggLGFjf
z(-*|>pof(6p)+J#+8YRk;IsRj?1eX@TwOahg&@FXJ2ggKBi#!&Zc2vQy$^q!i^r@c
z>apI&-;-plsl)3iOg%EdE6(k2izp+M2r{(<@3+vI#b3@zttSsH_{94+p1xNQt@QAy
zizXTd*5*2ZQUS8gq{zR)HHC8K`Yq@uRj!;wm;65<n!2ywW4f_Lp1Hm}D>>7O{|!`K
zRyn7H@93}OJG}_2^XxWwp1C%~jDk}qD2;TfUPW?+O4c?Q$3&#qqiZhc%C_p+P#C;^
zEjF;hzQNB9HrMYK5Qazf3KtEbXI1CjLSJi$hmQqN<X;chMs1?Na?H7th>K4-VwI<W
zc7O9p0g<jv@$=&F{?Hh3dB2ky4~tVqe-8+i{5#u&-`q({tKG<eja=j9E1Ean{(6wd
zm4@{L(;}0qZWs4g1*iR9YM6&U*aiG=V!ZoN;BS79a!X?Bnv1vs64G>foql$njH<w?
z(moAM-rGdRV(`kFf29s?v#`N<rvhh*DAo`S2t<kX6YH6{*2XBN3?fjpj>unbl3@@&
z5;BU5Vr&LuGm;5oOM;3`e*#G;UN-`H%!`+Jg7EY|Ps3h+_!9W1kEk08Di+4v#w&88
zH>MA>XkP(e$ItwWO&!)B-Y7l3BsNyPt2yZ1o6m<vFXQte1Eq@=$nft}C^>JoE?z>2
z#&rYoUj1tXnQq7$3A3QIh*WRNIJ7hu!nGb;wX_g8>qz=ZU7h?@udoT5gmd&7M1UUR
zCYg^7Ny<2e>}MN~>7rIp?vs>_$KVl(Ob&ciuEq}7;Hqenc~T)>55g!9kcvND$xfd5
zY>O$^wbNei&a0!v)7(;FNucAN#H?LRNtHW#xo*Uq_S-$Cv4lA$WcBR`kZy{K83GRX
z5cHH~kQnm&%+AyqRm=uy!SQB;OrU6{XS0l57Dj#o<|&<QkJV~j1<AIr+gB+nI`NQS
zhC!&CQDy5F=mf~s2#j5GDe@t5Om3t|x*=w$%)Bt#zz-)RzI<Aq5G!IKxdzVEe>#G$
z(Q@X#0*v-YmUrPP3P@}cK;+XO*MSh~5HUE`uRH~G;OjapF(`I*6csu!;}e?%?yG4o
znvak#5bBT5ElC5-C=d_207Px_|7^N9Is9v?7qP<cw-SpV%PP5or}p^w1xS-SX?F6@
zu!e2{H{VW8O*BMwk8()THr(nl?j)27?-1jZNNwbIR6ofI*EiHtogc5hg>Q=b47CaG
z&(6D7c)AHJn(fbe9~^o#WDE@obS%ts5NpGj<DquiRajSU2F)#O<eOE&#>I~JIDr)A
zT}nyJ8Se!XoVCEP1_RG)@$+4;;}ckeN=C;U)MCf}2Q*ciO-X}fb^c~?hQe}KqJeM8
zPKu!+*y+KYB%|y6aH2wQ<*?Y%SyG`O8VJ@lbd7@&YwKimP#{L%dn;~<zn%a;x5|I5
zR2e7BTm}RqYfwxe5p6nzb~5Wf8Vbkr$94VjrW+4a?;%vP#&%UAE#6y%V-^S3Ml*~)
zBQ(df@w@zB(a0t|C$A{3>#@)jUYGP)Tu0<WnQn^KlScFb3o!za#S;t3PWLfZNZ0!k
zq4!BFHHVYIpA~EjpY@$T&<biE?RS1JB9WL#oB++s_*Ubxw!lhwc9`A$aUq)CK?ZZN
z|KV%mH9VZIqW<gWJu2v=Orx#|?dq>8cPiB%oDZ9PG?k}Lu=sC#rt$AOBoe`QG~nK#
zYNAk_AT}G?zu;8O>BBq)4=j)42Nh}(<A|~h{YG%bwnsS$cMtmT8}UP=1xq<+_EYU*
zl(e+0M&%i~xw@u=VfOi(B8FIkM5*n#3PNy}po@=AcgqKt=E*K*dGTuNpZSeOedlh=
zPCGlL^_W*8TU)wx1(d;r<+piDObqc@uU4ZL&AEGZ<Uq3|s}yQ<Yf-lIBqPy_92m)s
z_%xyPt9U%`p(nG2SjZHKp2nPWDd0Y$pfhFnW!GOPHDV6B^Ydy}imAz8r=mflAm^`c
z#3R4HnS}GStWSRX^K7XpW5N0k%OyJfHFLr?*uK%qu|L!0-mID<sPy4Sjzfkah^0Ty
zAOHRjNh(I|?E*j8h!(Pjm1lk=TS-#J&3OQyyhnursCJT+!=XXNzmqCi-b2`y66t9T
z!fU><r*>6V34Hw<yJ5jL4eapl8*z)^thud;sC}0_YwZ5qN?QQ6<yhDy3I#l#u9hZ1
zLq<Gk(C)UKU(Z0q=f9a6q#Z@F^(n`dK&w9H%i+<~r;@f$=uD%FBK<$CitP?X58pU%
zyxiJqB7G50W@+9Tjq=L1*V()bwA;59!nOTjS@PM#YpOsRuu1r@*UqgIIPUMZl%ee(
zf?Y!0+Pv?n2GPJVS@x}_EzS&To)oDqKV>}<&RFHLfDNwt9g%S?nyHb1@?Qb|J9G<E
zrPgXCx+`?vM^ed2^?WM^l#vK3a-$78tGB#jLHcEqsU2WiJh2nM-{0w1)XDxdGO0tX
zdQ2lh^#vhuK9pG*Jk&y=>P1Wg^R`W?WwOKPghif5{LK{*t*HJHw;$~(1;&RV)Ih$q
zI3V~zW`l%P6~d-YG(#f8^fR~mg<J;kSz4l0X`B)uHCw8c=Gqc}Lou@NckA45!HA<y
zR5Lw~8I$M281*szT;L3|B8?m(`6%v0=CUDn6|{MK?j^gBm<%)DZ_~{ZJPr|sJ2DR+
z&bjVs))n|geXFqM#AbL!JYnCpAjKef#u$>6{gOa~wvcNTGOr+9PMe+~IoJEhFA20x
zs!RNRQrAQsRhZZ}R6kg&ZotmEK~oKO#LG4;ZZ%ic{jA>gLpX3L@o^S#$uPVJ<2)R#
zbf}!vnHI=^lHkwx)|g|ow40b26eW-q0@k+VI!6K-V3<mul&Lx3e2FzZIkp5L6sW1M
z`DT1)sw+?JjYC-Qzmpeng8S1aKZ)MnXXq{@TRBlFfczv-lt@|6!|G!cvd|jnV2<f`
z-CVe$b|&_XDW&u<9V5%U)t<|A#qyq%mlBaAsY?m&hWy^@C-n?Jc|b694JmDRG8cnj
z^9lCoVD`U1QCNtR0}}-<Ke9^u*FUSpGGtPf&f%%88!LMj_JJVbZJw+PsFt)AoU4D&
zq`vyok4F<@)d{a=L7^Gd`FcY_k)`f3cyKw9xzC$LeqkNWz^LZyjhBP|7oOx`asweH
zQcN=5VS+Nh*V|~hWWFC$fE$YPuEYmPd50s3%P8n@)5MxdJ0=M<T>M048YRV&)H${d
z>TuvBvFw4~rb`4EMpP;O?lrfV)Xm(xH=~iyra4q0MCdQl3Z5MlIPN8D=SCGyA{gbS
zk>=k}_dct9u?b7`92Zdu*^IWt2$?IyvJ`bQPqUGV%xLeC#Z1nCelFGJQd{tUVPN#<
zTl)BkTRHmYlzbaHe%o%Hj?>k0Emmn{3LDQFCR1YqBQEGp@6NTg6?Q;ZBA3IgTmE<*
zw;eP0NA^ZBn2t>_A+6>fA#H)NeWT0qn}^JDjyMm-`$9Gd)31<n9ajgHLCNZ!0@jlw
zeLmedSDq3=+K(iqr^ZY+$SUesFd%Xq??D7!{jDYJNULzdXh8)0K=h_S>x>KLzQtsm
z0d=>IDn@f})6>l{udFJgh@FomXBhuAm}wkSDTUO_`c<90-0yu+H*kYQJ~RjQuuUkt
zofqmv9W(5qGfh`Y`5L?4486i27~^*Z$M49*4|dkm>F(w?N1juh?aCi-fvIn1k>bkR
zoXSkDKOzB3acnA7?gLgdM5#91%jV)mRss$D<IhoGKNiY%TSrONQX5KLP2M&7MmsYQ
zRZq0r@_YrT!XQ8D)$n25<xxbQ-8!bn$rfs#rX#*lmne|RF%a370JE%p#9&A4O!?K%
zM%<Yt@L|QFS7wsQsaEbd+3;D_gEow&7V}_zBh}}`S+?CIZ%i}rTwS{jB_EM*6H7c@
zL-EVkcF7dW<Bd`jiG7+kgBi1f>w9(I`GwJ<jDk9=nVcA#IFMVOcl*70c>XyyAJD=3
zwf&p9j3w`B5%1r!p9wnR@^?IpoS1x{<LutwlZv&yLdUnAZ-{@S*P>qxu>3@h$eFEQ
zGx<k(kX!Mvz0k75bWPbd2W4~(n=liG8cAnwqp_AA{x@;-r=FmSLGdz5d0WkZ3eI8u
z)%=NA^B+>#din*}aU{>phlX!Hz=1aXL+~^^?>iya_M~-GRwlSiH|-dfVNG(GLkaZ<
zjY_}rWEfQhZ%^IS7b5Kt!nRJABI}Nga~Fl^To-;Y-i=3P)$t|RU*}jfDW-}!r6@tQ
z>1M>g>o{X2YzQt*sdzApli4j5HuUusX~kOPE}$-+BWVz40#`9J`bwi-iQ=uTq_#zj
z%YJXx{dA-8!1sE{D+6iTC2!!s<6q`UZFw|awmMB-G0b4j(6}V?;v}D%w*=*wl#Kk`
z))JeJu)M?{yEZ!p)tL02!AVe}1yPUA_Q^Aw_ls<WE`CR|v5y;q(tw*Qw<v$DKdtTV
zBCvSb1H11(yXi7_(tKjQaEi;hS<jhveIBn@I%|+Kph^IxnU3P9$4z+);^J~mTE9h{
zO3T9Oig#+4?UZ+Kwe(Az$x6oCLJ2!4X|+rL;&Ztj_VyjJ^(O%vFBNCJF<qRTq{3QN
z9i0k8_`&-6sw;_3PK6o&&*Ht=W{e$4h%pj6#uq`}x1AD}$cM|S{N8G;@sU&^VoR0a
zdn#6OL6)yyU{smPV8W&Z15_jZsJJNDrySr%88D{`X+b`a<U7`=Dk+u*>h|}&XXjCT
z6CCv&LPo(alh(CO=iuYukM}2_Gv!}wW_-oay%=30ugF3n7S=h{6jhLWMfq8nDfLPW
z?1JgXOudJ8aV|ghlBlE-3U#D^fI%R3h1!de8*wR&3JG1xu+fOuWn5nk5bwvQ8PX$i
zlnP%H)-0LE1&~^YHmno{hJEcTe69Nz<KRKb%hZ09Ko0{?elU6Nhmv#A?Xrinv(7RM
zBJ8i~e&yK77>a|Ii$&&1f@l))SJw(72Q#QG!65JSXP&=Q0#QRd8NUU}&s1g}ic%m*
z==5jyUgQHF<Qz{n`bwes=sTVDies=CL71G;0>xbtaZs6*M{H}Q9kTZOX#JVLFSS4=
zjpF^;`%Kf<)#BsF2CB3kdDS1WnSn=~kNc;A2^4caiNezzrU^2j3!Vk#q?g1Vw_*{Z
z1-?qWBG+1Lu;+ue;2EqMXPH1*XR~1n>xdVxil90>44gDYyG_1YJu-tA`9D|H_4P$9
zeuu$qjV33BO`@oQ@}wK_4$WSES-DvI6aZ)N7xM-v35|D}qN@4G1SW5b0Z4@ujVJ(W
z8koP=gGULT$&AuvXn*}85_GqUQVCePVzt2?P5gud<{NQASAK@CZDyG$Be}oo#PQYJ
zRZ>v5_N3R8PjS5F#W5Zi+Q|jhe(z7|ds0=iIw8)F2^T5w6w`dvaMg%ATM3<CZ@7R-
zzP+MV(EX_?CZQ%fUC_k?ZN%PEEOD!FbhBPy_ue0iWqfTSpJ~{4Y_%-TU0c*B_0Bi^
zuH>nLbOY{<p{N8QGn8F_G*|^ap<@*Vx+W4<+SPYSqi&@5(I;L3>Jd>D5r(5jpKwUA
zWC~D2$KACt9STh+pv_72rwwxEu|(B}SFB3B53|fVo~=f>gZt~S=Qw(x_&Cvi0CL_2
z)31D;A`}y1s;tV9qJfLBZwGov!osr3%1L00-ybbR%Wrc_7rFz(u8!774Snzzh#LeF
zRZSjXoO+Zx=Iox{l4c|kwz?ocyZtrThWrsMXIZU5`1CbZW8kF<IFlFlJ4E0Bu0qd)
zp+56goj4j>2j&h=JjN9#n+*F_5hZNmnzApnth!NQW(z(OtF#?_A!_2f0JjBue-w?d
z{tM$&_S!Bo>|Ulry-QpMmquOi(;yqdiAunZ3e~CW#S3sh&`;=PB&S5Gqc^uUpHX1;
zeONB5MGRh&sz=i$OXMY8orZFeZosi-(K4ZznicYz#!K-usU!jJYj@t(&F$lUo7RW{
zHTPqF1O#^$CuM^QBD$Qn^W{Y!(Ip%DHDx<Y;soiiexS5Xv?idT@!uY6jK9txv-)D0
z7z)g=ZlM4BQ3W>sjB<SWdS|bf89Y7Fb=&Nff4@Bar<v9s?5}5>G~bK?o@*H}Y+NB~
zzgQKGk|)im?~sGrO!}oXUHqbwY>v9Kekqunt#w!we!eqF8zC7<oViHZSM4j=r%WMR
zs6k<3b#g{Zy@M-Mv1#f1+MC%WD)?wkd$*z5d{VFn?G{D5+?vR#qFRt@2`Igho^=Tx
z>iEWrDe}NtuF=xPQax;J(O0`<ZQRb_Z?&pM^jV}Xl$Q60l&uIjwm4jP0|pU%)FQ<O
zq%Rws9;av4zEq{bTD?l3mpl`5Ek#fN5O##T|LaX5PCh-3rwVI+7Zoh5M5mrgl#gU<
zsh_?$EFL?z#x8VzR?rOhg=a){-iI)zJnw2;LNS#gp#z>uUrYM!-=}lG2K$%S=hv^V
z{TDoXuEzkZ>JU=$n+<v5EoX@0eJ+4o#0-RX#si1xLPoqru`odwhcoZ<PXRW%$s<DL
z|I|~tcucMQ_qaxHN&Hbk=AX@N4-z~u9T@)jp@XDxr4c7{VMKkvKu&TqQn9)Z5WJve
zDmZ!`3N(re3k^ZQ76*P|%#R?j?ED0&P-9?7)ElN(@O~W&l*`7DyaYjQ(>$*;XmgCo
z;jjo9@c4_c``mt~bapYcE40bZ%lBzV-}O;{3o>BX^2}La^3Z~dvrcz=zfjFqJGOwp
zQZ9DSK7+NCM*cY06%pttD7_2C(tplO`K%g@>TR34LQ3&`4ELdnK)Z#1$YW2_=`mYZ
zOZv~7@~&E&iS$pR*|Ar<BG9NSdGlPeR|k9(L;<cAO$B;ARUPm8074j!M?QX;Y${0T
zn^WhLP<WQ=K!Xg@!VdadT{!>LZT%Ip>UWBU97W4<(B&@jNiO<(9ggvZfZZP|>PI&?
z851c?$dWUn%S(4DeVP2;5=5vgpZw|R8g7Q6V=L~rZXkZbFTT$EZb<;z*_yxQT)vZy
zvnFEa44xW|wnn~Wg29jy2FOoQ{fO{6@4s9uS$}dVx#lZZ#x*~4ZLK1QQfp9T*8x%^
z__<N^J{6;E?o-ChB_tiPaeKO|%%*%^`AhWESiOC00#94erc;@)s>uL$VuLNwv{KE8
z=E;S&`~q<~#W);f@y$>VR}gPT`_c;194}yD-U<Pipc`@)*S5da+c>st%aT!bL#pa?
zLV0_R^GayimKTm%vp!&GIt3Jvy{*+HIpI7@nu*y2UJUM3Gw|IAk)tROmG}WIooZGs
zt;u3Y@)727&6U@RSFzc&Al)rERgkx_Qs83izS)H;-p&x#kc75FqZB8+j2Yxp@Xmj)
zvs*$az`G@O&l%CXR2Art8XKdW*1c+iAOckmDXhf{&6oinx45R~ZNJ0N$U}(#09QN!
zV>XdEj(s`islTbE(BMG5BBZb+LK`EuD8pmYbnRczxHLU$h*Wwl)(Es*vG=pjmsuYX
z33!)hC<NSVFJIC);iL^k3Zs-cQ-k1jOmOYgceh9#ZPG?No6XZqCkU96N=DL8KtBz!
zrMKJ~D-w(9vWvWS?gyr~utyq^u{Vz<)ylecgcw9JW9ugiBqTMoQmP%}H`!MM<^l~X
z2MQW;Y-^Az;edEATZ%bS0vE0DNwB0I(Yr%G%ldk~`|=Isp*N6VR2__<-NO{HlE?5U
zB}8%l584y~-rP*@Prz5SXSwdEd#%=Gg#FMdv>>k_19{uCnF(QOnbu67^Mkp}-f~_)
z%8#~Q%&3{OZ)q30eu}-=%jjMEU5rfo0hi)%ZMVLAmTIDaDU>d&bUwH7CK!bJu=6=v
zXVO*sM!U(;VDjaSfmPt}HsVP^`~G%hnLx{uRM=SR3_=elB0GD@n%RIjhxb>Y)RD%0
z6!=f7Z}kIrU!|4_PeN4nhfOFFj;RSm;!d84;m%k0b7;J7hT1>f^!S5(84N(o1qQWy
z!T5~~1Wq4pQ-z#|<Jln)_Gbvp2m`3X?91UMCbaOb?quiT1NI1gyBBQU)M(6u?NEoy
zO2r^2q+Wj+h~(><2JzR5I28`RXI-InIWMr@TB1egG%;aBNYZl%@-SBxm3&fV6%>>t
zmwb&z{D_1{_`3DlV!HEBnU#{yNkQ}0%qGp^p^E+clUv`<UY^t#k~7bWIp=@!y)#T#
zfb44@S{K?3-s&>5O1=(&Iu$z>9~(UG?9I)t{W{4vzI=bXF@2HWyV78wm0#gh?{}@K
zA^!T^n9aM|1+}>!Fj&0OGI%x5mP6Bta@3EdjD3TFpsdLDYvJS+@+vpwyD<Wo`QFrG
zwy+SlI=|=cuTYNq;0N{Q9mZDY0Vn@S&Cg$;3p4bm!RO{ttYm4>d5mm}Ju5t?Y&bsi
zC2}3vFZn|*kqS))6(X3yEN|1_kxp}ycFrHbT#ZTUL8#+_T6VpDxo3@7KXHgQDws)P
z1|wI20M_ynOd*lVkmyDL6AS?=GVLOYSl~33NkowZ#*<r~t}@A$c)x3oQvwp^Y-bk-
z6{}nfMaqmfod3MKo9xhLivjW`T8XDV@`0>(Rg0c@NRYx*Ho!i~DjT$j9z-Ps$g$cQ
z1f+Zth{YlVRje@TC!XZW#RB_}HuO0o4@5SlaFm-qkxrr`gogkVbNk^}?ECW-e1ZRF
z?=Cs+o~Q8YfAr?|%P@@vAxN>nhCLoH!6M=aI}jD7Q((J1(<vT8dqhPdHTl>;JT+FO
z=p~{;3i}ath!67#`rp+&g_%2)Md$Q;{KE%WWm{3$fDLZv{@?iM9<AzGlvGH8KK`TV
z>|?p@qR(JE55T?zv%YKK7w};E&`3^@`$>BM37FAP^ka+?gXEgVh7a_8qd!kJe>$l&
z!bGAJ=q{y{IST=egu3T9#=_h4-gtm=D=dE9=Q%<}5Y-U)Ho`^TU#2R8d<dXb8_zOW
z$nBx5&A<;=34wk{EZmR>;msIj5qatw0Vp83CL)=hhOiS#S_JDbsPAbZmz6NwxxqWJ
znIfuE+8cG8X5?s!SlydoqTb~L8_|6k4YIX&-gxD=l-aiLmG8Lf+rKv)Bo%HDLsu^5
zTvA<th?v&j0C4+5UiIj`Sp)3W*Y4T6Dg3Q;NOZlwyd;U8;K=K5f>vnB>#0tf!a^b%
z^mz69<#)s@9V?GLXJnj`3X6yso<KaF#}UmfH$d1l-5ZMVN;xTPHW-)~?CbM{MB4wQ
zi}cN6+8o;xZkSrBg$*eGK=%6|a@T**{3n{8P-cB%_=NAjS}Mp9VFv9`cc-sx5KDX#
ze*bCFUOXLj6)B1li~1$YnfhIdK~kxyi#>*kg(kU<mI{ZKD&EGQ@cv&d;ad!WOy4un
z+$0>SMx#AbnD{#*X|}{(ghIa(cN_%16;7W}oWI?uuCRhI3?Z{IpPj{1I-Dr$#RISJ
zA}|&|{70$((k9ndF!9fSYxj<c8xk3;ico4M`5cz_-&(W%V;Mj^^-25x*z+(H4U5Wo
zjYy&&(6sG~2VpfLk=Dc#HAF0%!qzm0A1k`Lo~-YuNtFG!5c3Lv3ZqseF!5S{hNff|
zKM_><{68$yU6>O7m4=C)`l-~KU!TTK2A1j|{l|T|<s>i_El(Ksxjvi-2fKH`o@I{v
z_?S^nUNHQL`YqT_$Vidd`-ltN?lAk8R4B{-$!ha-D1mQiaU#W5bn2nDG=`k-5Q;j;
zSJ|Tf6eeC_LWVHv_7Gl_WI#F8tIVh1XT8%h6W{Jz@NWIS4WLUuxx)^YE=&9Zg=t=F
zSvWTpUw*!POFuF(6Yuvo^SqNV|29ce7NIg8q24kzxO?@is-&%)*6=UlFei;*Li%?p
zkIYVNSjD8Y8}k4(hy6ml*4jW)Vaj~YQqo!@A&J*oi+9K5K24LuwI;`WWKTOcv3=WQ
zmj|x3W3sqsh_dUF`x>mRtjt1|@`D3TZcBO{FVJn+W4Li=NTkvoXgC8O48q@Rt^JjL
z`IydiKxQTLWb#)inp3S!W2<12oy_WS<e!yO=w(Dn`SxA!xPD@=b{NVNk~24Rv4Wa_
zf6Y$+iV7%(YJ(HkQ<R>t*tENHJ^8gf1C}VU>D-CHIs^(`NNyV_IE;>tF1*VFcu72s
zP6B~eR7DzHw;`w<Ott(S|BP<^;r4@lqnK410n9e(>XiQOTYp7SgxF$>sK$dUyZBf{
z>S@d>fxn2<SR-hhz;g`l?*~}Il@BotH5#M|G4rv&N~|_Jv%Y{@|4%kbmuqyCU?!I@
z(2Y*_bvcnsp51|en4`dEQ872l=F2V-lPV$>H`RC#G_(-F=gGN!=y&XZT5DRj?IyNi
zzj~EYm77ZfyQ>5gL{y};LD!9I&Y9A+AhD0ky-|4JLMFYE2vVlg-qWXG5{89{Vzn$!
zbxru1JVVq8U`ltNQ@zIyLpH(2nzjfhYhA<eEf@qRV_yGt>J9@Yda8n4%<oQX5vh#`
zc(A&b1B?8i-V8o7G9rRg?>Gj-?_yd{HHze((xEoZk4M;N##l*Ig|But>?yEk=^ky7
zVdj%f9|3^~r{*Y>h(e^E<Y<96kY4<E7GQxJm@(11wKU72(Tj(~w?}WlOt%9I_dj?@
zd!La%Mg$FUE2tLdF+AjyoHo9tbJK_+_hR~}Q671Zn>75%nm8GK;m!$fKQB{IK~i!H
z{|mkwZ#drPu&xfOEwka0iNCprm(0oUhjfP)9B?;`nHGJGd?+_9^u(Fvzp*|#B-PiO
zN$xBOJmG2vcD|H*iX_nY@`KYd-bxb0^ud}aReVBh6k^C}jF?721?B$!2TOW!{|o?v
zzq;fraW%y}c^C+OB@{_fcK;N4<*G!PeGa7QSGow(pBRryKK%U+80$iK?hmZM-oEi^
zLaLQc-F2%XS$I4d`9+9WNz9{xQ(d!RHePnEx2Uz02wKfU<8zU_2}V9&eJ;cZE#D3K
zwyl?wk$m3mC-fa3D-n}P<SppJT@pa<C3)MzhWOf^D5Q;uwByI$b0at|+<8_v!;r%#
z&FKSxPTTGw2Y6&Q%68Y}rsB-3eTjPFAZMn?@wJ-GJsEN4)*QqH7B8Dxy-jXf4kK4;
zc0w$w?64c;)3$H<_Lfd24eE6s$rSI24UNk76G`_t5Z!NKs2m*vs(eeWMr1nnmKVu?
zk0<g0Gf9LNEV+B-u#jX_8kqi=A2nZ5){E@$*`EqJAiJMNfzSd?^Xd*2qB3AT0oZK-
zCwS|aBDBLMP5<IX%U<}x5rPORa~2!)e)JmB_gOoyD`h*T8o4Ln|F~%##HV#}%0!^e
zV_+rE%~Akw=FPuwZH`P6(q`9;Y-V2{q^?j+)|>d-qW=*yQ+>3-m42Yw*1&$qk&wlC
z-IZ;!!8xg+&P6UdO^mB5@FvcTtTJ%KkW(M!tH}bUOpAUnq=>c9N~-0rYJb@$mAx`Y
z2Jp@zc;0tmsrUdb1N`)1$O6YY2fe%6b%~n#O8W%4f=Yubk-Z;ylp`&qhv7i_!RSIM
zDCwTWTN3m>iLjV8x1R_WZ+Stxob-?5p4OuoQSA+qWS1O7nOhTN9HGzfU(3hKGC-?*
z<<pKawFFjK2wO)$`k!@f&moa;EmMP{?HdBt+<p!4HH>3b5|LR?t64t;Hm><y?BBJg
zQ3&yD0)-gl2zeziXdZ=>Lt<XPPU<lWLT)jOyy{yKG?-u6o=;K+*}vr#!IJ)`y#Epn
zrBjC(vNzhefP-h8?Gd8n5DUpC7qI7!;lEbnIabz+aH5EX?@!8@7eVFROo{(*j1X%6
z7tjCKB49ajeGdkHB1HJ6-PwZiU-s=O!=OFu5v%zM_nk=ylRVt_rwc2nu$QjBgk2U9
z$us0Ge~_9>u>dmUHgpjz=1Ayu7HX*O;A8ZRQ(NIZ$F_qMZLMs@sXY%=VpkfZvC18m
zJ;K@20A3nU%ARM9f5NXhxfk!lDR%sUvuWV0o;2fGIIg9kKscU&f~N;CpCmsVI%nQ4
z%@<am5zVf$QFR}A+FQKS1g&+yyN3WWdYl*4YY8yF`N)30PM|Qt9#WKi+Jn^R0FEuS
zJA4O=B%wbt^(OOae+P)#K^mp#9<iS=VufLt?rfP}{c_cz``dP9L!GtLX7fAS`Y=0k
zTJC8R4Pt%ddiYGTAn<p<6Gn4Tve3|;C=6~8oq)!W=ztEwphA}eZxPOJR7<s|Gi-6?
zAWP87iP)>}$1WfQ?k1B#X1<q)^fRnG_La+D=zQ^7%K^L7FiMzf3RtJw^%=v<S<2?W
zm~~^&r5gs9g82Yw66TQi)%`c?;owO|*4z(-GZdB@7igjiV`aE^oNW~qT`Dd^;t6G(
zpO0&YiG7GFEeO-JFLtzb5=J+n=Qqs&w+H==L02;l?_n_CZ^Q!c&Uy)5TP{fv`s)iy
z3143*>Vf+okSzWs@Us5<G`BI?h^0<5do@yuD-cdtYy2h}OZDs(3cpTx^qr}Bim%D#
zMZc$1`S<4Y|2u^K|4Ez1+K{4UGni^MuPNjGWbACQtyYDfW?@G{=FlBrPv)O+lArl0
zXU@E0t{!bw%TM7~Wd4-=_3|?xeMC!*qDvs`;ULX_zA>?b-3{kvBlwEa7;WE2BhszB
z!aH@pZD%aCZI2x<bGLL8s&CG`Y_5J@;>1o%lV7&TRdha(*9$*p6w10TI3x-0`(Ahm
z-^i4t-n~lr*kZk_gF6C-g>5M?(cT=2la4rC3@#PgF<~t&`r|L;a_Wo?qo>bS*FXuq
ztI@^;=W$WGz0k$gIqV360@2P*@F?oC%%(5$Tp2rmVGDT1QM?dyKdl>Mw-5DnI;7{V
zs5!qe+`bgzp;LTIeRzLEZ*)@lY#92MiC+98FTUUc1M8hSSAG?ii@2#!0zc9NQz^Yz
z(}(_8{3GytStf`i&HzhTfD)5AkyjuD1%1fidEvH?x;fjcE5w0m-AIA(`dt{&ln9cC
z>(`jw70nx;HQcNGL}2w9J97v)DmY2RHwc14;y?C0!mm*5G|YojeOxnagLTXT|2F~N
zl47~LiR`M&D_rDBnGSnRV}PmXaGMxB5R$YQA4w5@z7&#x(jF86A@crpb71qu`&00i
zZK1s^*QQtcTdJj);#NQm*z1v99R8M|-H(|<sG?{r?s9<0(VR#2qg;N&w=t=ufi)PK
zHOR$VJl(a4lJs5TOYn?KE*6`2M7h?M=6ScnL5%{tg5|Or3KAq4-}EOjk<fQAPZuUv
z9e4sY7ZB-Cl#62t3Ro8%l^E*2#@Ta5{sf;DLQdIyE3$%`TYk^h7(_D5#mJfn&NYz>
z^veNv4z_+lDXv$AhfI(=hV*cSA4s>|&=+V+ecT<<AsWz|RX%_r_tW(RLHA}T#Nf}#
z9me)TFWK=Qnomy)jLVDH>06ZFQ6Y^3N$w>To<S+y+-+#}X<k#m$7HnI!qg3+`uYO#
zcs2}tLCTMKyhL1W>-+h1wG9bz4-OT(PQle&2XDM%DCc|7H<#iK%tJ*JHcJj=^1g-3
zjka5M(eDJUwcQ&eANDL8+obujPxYj{zCvCR?|lj17I*uWG|u^V+jGX8P;}8{|9ah*
zVe~Wb*T$<G=FL~FwZW>!5UE+4pcJrS*<H59Ta01gh_OgOc~J|S={YYxKwnX<@yEL2
z=5XU4?>)N~s|U#4!<D42e4gbIO+4foLN$HL4@Rx8bcqJ$%Fmi8;FF{6V2^tAt=&R)
zIIuf${1<Ro>ifpq=Lo*XKHnV{N`p-J?znrV(UX|zJ++Mvi8>o?86DukWD{C<Y`_|4
zTFcZa>o;+RIpAO2V>&(Ko>iShxf-<nf?C6qHGq6QZ`xLC55U)ovbL<6C~w|Nl>kV1
z%<k8`jkg)c#q{s~EI0G03-3W~+uh^iDfNhy7KG~+mW`Nr$aBDXt6*ehS}}MLK3-!_
zb`O68C%qdWa$^7<0IvR|k-b53EQ);-0C`2q9Ryuj<TxE9Z@UZC>-0Jt?9J5b1-iF&
z15O)1sVeU`Ak1Ij*nL6{8Rb97Jv2J>mfA_ElJs(PXSnW>X+-jotyUmjmVQa>yioaJ
z#fr}MhSw{sP|@$Xsidj+J<^-^=&!VLl!tTGIYUxhcL>hN=f~N&0wI&ZO^XiI!GK%y
zzHD#bZ>pp}zsZ8xkoXz;!ANL`A6$GkS0VjL@EvPuMhuLkoC6H7KudZ?&TkE#s!{Ch
zIkJ7jigXn&!=qigKdC&hFuyRelgsmlbyslP_k1L?#6<)3D9EA;|IZ#LaBL=<tVu)u
zRxwF2FNm^MfCZU<xM6=>g~DP|CpBw_2w4xv{f4D>r>U}7e}*{pZIkrrWKpOj2#7d#
z;984SDR0c*ip)g(L^*d|5jixlhTI{Ohk>Qs>vaIZXv4b;9qq}P5$jzKa@`AuFh2I<
zFZdu|Gg<T15SW)|-M7eQX!Q^0sj3Mn4K1f(?fd{yU|Oj{BX<O>QGmY9XJG22FRQHr
z3eJ|thLvs}kvTj!dX^F2AUr5;#KK)0aGO%%F*uu|(mCW}6wZDrojVxC%3rSFdN!_E
zMdAiJzibPJ=9M(M5(!)6>iEv){a%Ba*`{t>K?BjgO>sbco1ZTg9Qg}2QYB^1jmm7n
z&sOX&S*2gfZ1s!hJ25Vj|1PVk+@m=vB%g5E$WZ3ejU9DKCm5f<*H7J!)7gTpIUF5z
zV=G1XGZMoiSkvR;)@S3yAK$UIvR{i4`Jg4G9=|uzfe(GD3T18AzR=Z%X!ZM-M=8fv
ze+p8kO>)bqfG1VJBL1}48O?B1k%WHXh%UH=U9d{?>Mtm^4i=xUS`4^?ovg=eEqWxl
z3{v&`iIFmhN)&vl;t5T2t+U#kHmcmf9f)`OBNy#0h2ow!-XQ;u+uVh|7b(Z`^-;>3
ziG6Tw0X8f@Q@3+XcTsb}2F*MqrAS7*bLYzbsISl#ww-yL$e}dP&xeoGf3kTF*CIO^
z^K#Z^!y?E^PMjExw(sFMBN`KeTv9@EYo5(c*PonaMkXERB;CfH!cb^4NMgfw?$OzE
zTWJsScGZR{|ICC_(8p>wYslYsI8)|#JMnJu>u`4Y%N<llyg8`8@lEKl`@L&0u8nk5
zob)RE4Uqfvy9!c(Q9IHdts3dUyBGWnt7@6`uaU@=casFYXS!@r9Rbq2>zHmAEBEW9
iNaVfix5Pc~C<Ga)3RkM&h`%*Jl9f`DtQ7wg_`d*-JvW2^

diff --git a/modules/web-console/frontend/public/images/query-metadata.png b/modules/web-console/frontend/public/images/query-metadata.png
index 698cd6eaf6a9ccdac81e0c9d1b2ea8dfff9cbe51..1b6c73c10f4862a0982018ddca7345b6dd7b8dc4 100644
GIT binary patch
literal 39361
zcmXuKbwFHA?>~wb_aa-gKyeBzE`{Rm?z%X|t+-2(0t<90?i63#q4?rf+}+*n^1Q$A
z{Uc{GIWw6_^2yB6C>14XEc6fPaBy%~vN95CaBzqqI5-3vRKzz)?TUE*TSZevK|}KL
z@{*I2b9i|8_V)JV<m8Bq_WAkw`uaLBFmQT$`uzO->FL?n*jQ6jQ%6UKpP&EW;9zNK
zX>f2bBRwN0Cueqcwxpy)LPCO<o4XvFu%MtYDk^$>d|X6W<m&2betsSTfiyHUyu7|5
zARw%*ty!2`R8&;R%F1qTZua%{IXOA``T35GjVUWD*VNSL8|Zg;cZY?Ab#`{f#l=}$
zTYGwX-rU?ICMNFf?M+Thy1Ke1Cnq;GH4hC9d3$?XSy?$aIM~_QdANJry}q8Eowc^M
zy}sVHwY3F@gq*y-Zf|eDzHE+;j%sjm+1uN{-cMf-xWB%hlB3|)*VnJTyb`ihzg~Bz
zrlxXoNj~n?KX2sE+`a|{1wGG&%tU=*)2ZMRa@fcgb>s)4AOpF$b@Es4&-Yh|jR(!!
zP8u5<MNyDBrTxpqkWz3$c~z44uW!~`J^QX-3p9Zz1qL7hyi)uaSHSe#$z|BoW%uC)
zJhkUB1>Xe+om}nN1uVy7_$pkMb9DW*HOVZkW#vwjZF#I7ksHDf*=%WPiD5xJSJcoc
z*f?x3;{En_S|{N8V7NOk@uo@f9k$vLt5xm!>xor$<JQegbLQ3b^Y6@V8mf=Gf2zeT
zijU=EZ?D&?Yy}?tl`Q0V4(DM%mtWLFs(Vcdx6V$=^hy#1N>y#cQuHPcCwBT>ctXr%
ztk{cv>Ngxy+PnKVE^=q<X4mWJf_ceH(mgaQ2WC7ghEBYuFY3>lX&C?H><+&t%PaSP
ze0<v2*f=^mdVPJpxVYHa*?D+)SYBS1L!Z)xgG;5Cl@QhNGBV?<eO(JOuZ7Puc^?yu
zh;mD5#`k@TJusuj&tVZ2vGKEwhdJmn8{^BspI9|LYq5TwaUOwp?14mdJ3s~(YyA0v
zzFu4aN<Rm)Qtq0vY*oJwRBUIs4+)dQ+0N<G#`T~)U^Y`wR}C!SIFw$oeJ@&>>cj0M
zLb0E~y1<5uwEuM7P&=zU_{JT{&RUCz;^woqWj`Z{{v>`a0HFqFgN(7`vC`AmZyDeC
zBCx&v>BglI=9T?2sbGO6YH$jcF`TE8Eue~5K^gh21=UmuOADXWBU*4u4rpA-7z4Lf
z;d8+i3n4=e>cq}o0>dB>GCJOkN&p14i>7?E)*WUdD6hAetWjAg1Ld#?^OS&Bg+`&y
zGmVHeI5b)s@ii>B_Y1H8fJm%E4HZ9`>k3K3#4z%2_N^frir?uh!eC*j@hX#B94WHQ
z77*;oZZvW$2tM6;k`B=y-jmE45UQRQlmwo{?$>fSxLzFeVKFo$z*5N?kg*<@-V6K=
z96FIkm0kT}GEc>J@;pyJ^fq53)1Gkm0y(Ire|GZU;P!eknFTPNH(FmU$wX?8;^dzK
zrB8sT4rIR#uc2{trm#A1^rQh_+&)WFhAMPs>lW5NTit$hv9(hWQ+aBXmLU=`8g2xC
zl$q{Wv*1Kx!G)c+scyAGjNw$<6z8{@I)0nB(Z~t*W5O5)%?tOrBJW?IoPMGh{Tp46
z(_6|cBR~JwiLs8G{x=VVTpo;%XxR%o(+_WO{CL!?CIN>bm)}SNHVkqGew~k~4Vy*L
zmFLTFiwD69>sydthSNAO(##I=bSl*}t`C(Nygn4iP7i8?qcahq(Vj?%T0eXIPz1>x
zvZ}S!p}IzT)R;C4cC{`P5OG3HAeOCP{s>Si!iw&q=g~AN?;Eixeh{{gj;Gtk)N1&Z
zH-K^mV*j|0L$kue!!uU`xVWws8=k8$MTfwIg}kgPEE(}teB1e$k1=whDTSgZsfOX+
zRiyC>5T`8yT_$Ga6IDi#L`L$4n?YOpfJ{<r)GkRIv((QSQYOz}`|hpqZUNlgoebta
zwP?GoV~t2!phjj8&ijoW2I^sSfh43ph9=M5VgtSLnj1gJTriIOZ7s|Dl4RK#+q+7V
zCKK@|N{L#3op2KtjLWZ`G+H*l=d7kiI#K4!_I2N(G|qq1UXu{ym7S#ZN+xiJ)6*Uy
z+{AIH0Lxim>lKdh7f=m+$z^8oD_>R;Gt}7`KKEc^abhz!hTNohpf%btdhif%<^P2`
zyFcW42;y`<rFbXeIjjtEd{3KCyzYLBR8n^+DNm<j4QrCR)W4I9eyCU|Y7F;nm2#l=
zw@J|*7YF6t88-%?-3_#?J1^xcev0T_wXC|X4&4v_i}s8}rioHT`p1txDeldEJBJD}
zwdN3^u1FK6&4%G5g)QNyls0Z*q$ig;<#`+=bKOMe5r)*5UgKuAOWi`_094-{B%+UM
zoZHKYVX;;TZj9U+m_7R+tL;~ahyLb2{H)oF2R(Y^R+|}qnh`hGRhseaAeCvVwhvZ(
z?JN>L?+12RR8psW{Pis3sB3gXGuVrOjxmmFQFB6Q-eBw@9?jML0e7@G7T<z*eQ%6}
z)qih0kA<LG;mCqb<`Y|ryS!)-R3>?jPC?LGw_g{pFqvmrhvx*F&sw+m3C;DB;Oyg@
zfAXl40zk%euRbrWF<FLCmQnY@->ta!?>-D22_9|Qc48?nF%(+m#6}tQLwnY%6)Z%G
zrwd<n=mZ6W_SUK*OZ<nPn0GWPt`&G6Ir@vz*mQ}O*jcrq+KJ{8XC++gcnVaaUZt@@
zxU0`_v)M)+3i$HWNv~bS0(M%_Ea#XH5^UBJl0h9`qbGwKJ3`7@tA7WXkJ-jITrmFA
z0ZTA?6xR;CE2A3wO84qv7%z9PsOGovhJlS_c~;G_zGx<Uw1OJv_E8grfcNZrI7qdI
z1U)m-xo;+wP(*gv1AhOJAR8RL^#%l|Fv_*}Qa<yPY&=qIV&zJgc)c%oC@WpV(Lj_a
zU5HPMwdTbgO>1X!YV=ePbxMuMH-OLz$9(mxqmmOG7LARlR)Q+b`3Tbek{xjCgU|>H
zP8g6g)p1uB5Y@@=3TQJpl+ae!V$EY010Cj-B1mEtaE5#nj)XcMgR((^!-kjKUxx%`
zH5b?YjEbn6EjYw-)7eSfk)pN#_E2QVHN<~+?_C_U35jmlegT2IL~rwCeeVvh9ZZn|
z?x}Yv%O>*BM86>h7~fQijsrnIDWWu(Qs%LbDXjs)9UGXEwc$~Yw0ktDdhZ7AJjmfh
z&lH`B-w0JN-0_R#IFuI^NpuJoiB<f6REznh1L=SCCcKyKCyARNiV0R_M<<DcpDPL>
zg%c$S%thm%z3vIa`W1x6*Thwr<I@z9%Jl3QQDh89*3A40f*XQMsm5e62XsOW13~q)
z{69gWDM27DeSUJV7Hc1{%D(Q~RM7Qf^jo8(gc(T+d~<+Kib96Y_l6_S=T2ye(hkK?
zColz`#b-{iJBufdB`}LWR6?Dx>4Q>n>YslCe<~6yJ)0d9P8RE#kMCJ(Qo2o9FipsQ
zI%lhNmV8>QW?h7eMT@@AN+M7V<l=&d&t>wBq;{`3UUfem@v{79#P&5!Fry*^XXX0)
z&dB2JPQ&GjKCRY;<7NjPsqZ7ZmHhm8`xpnPK*8r~Btvj4F7l?A+@4K#2Rms!>~Ne1
zbv`u9!x(z%#4{u<?B24R$H6DtSe-mb;wxpf;{?515Qr>Z3zPIS=~C2Z+Q^)i4WJ^V
zz+Stp<KN?ze!`IF!`EL>ZYlihPx9e;m#msc*ezyAOcHj6NGYe4x0vr>{#~NM6pq5&
zKxEu=>6oIm5}muaXQuCqg){BI72T4Jb7lF6F+`&a|5L_3Z%O!u(=WcC=#_tW*GlAM
z#7#|;*?-@p*y6!JQ91vJa>!8!@0KRWGXNjIuP!9(OY%iv4x@DTbor+vhB+BKJ=Y@n
zOMPvQ^`B%~Qh1T$!pM^s$U<VZ%NY&~fahu?_=Z;~NjA0q3W7cOVM}L))OVF(feghv
ze70-U-l@zOU-?IYkJ|%zm>CwIcKD6%;GGt~d_Ug<5&Pc1dw3Ej1Q#gT-sXCCk(Us}
z_Y=^WPl$O68%<6W1Tk{?m9*TEr0k^ZX2|W6D}NhqtH=H<0V6V%P$G|RncACIjHjpM
zLk28pIp<6#AO^&`X%MSv4RZ-=3<RI!$cu{Ijn^=c?w@%zySucQ`yc1)?>;1q|Gkps
z=*V;7PGm?`<#`{e{70q-UlerTwxDjgRMt!Bl}P*E_<&U_m*X#?QEcX4ivfCd9UV#J
z7H^?hSFNxn7XE30%A$HW&WH`p(=9|0?w849Z?ch0QOKl|l3BS^+k^X@&s0c~m&+2d
zSV!n%%1I>>-SJhD*1F||cA%^jKuIhrvtES=AoQf@9MEUhU46-T=scmN^8AHViZCuU
zXj`sNbJjQ&=!-s3U%YSxiUzW;ghSapP0b{cVVN}vE^Iy}8w?kyY;Nv3Q`PagGO)iU
zv-$!Rtd?{JDR_76_(@T+gbaBY?}a;y8r+W9Gbu;{g26dVd!eNTRM#x_^16TMt~BOz
z!;BYMSl(Pk-94c_;lOdF0CS=qr>sk-QR~nQ6b&NXw6#{A-B6E#R%_4_hARc6W5+A@
zPdynOGltj{FXVWAC{RQTGsS-|B;EKIy#~GJx1zYNi}fR9qSL$<r3VhP3Hk-J#IGU3
zA9aO1i$lTCK6i$MT05bv7xsJ57+|0;4ZEZ5o0>LY28p*bvfrHTE!nB;loPEy%<9(#
z+b&@eAjVmAHaVN>ffWp2ZG0e`>1~)_ZKNULn4&dm^=RN~%Aegk!im5uKMc@>``Oww
z(=TG}>T_KxMI=zk(}0W3n8Li0kexh)7`OfeWM2%Hq%P?A3}%x7qy~f0o=v_{nA>2%
z)Z!iYWgxuOTvtw{QlH~i%hQcgfTx?*!wLZ3K(CUsCa?{IJa%0{%madoevxq>g4Ho-
znE%dgVw;xkzuf2<n7UXA31tQSmmRo48Za=BKWae?LX^V}$l6<J9pi9@(|VlVCHXQw
zIA^`W+lVN4zizBWanSSVkY(Qd^?}}u4tC~y7+F62`Wci_*lPbT1vu%!b{XJob*me9
z)@q;kiER<<13s&Fw4zsiLaD{R?H~&{xDf-hw<fmrNP}Bd>*@=mw3m_AWW>gPQIaAU
zj{qcvU{S~O7TXK0l<{h_J#TM-;hYR)9LL&8y5n<kVOyDoZ(}UD%fX8oL;{J6QS#b&
zTQam@dpMRb_T6aNFh-{C&RUIM>U6GO9@{V@7SzI1a(PD+r!D~A08i|D7F_Z#m@wH!
zwnl$FGZ*i_p2Q3_{|hWyYkfL;w~lbsoaQG`-zLiU?j(XR%FB?&{MUp1$NR1pi{a|2
zjMJfD-okjh_D4O7t9(9@+=vyd1{JA5*4cLjEE1d24yFB?%84Jfl_z8cNhFI(GegAV
zOLs~5R?PYp+V<QsO{_It*C&ek4Le2*xy;DE2+1DwG(NAF(*HOpxB<FI`oLeur`po6
zJ-XW&6{d|(9x-$qZ11q>v)Yj~FHN?skq`I?&D<ek3h{#mCAf(2yaC@mypHFZQ)OXC
z7eGe4J=2sc>Dl{n*GnkbLa-X$H4XC&eGuNV#Js&38s&%5amUK{q1RkwzsMqsdfW6E
zncckvm@Hu_0vD+7v~{wn33tGzE0+x(>xmvVFm|>f=^}eZg3+M)(#P{@fh>u;;6oHJ
z7E6BO_>7W>Dd9J|0OW85c#v-*57>J*RimCFcD3|wt7z{2w!x-nucM(Dy8~tYU}fNF
zMdl}F)G`XAIv=R#)B<e7Jc^YZn%+NadrgJbbinKOK4X7yJxpcH*=pjR|K-IHPs+l8
zesy!AcqU2z3kR$E*m4EYG=^%S4t1aW5vVxn!kTxEN+!Vv!fnPi8cjlotS#@Tb_A#z
zqGB4FZb^r-|6(<ZBbi$1iF0R5!pl=MqO&+hC$H$lkct7aB|YFFD?#aC!OwR0b?!7>
ze-f)QTW7HHV^$BtsHM0PxXr-mSy`!VN<A`|xOU<YH|*!&_f@j_Z;<&4(PJjDl8E{H
zmNC|$ZTKWX%9C_v62%tf6YIYasw|AcF+U4djUaKhPjqADa_xdXTa$)<y%v{>@e0-s
zEM&jN3Y&NpRq??f!W$3m@DX`Pw+NJAGvS8Y>guJm?L31`^fJ&;@>2KRvYWLk7||uw
zWIGszLw<wUYAXFr4&y1VZu~#Y6_(C9R`{#zrZMJ?b=}R+#4pa4c|*X-3NdcCMwJQ&
z4h&NjBlS57M<6kgsLK3d<EG|%ifFc*8zN#@Ri=}#x{;-CWtKGHL?q=PLYs+&F$rD&
z7RWyhWbP>4Ifw|d&1PW+P4Z)jrSgIPzU>4^ANnL9y~O{|=$r_QPZ`Q)fpV;1yp&XL
zk%`Ajiao^5V4$Q5kF)P)Uo35(`=j3v+e;&@y`v+I-;ClnB)&NmJf!wc!@iO%a(%cg
z38{cHP~=VJA@i5hOxnTUF(k3KKiYd&ESN#@ndG7acj?kGl!zc{uiWptJ;j%D?Myeo
z-*W!=+8F&fhAMrU!_jeCF!tz!zE(`iibSCLr&oc}t!^82=UAL=Xf##uXN@-z_@*-q
zaZZ<UGubz1^^Ec$QxEwX494@-3>+dQK90_gS!Gt({G4StsJ!Wr8f;){6c^#MV=)++
z&<8d!U6gAF{d?ILyUp~8=F+~n4Epqg#gy<QJ?OkUZ^M_1nPIaX_dyo+%YTVj3)TqX
zVJ2y9map-}#}$~W=#xR6CZSOo#<|z&iEP0svcGF<sWmT$8_Z*nOuGbPfCh|;m>4d3
z88Afo_K)$vc%l$al;;3s0Y|?r0o7HLSm$3JdiDkOhk$9m%(07uephbSh<c|@R=2Wq
ztp}MC5RYTQJ!kwew)It|)Aj_pY_*D%c#QkOL}V1SuwjzZRe>xfvF0MjfJmr!UW#^A
zu0<E8RbW02c&?Rp$Xi8e(jeAM3;O6)8C{CZ$VUV|_vuZOd^l>Dyt{Ov8X}acqf^p^
z!U$8m0nF)0tmv+~z6>@Q+-_{0Mx}NT<a@0T%6cH@QZkI)ElM$bzwBtNq(`wmdIo|A
z4r3ig0!uNPD-=0q-$bgxm<DfSj|ZA|Hhl`A@@3a0?MIZH2M)ZDjZLyGkbJYJ|AQBE
zK=$znnd8V9lHv?b0V~KWn70fS#$(fSSEQQ{Ta6@%#YqZ=;^wI=!ajO;@#>muEWCy#
zThcU_-aAQV@}eC>jktcCm}NvF#Ac?|WzVu`>$_7(oet4SSBYv3s@>_hh8I>TT?ps<
z-+A}uhV2zQYw9Cd+Htzu?R}cYv&9HYCaNAm*FWtSC?%;>alZJS!BQaEQlV8xSdrDc
zPGlUJ>7vJ|cw<|jkQeaC5rO;|HRY4%sxz4NZh$OqV#No$AJK1nvAw-AN2_{uH!!3r
zv18ElQwOIMW=dR!FUoYf+qY-kB5#00ZJFvU=ji5bX~i$OQkpNOSW#lI7~3zN<&8O_
zct|bfTN;~|_`g~tdhT9tJYyPd+KjrNo=5hwYncV|j~_d{!?w#r6jLs;uEN@!WFZ!+
zTB~;vxjZ6VXD)}U{_i!k20h8O*MC&ZP<7-cKhP*Nq-Rb2<xkmrMUa@u4m2)D;Sed^
z%|2W`G+t+Lui5!K`3ghvg1d8@JIC$7m?a6o4q$zff84GN1BBv#liapQ6Wz{0nn+Vr
zaQE*i{$(4pg%5uI-B#mcvlWfN*W#mlHi9merB$B*@*~K?EOVSEo)sfJf!3So6+uGF
z%o~2tNsV(w%SF+Btc}j6B3&B%wLlvJz^4MQBtCdvr#C_3E!CFUcqcLtIeA@A`ERMu
zHhqUPn42gz9>-HC#uOja*|JJolKfl0+#B;TJZ1Y_IIKGnGaQ=k3xg__8c&Cs<pInp
zAHb`Q=+$6fZS}iQah-i?P(>qH(aOm0<!DP91!QAXrG4q8ts$mU!NiF5S4}PCXN(4I
zVPt_yTpunSI3q5-jS+ODlIJ#GVz08JGxI(Wjke{Ky*KB#JVai1we6d#$El+7Y!K82
zv}WRePD&4wtejn>-!16wiuwbl;@=<tov4vr#_&Uv6f~S$R`|zu!;j&T(Av;XBg#k&
zQuk*WL>?a-TXI1!^``IAw1MGw>r+18BSv9mCUQEM8&%FI*u&7a3QSE<t#_o%IMb7&
zcDywdEI_VsZr2rDPaGDPC^f_byNH&$z3GA@H;VgP87%c|m3{U#uNX4x_KRdNs30H-
zw?g$@(l&kpJQuUG)$SGOW7{&}H#B<F9EBZdr6tlh7o$0;X}n6p{9g}n=?_v|g^l3D
zONueGBR#bpwVN@MOC?^CmMPyMb!E?v*MeA}rAo>>SPvMCo51{4L5WZutMNr#S7^6^
zfgpLUm<TYVr32qNI1wldDIQ;tK&u6!7T_o31h`hg;Q;g~zGs&^IpFeZYnz%-D)K~7
zd<4}Bg%#>AKn3*V(n>%4Wq}>Ag*^ltXI>r+vRNk5f$TkZyf$P|gWd^cR`ku;)C<>r
z>_M}?Eg@%d7@Ca-rUHBHp@Q^Y%I(W&D~)~yu0~qm*jAQUaFDpqFD42G%^4-YIbK#g
ziMMV#qaf~TP^swAxFZC-U56|Bb4{9z?W?7vr(8A;Dx!x>I->ziTwtIYlA?JTZV<#j
zEb9KxvG;s^>+j{)Z`?c$<QYTnA{)PgLV$RFIuE<q=|<7GttOlNS7QB3wwU{vz6vmd
z``Q=M-er2}j`P}KNb0i^72Bm{{=1@>J#ay(7rleo7DzQ;L02616V*+Z73K>^B8??I
zp0!;pC7S5nmLvKD>WkJQMxvF*E#(WB6>j!-Z(t}=769!w>W)<}M3iDTkPr!fi8+0D
zb=OaPrKQuv1T%1E^l=HTiy!p{bhly}>S);C0|9rilH4R9vl$noc)i3ApqWZ|$0(RQ
zGmWEf+)i|e-yS)qi*t{gk%kB)*c|q8wDrgZ23L7uXRk-}TqSvfUP}3vC?73!`cHqQ
znlg+2ZqOF_S$U4Z!087e<VdWK#50)Uc=tnyg4O^{QGW)f2U}X)pj!%39cdWe2K*(!
zNT7#Dv-ZA0ItKTXFb-l_%{pq9$>)}UFmTg`8js(VgM*XPKtgKN&4)bHNJ#P%b=*rp
z&#x*~snkdGR(2WQF;Lq>&?ij8h8cV8I6_2P@uWX`EauXNeg;tp9zg2|uj@_c`FzY$
zN~;y|M^f(k><+gbyv;~fwo8_OJXZ((M~SO~3X!VMR7k>j-D$%mDiDD?8i{}1V|IuE
znG!w`z*OjZ!sD&<OY>Qk6u8K1GR~a$^ysO)E;ILNzqK9WwJl~W<&tRIn%v%*s;W>!
zy~$&&mD`#qqC39#!IZjki(q4ML#Y()*^RA>`Q8V9gWz8OZ9O&?ou$uxS2P>}UPn#2
zS!6VCefWrEKNFuN^p~T#>4n&I%yzhVD`ariRD_4xt#jABH4*1en;1lXet8jbcVx#3
zQQebGpbXDhuPno=9kUM7y&lmf4spdO*d9VXwE8ZN8Dw=Blz{zLrK~)!m12)Gypb@g
zhh_fp%(f&j(NxZ#y}2o6W-YQ{=e=b`q_?j80zuQYQ{=6FNTWfe%8r_cPvAa~9VAkA
zetOgWp-35Cu#tZ8xaF&E!wts`4JbqUL5|(6z)W?AZ2tSa^B$erO(izAQ8p3+6+QtO
z=VTzlt+^Z(+}i|Yyt{Git{hDapgb+yjED^1i`~=Imi5r0lEdWepBHJVwUpG+<@KN`
z9D;<!)g~V!3u&@FKMO`Or;?}Reh`Tb`v+(B`4BCw)_<<DN`P+<o)k4uM!}?;78O~6
z$Np^yL?jt!S67sLE({(?7RyUVnfYoCEe{S-5-;MBb$f@s>#NVO#ia$@Q$PxMd}-k)
z@n0gUmM#~6kB_9!a4a?Ch>=Dud`&o5R3_?wYr+D&5UVKX=jIwwL~z&o-t|Zz=LD+9
zkRrAI_@2=u8t9N2&FQJ5_D&~Mkm^%1DdtVlwyP_>rF1_`%gOb27=8!(jVgn^*T=t=
z-V^MI%xehGSBZvw+BTU72kvdefeo=^(X1gBIY55dJ2-Svqng<L$oBQkHeip4dig5#
zLdE-=Dlnr(Uycl|n@`D$0XJMqI1ibo`SKkt6qh|3ldsgAqX4AqI!@sQ<Yo7S&W`gU
za`hZ8V2_#<OsADUGtgYU!cBVfpRZ}tIatgj+gP0TrEp*TJ3&cZA0XBJ=jD0*Cc^^U
zo-ysWHm2*8Rw&S1KHR#*Xv_1a+zt|I6<#Lfa7AwtPW761RMrAx?T@bgZ-qi@WoQ6?
zho49jj19TIPiSftCdYo!$OG06$Z7Fh{~OXPFy=@qEmK%cZX>1Pz^H_V&s_F7eyQYz
zlr;hUw6(~?v~^wHr#g1rF*L2fu8_?giQXC&(`$cZYyJnT1=6Zb;WCpBX<E0A!fK(t
z$mUwikH%APtc(Kl<Le}h3yK1ImIf8$PrUy_a*JcM7mQ?7!N3sJq5Yeo>Eca6TQ7-5
zxNs4*1}K-`pNAHy1>eh=FOSnFykma7GTEU@Qu<<En1d%kANIgltoD4ZT9|xoDGFLM
zqx_SyrsHv|F+bEz<|mqc(4kIL>3ewlcuWi)+#*%?aM)GvWN;jt)rM%%z2k!4aT|vJ
z73?9r!Hh@vbCrIyB4~@pArtM%041W670-djxywc*0)IlqVy%H*1`~dY%6SP9yFr$$
z$EkQ{8ivqDx2r_Shi7rLo>Eo--1nG~A50!=c<cjn+xIGbtf~BOn7&g?Vv-M%*mGO{
zX4LquHp1ORL!L2QRSxNC25Y+ld`Q&1)1$dVLmA|G5%17A(PpwVqO=Cy87@zI!@FjV
z{%*|$9~-2#qG@<NkIQT(qEEn^-r9c9*zxR(ULL;x)}qyteDMowW@8fvlyNyMsg@Qq
zBOT+HLOHDG@aw0`q}#g*Tt<DMK9#;PNDtz~yx6I98;4kiyp$CC5uRZr$Pq!c8HuOI
zg`DJJ`w8caIa^{CFAqpmcLPsCzD3iih4on(Ub;f<hQg^H?h`h;M^ND}3ejtgc`ql+
z-nc3H1o3}A&nuD?jtdAdKurqEOne;s6!P9_kio@;k&`v0jib>&Ru3Q5!|n9lW4YgO
zb{;KRK(=jwje5aJNwxh;5B(PxLPd{DY|HkUs&2?Pcqd{GV71IskYuW0`hcjz?s+>V
z$~zXl=~L2;Lq=SUJ|INFkerN|kUAw9=*I-=O~xNBsI*@{>$XytnyD&{dp7pSs1+)Y
zEf8{yP}F5{T)dwUV4i|W5bKjJv$B8-R!gBcFdkml_6q7xzMvs@6A}%&Os3zLd5ra$
z>O8)ca#YTbkcPL%rlhe_G}q#g5h2EoG#uELvv<x>_EEra@~Q2yQ6Y^qOP*O&@at&w
zNA(?=OGtS&usp_7^}Up7PT6($UkdmyObELOsqc}~%$l$<DJw}@IzK<(6qn5)I8i){
z@pn3{T>zZSJ|_o>Fw`msVoPQ<nfYqz*5Inp{ZL#CQoFGdCuGBXYEW$i-w*7}3fI(W
z7AAx4DWe!CY8yoTK-mCpT^x6yJno}$8Z8v9+FNhuX=hh}J@g>_;A`sdAItG*gjIWj
zO%w19E^fZCVnWEJUi(*vCn@i%(HKBQ@h{Kc3>P>3@FQ6#{t6>jVDJ5#O~Sl@i%i1|
z8OEu?s}}d~>x<?~ngfAN4!i#}AzE6Wa$WhbKg+0!tNz@h`?-*d=u5nnu0re`QvX}x
z&55X=l8R#v0;gU+%SmjRO{k1pq7*wBITweBhe+Vj;Tc*+BK;vwj}a)VrOIoY8^n2O
zE8CQDtRhZLW%<;Ww}GNN=vJ0AtdXK4E^DVRNl#ZGWTJYl=PB^dQ=QJXBH68IT`hcx
z?2kA0VW9%E&-V8=W8ug30nWZ9efMcklEtLtir>P}ZTddorvUnZpI|1=FItu7F08KS
zvam<X!g}o)Wu5(6Xuu7bd2jOAm4osNuMJ3`e7NP{yRwp!GNd5+S^DifMqiq;Qp%J=
zN%C7o<v9SNRX7Dp)NDf6Ov=%GV?SUup&KfTl`0e^OUk(pG*Ik(eEFxIp%p%0f!yBZ
zgWtu>JY@rFKKt*e2T}T8F>j1dqQZ`j&GGgI$(#r&hXJ!&+~oZMKG%D&s3e0#11>cp
z=ZE~9*CWwaZG%=xTH1A$-p6md`lDA*9YC-jj~D=k3JL{+@!kps8nB<_+W`-zBK20}
zunf8D52ZW!Ef?;*iT<HKsd%=g%e&UQvL*f}s6U1H8KxZ!gXOT9+|>w=BzWBoGU#V=
zgIbTadcPgHXZ_Q>^P1_nJ68#ZzVsZKYcefH4yRJr%j}RMgzc@+v&(6GV-5FLEiHRM
z0BPSGn((SJpd(f>CNW2oFoq&4O_NY1+qE)`Q;$-m(WO}eNr}HbAP5(1Qtr9N@`lBZ
zxN(%p-4R?k%8VWi@oQH-Zlo%b%h`}gP$v7$fp}qBX~ey9A`2Kgi@7wyt_YAczDuW0
z1u9U8yvR9saO>5o72GbR;!`xwZ*ts|atnA~o~KcGl0I1|GjA>o=2k867AE+3ZHByP
z7Y)KNtmG_SnAG4VUnCNK)7dV5D;~Yc_|Zex6x^bC8U9CMLdhRrz}14a%?>JlPn`;>
zid9GG$97>EA_Y<!x)7b5Y1jhe*mly7>Y+T7d<5kWYa?qdowVyn>1a*~C_~&`Hz(A$
zf#f3?6Y$<2%mW!$w|K@=mau*fu5ajhP0Gg>L!s!aKdyTRf;*wX<*iuhi-wkP%;abG
zcj_j#&IlPq^p|2!r#YV>(9_352DZo#7i89Co)LvCgr`7RC7k6$;}ktxiR9#r9$2AP
zy|Vg@jQUMty|8LMoIwOZq(QrN8oL%J7FH8{Hp849e4rcVX*mTdsqhdn71<W-#FYm0
zW(7b=bKbHJ8YbNt0(qz?!`V9}HfMX_4f2!PVWQ9e0jSlcs;hVGEQL2?i~g=q?Ct*x
zWj8~-k4Hb)miG8p6V%cQef=(c{h~wq33M9gf!~k79!KzzymRVZ{&&%&<c}$HW_tEN
zPzAygGfe0RzP0`$H8xr8X)wDF^{!y;okZwfA>kFge(Xn8@%er|6r_fu6rK!X$Q;bN
zcEg+bY%=qMG1Ls>BgV&r{aeruQBX>yNL!gQTpEo7cJ>!I_SxW0)_z_F-f+3>9p_72
zw^+*Vq5BY2Ixo#|*8qV(@+!-`WWg{XX{Gtq!`w(+)ocqTwGRxtzql-a*Qx4OMB*ly
zgUJW|VeXV1JAoe*10#8>=K-bOxZ)U9Kpv3ET%+$l?*GKv>PPvE(bN>5;itFp;pDNi
z#(j`cB@?<IlUjXk%1^(m7{W;{*7uLWNwlD4J4if+x@$eoS$<Il>2olZBJ$QHghDEA
z;B<rog9|^}Tf?Cy!{6B=M1pny4zek`5#n||mjWpddk8NIJUnGVfOyyq(MqvF{IC?o
zw%wScB7A?k$5>F;ZtUC3uulLS{sbS*cjjI)^OW^LRf)bBrjdU$fhPOQ>biY-(6%t%
z1*Vi<0oydNqq6fSB0j+AxG^ov<pQCXU`e+UwxfLV5hNWQYQ2bLrs{f|4F5+8L`lsg
zPqe}?gq+RUIRHne{?v%LmONQWa#`nm`I~;d2u&v%x4wJ_htX9?;+{3KcjS!0+nCM_
zR8E}?5@K3>w%W$fMW}F#6g~r?D|%(p3y^|xn3l`G)`j89F{Y)q&lB_IANdHS^6Mx@
z8~F!(F1;>33Xue{xxhR1{rb@P{Jw5sWj?;Dpb`0OPXCAMB1*C!lsq&Ud@l#!s%tDQ
zV-e7}>B<Gx9u73<H%{e~21Kj31?%(1Pdey9rX=T<MkURJ_eVd1ybQ9;g~apAsDm9q
zHJ9I1bD{A>$(meK8mYfT_RX2$=ynE<?6b^~7XBTM=ndDr^nV1g`!1dc=rD(TaujB>
z`x@F2@__Fc4I`<1HqrPG3?#|_f#Ls$#Tyv@2aEqhOFvQRd)|1@$TyVq8h`Ne_tW+@
zwtNGSLNoIy&#IB-V-g*e#i!wzOx0j5f`NI-(5SQOFMB36qY-ptcsI0^MIL_;`BH`!
z9^${hEuiL=K>JGrcRZ^tr>ba17N*DLJg7(ALPjf-e(ut_ZD8Zy9YDz+!N8d*VjV|5
zVKx7671;k;i2`9N!b%>Ky^{HuUn}2HX{5xo!Onuf&<O;Tq%})PT^{q9gL|8j*XONO
zu%An1<w#8|<C^2Pi~RhDP;WhLyi3jwI@kvqxbl`-Xy?W67Fc360cbQt50hdY7KF#f
zy|G7Aw`WojE2Q~1L!cNW4G^|XAIcRjN;Z9J%q#mRZW9<Stk3)#00ZS7mFgl$J=a2W
zxY}1WQ(`h04jGDynTE{0-_!uL|1@fo-ElLXzJ^Juh!XvmetaP+TDlXQBY$3yPdKvy
zfmLBDPt5sX-7o2JLl@4_DO^p}IjuK6T096T7Y@ZNMz%cwxJ0KN7vx{H6MG+%-ogMG
zD``?d<?xc?@A-5@os0E_*an2`e7RZO%AE?&?WwN|^vGp-vAsQ`H?rNTMm1S87+B9+
zhr{Cjuk&>>Y?N85^8o(mm1M2yx7#V$U&?2Y_Wa11wY)CyhAp{QK9<isyF`$mKcdS1
zE%YAddscIr|Hve1xTdeeQ~2eX1p&=-n&AY>vdr>*E8297BAW$L?To~Y4dg;IAM4Ut
z-Z~XB+{}qSc|(8!n*WGv_N1~T6H`Q)%a3$K^cdwmK1h+)??5e%{6KqX%x?x>ykqSM
zX(B@U27y5dlb?$K!q~L%B!1HP$<O~(As=}FuLK}(=di?Q`s6l_KD&!N=CZRPC55r4
zzszMM$<+-^j}dJ($2Oa=4KI{_r6}tKjZpl#vN=2rxq(7c@$M5TG>0}9yYVtBjjii$
z0TK&pT3V!Pgt8;#ltxDd8Fc`!%$msuy8>HM#!qU|qKw-zJFdt4vz#&X{+;YL=#yU@
zcXpP1UR;frvQ8k~*CWR$`n%YU`Z(W6-Qp7V)yJ-<|B?N(^>H~1kKNICp_WbtD$O64
zr)qKDxE$73xTY&)eLt}u<K`<fJmh~Y<jA{E$sE8dh*7`kWFcdtv=-<ex+=N73Pj^1
z{q_>p{2Hm&$G8Yht7FyN{RI)eUa&LitC;!wn=8$+DT+)L#s}Ihvpi1!<2tO6)g1N7
zhVJFMqjIOAAcnLkM&0sd1?OA(+xQ8Mw;}%%wBCl46`TQa0_9J+ieIJYLC<fWnvQ;+
z+drr{w6E@rx(%AYE(|f4?_!jn3xe1uLI?2KFQtq{@OAfMk9q0pu=b%DvUtcawVlRK
z>=;;G%sa`U0aNPaJPX=0e@@%PZ8H_RtZ&s#yU--V9VyN!!=aVC|B%LRZ;!tBe>{1H
z+XPWu#Xl(NbAVFe0><rwGHwo?3Vt-%)7!|Gy*RyJ9l~E+_Y!Ka6#?mb)(U|_Avo^P
zNA(mnGn8HyT?JhWMiV#Mqy1=v<Qr(v+GS?uK29_P;%&APgwJnGmk^X)cJsPB7XOYu
zT|hA>i!KL@1ui)9cWNaA)!WFPJ*nb{Cbv_=u-#50&Ve{{B{fTBjIz4Rr&Z1e5)Ouj
zf^PnEe(<-@!)xtDk&-ZV<J#q??xM%H0OZXyQSc}OSRc1)uSwL13JH&2?5n9k7Mf}|
z;vG|PKSRhW<AGUQ^IzLrYkWv8M%Mj%ZCZyBKOC$pk9=aZ+izd8?e<V7jyoPu?VFqQ
zM*rL(E?;w8KK5cn2ucU02z=)_cI_z}B=a)gyUW`96;~*s{eC%UfDSIdJi*;1rKn6N
zLhKAOC#6iZG{FKl(2RZUh{(gi+7pPdI*aOWi)LW+F6!UEEhU6(g5Ba!h_WtG_0*ps
z>t?RLIwb^q;Q|JNC?FJGlmuHw?DZG!L(%lFb50nznp^IbOawL0?>20F%<k{)=8TF|
zE-Zbz5vasyv5DE&aVF%Py)c0LO`7(g-_T9z!h(dkLigjPThFQg`Dbhs-J_}xI4;)3
zYkn}>P4=%Qz58j`3gl_fDGMufw#zE_o39lPMejlN%p(r&v&!tS&;Hmk&;U4t1Ekg@
z^!Bnipb~(a#ni5zcwOc1xuPRi5Woa|aIiTyCL48e=%3;KbfQ5f)GTA5kePhdO<G+B
za2%14kN<;cPeO-)U?iVGT7A*<!qi@%{Un;Mewz<o3|!)%h;1x=dC`|7qx~<aQmtc%
z3!q4hcvdo2v}7c;LxPIcWXXY(R4}}CwPq-&7M}lQ(T{c{L~Ir0vI)y47Q87M78G3X
zQ8-`GuE9`pdz!6SvK@Zf1CKdQSH#8mpm4UBPri4_D9Nx(4-&G>iKRS7q#5hzhKD_`
zbsXunHVhh{30H;^z(We>`Am6RQukB6m-rNWU&7VE)jxMSb{Ja{ipnU~*p^kaG0t9I
zz|OF$@}9X#qxvG}Z5T_bd?FjO9hgImByo#T7NB1#DZoOzy_^^08zz8;@IwHtV1Q0M
z+5J;r+2Iu|$@E}yA&u;}nwsgyxqqBeic(Vr^!#=A;H4p?+81G5A8l#ln3xa}zE>oP
z;hjieKbw-u3Co=BHO(5p?0!`koUl?%X%KSBa&3;sVx-$kJ4UaFoDWc;@RtYBOiE6x
z&4m8)7DB^C#ZsL!YUMD;<eK!#9cIp{kcC|h2?+=n#r>DX^q*DCZ!^g9$_jr%M@fry
z$w93TU5&aZdgVk^KT22tjPg<T{NTc2ZM6tEYEG5H!=G92iZq3M?`2uP7{!&Y{pFo5
zsMF+kmmA&{zcVTrZYYvhcCo!=Y)k#(=#Y`0;`#HFsRUpkM=}?ay@>!R7CMf*5&TuF
z(EbYZTY%@WaMa9aVs{ujEA8)n!(RN4e{ew(sY<%1WbaDAHs`%y<PXSTE?LMqF#cBf
z;R~FohQj=)bSmpvLq&W}&ZZ`;xgy%p&wv#qWS+l-<y*_X&G&ZS4CQi986)}OE~j32
z_+Gt_OYLxj>r*JLsG;7Xj4s>dCzfOebq(0`y~e|5bMQ#Ec!7N+!=)TVczd*q!CWQw
z&TrD!&hC(HR18Msp|9_m5PMotflT~^p<A^pr>#xl2MW-<PsWnTTu9NQdJPy~h)rc@
zYYDD?`H&3co0`y?{a1%*wb1M!1zWLuy*g$SDTxLu=|<aP%DD)(O3NpfE5ZI@0?Zp-
zVma`5_N*VGwi)tF3=0%ybNs{8f}rn{q9&YT$!rf@zV{6nu@b8n?gK@-xm23^<`&uG
zF#JHD8wYNHJCZO|o-0Z)ko)Pg>wC+XdmT@8%0`skAB=gtL1k)E=gojWlooV<C}&7j
zwFd3sw7+Rd0>((r#lz*^)!HZN$aMw_X^JGKuvZwB2}BCBATZgKeYq;!j5L~$!Bh2>
zq&N0EGZooyvZlo7cPIZxX*GirwvIRAn62^rwPfGRmVdRN{&tIJ`J7>KEoF+U?v5^0
zx0o*PV1oo*Wpcq+?ih4*3Nn^ehn252pnmvETP_qp%gTp_lm%`&L$KS?c|Hp{@)kqO
z=>UQ-3m$VM{}>$!`&vpLJp}fKJbnJBmiFj7`A$nD+_{@nT8D0$dNY`yH$>!%f%tce
zvA%pF$1?x9;>1%ftqaAaN7Fbc&g$rPdvRqj(Ep~}GUTfq5@qtCd4dHtMMSw?@>1H8
z$H4D1EHjjK->5H!J4X>%?n{yadK1ERNGmM3l)1{Yrq5q-5gDA{i>}PIl>9b(&!0vp
z8dc(8IGUSplip1{q<z8qwPpr%(VUN50a8`nRbM}futF#8HbP^*3A*I14n66wXRkCV
zf1dwig)aZ0i$l~T7B67ChLRx&_v&^+TI(!R3|pi(9#Qi`d@OxVG+uvCP>`8j&)`V*
zDCQ|<RllO>Z|E0FM_#Q9c#DV9D-_LV&4tzpw9*U9riJBs_ls=$vO9h$|2WnBQ)nVN
z4_sB$OosV;$zvP0LnS);g*xYjz(j%YHf=%EZK3uK-*xc$)L*F8fpr!U-PO+tb9unq
z)dby?F4hb4sPP2f+nxnXQ1{4>QSS>d?ql5SbDUjoIQ+%??WvH@Uw!F!$J^CL$2EBM
z`}e2Q=QUohOVZZYu@&+w+KMn!GJ5Opj1(WTI}wJr<%D{)Ch-}*!ow>l#5(u?9eMtG
zSsqB=xtkU|EzUe*A!BQK^k`i2Wzjf}<L<q&0jpcr<M47kC=rqH7Q<C51#UOt(iuNI
z*J5dxi0KIbt1j|Sc6GQOg~(fMPVyFw3i%ZtRH>o{(KC_C9;tcH7y-{)^U5;J=@d>x
z+4it$wTF$iz?%A^;l1;^0J>1_vjjb&MXFNSIi3M5#>d}zql?eDFt|FDO^R=97&%9l
zlY9(QKemG%>1d0G7vD8iy>B~ReV3piY|S*!$CS5sd@QExAP89RwWlY53kA}Pw9GLl
zp$uGv{X)CXSFjJ6OIqWRnUa&~9%lLmV^FI<wXIYzRjA&9dkBlWm=jXF36(RgEPF;Q
zw&u%3GlhNL%>KVl11iYhHY;LsTZSPHRh5gt%YGR2=$~c^55U|~ZaCDpd8mgc@(bFx
z5DmG`4rgtt1gKsSrok%w=_)#iz3&sD6i#5ztl8VF|DQhFt|~Z|H+|F~hBUkP?;3vE
zr(7{6Q54xaq#>>4;&FZy1(}gbSZc$Al&31Lh<rfr8O0v~+MhBY8AQIYDou^wtZSB+
zZ&tw%F|=r%!g_dQw5=pNpd(I`6EeE#)F8b4lz=)cD9m1F-k8Gu<wW`l7VR6RhLV3o
zDh<n3-fj*(lS@f%{xL;g)^4ToS(_DhV<-F_^nMxSmg?6n1CV_4j?{mY0Q`@l!K$jK
z{wnz~(O$P9)SziV3neMsp~8oZiAPja`Rz3a@1p>-xQq`=FaW^hRGn(nlx<6-v4Hcz
zfJ{hZnPSBECotMWL>TN4_ygZKko)YMZ@E?j4#Q~AbXG!8=uFOvXCr%dk)ZY4+sG}H
z!l2J-b;^J$jT5+I{x19+SWnk%m;Fxmw+n<noBPurLB_b(lfZ)_d8u)6;Yu4ttE-{0
ztq8_LqB6;^0E*qOOz}B*g$iU#Z=d(eeYjPY`TgO+UKU0juQyv4F*Sonk!bY=fx`dx
zb?EN1GQgZ=%BXws;_OT)^~_9m?m!v_Hsn`^@ck-=xBOOA#*H}6oB$oj(_66HTzHvw
z-@G1ENoi|sEsJik=OBmgYEgI%3LYd;->Pa+AWL73@0W8gvaird>D5#t^44fG9ip$H
z(e$`+jKN_c*RW)s$2vs(bt?vr3h~Yivp7lKM!A}euZ;RL<WimXs8Pr83W1#oT`?cp
zwuWk<d}HQx)`Rfv?WvW2TaW?VMQ}Tv*M9<aD&LI0);Z9iuPwoKlh|X>{FXt3PY7;7
z05OvzzFV|p*Bl<<K~=7n-9#-QLCUn><Uu|t#(o*ZM#IHNu<VOsL^4aqce+pqL>zY5
zo$gyOR_8X3DA;sX%8sd>thCsz(x1wQ(hx;5Yg$Q(_1Ya9e8^$+J?qQdczJPMx1M$U
zPOHo#ZeJUU^`{(Ky0!lWQtbu7_nQG2Mxn*|1)$Md_r3m*N@5`Qui{YUnLQmF>yDuD
zd*bANj#49fa#`?Z!#V6X33wWq^RxF|b87nhxKc2jc{jDWO1b=aIJ?G%Lm?J_KDKI!
zKmhw%QK+C$+-owP?bUBgjTN+4^1~B-f4P#`l4rYtIuiwEyHD0)gl}u7*No8qR~u=c
z8TS}o_LMJ=S-vJCOio+yYGO!Hy8XJp>MOWq1XE<ri{bal7)l66H~!IBn+QmLmE0k;
zPiejpiLphA4$Pt2`dv<=9LV9Ho?JOauzG@?ny$){@49yo@tOpsHS9RJAuga~?~_#A
zX7^Y46Ecy3D>GX0@o7kSkpiTrPz+1+t8#v`8Ry2oqPPbyr*Zs{by(RMX6Amc2djsT
zG=IeH%D92ljHUu<1Yd5SNkl+N&6telW!Xh?4x)Srvj+AA&#YcXwag6kYIm|cz0QqC
zG1wr}JeyCh1$3%<B?!7Hd2}Vzpi}np;cVfsA`j<~M6RtuwDG0H;~pHMxfDAIA9>@x
zGE5*Yx2ZwCP3yBgd-&%*U?^?jF-4LYgKRQ))q*XK5?;LmJ>xH{K_0puS*G+_=e(z{
zCj87bU0qZiniRaUZDxt~*vg@TFD9_Uosc#PrwzHAWE>EOsV+rsnFN|{y_Tb#Klqcv
zFB)%#N1b409gfPF?|*I8Qk`Mr*G9XCQ-rSS(OcIwUkHUNbFxk*n3#{{jbL6ZKX^J#
z&1-hEc(0an7Y<0>Sp6|il2lk>V&{4fk_?GsnOf7i+ib9V4gQQDTcS0#_iBw2vjvR~
z0KPw21w>XXf-(SJJ(}Mk=LjI&AwE0R8j`<w+UXsDJ*YzW_do-O5kL*!L5u!9XkJ()
zcJHfk1SLyrQR0=K99UZME5B@=po3Z)JP4x=8F9cxxDD*w-U>!e1kZudU}=;fGx(hW
z3Yf4vWEzwX#b*uAd>NUG&J>I29k=W)F+UV6+AD~$L&V*{>ZFmKP1zI3H*PV_XR*~<
zj+w|caYRfx8*;Q55>s4UKl^L92<&F>195?}L)qf9{VhHx1`xTp=dh+^1DN-yAJMKq
zt-@!%W2C`ZurQ&?B>vZ}99Z=U)tMubZo*?%3Vi)dMz>Y;9T;ye90&BStsJa)07RHf
z{8&SPI-$<ZYIScDo6WbK#+#BcDt?PNB@Vup5$9j{35fvOTaX=J7NDv7`1YMr6Jvww
zK}4Hd)i_b0Zt#3_V}fK_HBYEE8h5t=MzJEoYy~!h`afNG#=QCIT)Emjb&nby9%^%m
zcFa4HLi+>(a2J&~Bc~o?Xqgdz!Z(^XsFuyWYX4pF;fZk5J<<KTpp--WOjdOF55xiD
zw*6*y`r$p9kbR+NEBNgn*%mC}82m~%8$*i1w|_2GZv5Tn4?e*fKl7&CtB1q?DF3Rs
zTM#4Jkkd$N#X_n$pBR?vIwcN3sK?5>3vd!-{5=iI9jGZiBltq7jt#N4ievDvj>&T&
zX;}1)0+dp8uOJk=F?Mq$cG05!qrELhw`DMQWAS0UJmqkan`lIyO;m)G{^$qNn-9Ze
zbm3SUcLzmThS=FagDnxLAzUxb!&1ZYx`FD2ZkW;C3+$kO4#7W{jbL={fH=!%BY**$
z?ZpYo4Ffiee1!#78x~yu3Ns`$l07{W_B|?1dRZaommPb2H`2N7UBLVvl&$dBqyMsa
zKCQGE4lc2dL-U`Os%x012npCV4y4uH9OEg1U0>qj^5MhtCD4S)uYQ(D;A7;U#8M2!
zmSr5fGGJ{d4$>N4x$Gp{U6&k_4LO(04YGkl57M(KY3>ph1L=gh1k9IHikQ>EDiX+>
zxGeWJQ(du7_2K}<9HTz50@<sIOf!AMAhP3&KF^+3on&+6$%`-<@gdVyeZd53P&DF?
zRCt^uw+kI3iN&pX!EW1g{ra!w-$QaNR|bjC^}Q@jnxrCKB4w!#b|rJ3ha>iGk5E>k
zJ!HR!+ZNH?bY`e$@6&!d`!l_j;mfy+&SQV}mFgADNH!*{gV=sXq>-so_3)k4lcgq}
zWT@(K-DIa^2os}<<b~!tV@vGmBZqe!o+(nTg+Cr{Wk%GQ0L{xbD6IqYEVaA#hf&4~
za{!*IL+47^o*}<j3kfQ_Hp0f5jHw-oxgk#)FeM+`a^XP5@TM02=Xskq<70fApq?Tr
zW;fqSq{Yl9(dkT*>>KeAMGJyU&8e_73wj6B&{zbJId9b>$&r(NLlrZlchzm(J2K;N
zz-Mvm(B1ez`b<<28R3;y{+>2Ujc}ZhllcA+KQc80<=<*I*Si5D{;22Qx{nq|%nq+M
zpB2OA<1gAM)A)lNbW<=9x4M#teDlve4;DQ_<~sfJpz#Oa|39|AGAfRy33qY#;1(dr
zqQTwW-8GQK-Q7Y65S+!`-4+WHLI}DL+}+(>!`=70=l;7t_Vjk2nXanto~mcMs%xTk
zbz5__^t!CHQDVsHr2^ls9tNv9<w|RIe3TZORs+Pr5V$kO!bZ6~VOdE-qRo0VP9aoK
z;MG^Cw>kz!<9cRb%e^!ZVdGgA^lANmD~*ATf9S;E&a1h8uzDIXqx0|)EnGi7Y1WK1
zRnw3%FHjT89#;e@blfie_8ryW*3;>hRNC03OOcaz?m+HdbsDdt<Uf}|Zbk@GCqfyQ
zep)8w22|P|d8O<J2HcR|G^U2?e-0(#@90pzbK`MwI&CG=pOKUB9Y9BECA-nAy`rN3
z98Y^#BdNTj$X7e`u31I%(q_{$_QkQ#g_l!XtpJeaPYN4)sL=pMAen^Ep21U*Ti%xM
zisXiamzEsJ@zf;NzHSs!<Cak<v)IcU&n>L@!Y5?r?n#cI?|d?LkpOdVxwW}+mez<+
zX4DqI$ZhAnm<EV5Gke5SQPn-`=tV&*YV7TMt9bq<>|6;l)}X_@8>cM)e-8-c_56Q+
z5_Q-+4>UM>lVtw_vAAbS>zks&bQD_8T02{L?dMOvY(<s`lW>WE)F@;T5S#+h7Re`9
zOz%ZdI?FD(`2zz-!}{kns3!S~$@D5$xqxtnqxFFH*)Iw^Mf-owEzjR0?9y8IU!w@C
zpm;RuAj6+6O#UEUteocz$yK)$oVm;sPv<mJiH@~nYCamKFK-c3Zw~sKPZ{k|kfwp@
z<EvoTVfhkVgUp@jo`4PWFQoA`8C)G)BGG{p<-_}!?}r7WP#F91VU`F+#Grf>mde0A
zU0W`8Et#uZH2)Iwy_HpokLc7-J}n!c2+!Qgq$6}FA}Abk>T$JOJgyWb|7oZ)tNkxM
z=(LA<6m64KqRPfl3g}Rcl&rm<ltsr_RiL7A$E+`Ds7HD=bdU&PMo~IXpa+$>it7ru
zXFw8Np=A-Hsh@mg6tsw7OqlI5Gd(x_NQ--HIq{ebGSC=!XDHbR<hFz&M+E9v`vCoI
zKQIx1o!kh)l%LVi(EG@n;zEX;)JPu-D8#hyCjL=;c}Rww3DUfI{>LR-T{xGEh|5Sw
zqS6chfQzEqVi&`oEhBF#rO&~{CJmVHv%#ws-*!IW{wMzrbf|S@k0|ZsK1ka^%@-e6
zP?rV+%6`TO2{eTX#s{RU{a4|=^dFo5j9`6lfnSvrpIn}9d0{PiiBO#w0B<bem4sDi
zkCZVyfV%Q%m-Ob+@auehBxCIEM}11b50vVV&>%p?dqR$8P%K+4nAYRkz`)&*JKotz
zb+!~Vy%d%4jTFPCE3LN9ZIiBbW8Zni6yWk{5aC$_MF+_mOE++i*42BnFg~hQw*LGZ
z1YdlYsU|y^H)bm8hr?RJD)9L|b&{v+B^hjvlH&I^<_2)xspe1+h^481@R0#*qS67b
z`n^rpLTZIBI28UGL9nxVAs-uZp#V+f(9CFoF?a>OEbh<~5?)dAss1$1){k1~2sb!a
zTvg4ax1=S!o-0EBBV|EkbVBbBqdMNrUt!EUqWq;&5bayTazGQm7t8Y7;KlsxcrEdJ
zipI6u-|;YhqNG1|pYK%0urVTcrD*<M+Hi8fjBZ+fy>|P(eJm*`#5gc=mIHf#wflKj
zm%9cL^ii7peQhS8Ko3v?;g5i=WiI|FEA)zYFX-IlINIzv8*DS+m1=o}dzAjdx~RB^
zWI@DGmgYtx>izGvR;=uFEN<6A6$k<g;Z$jGADlYumoV|v0TU)OkkwS;JsN)M?Y4a~
zqlLZS#owp^h-Ie;?@||Wd6}4ezNajQajj)mo2_GjHmu-SPn19EG^W-Tq`h}h3+K9+
zh(a+a%lwY;QM2lk0W&BM1-T*vupymkFAt56uXG5N4NWR@g=*Pu*!n=p*5{d6ru#`y
z5&-^Av(2PgS;=rb#4RT(i&c@?d#u>*CmS?E>{t3)&^dKyeW>2gZgD8QBZ5e4!<!zE
z0K96Ikif=HwbO`Ar-Io)iWB?2&M+o8o9k?;(S-&gFIIfg_*vOYcqx<1z^Y)w3Lc%0
znTXe<eDv%S-h{~<gbsv79QOmcPGI6AJjN~U+OL9G2Op?!{docxK@%WB-Jl4Z>5{}H
znHdifkSHUacpq#|v{*VwmiE|F03o<A26Iw=oClYK-um=%HRt;9e0?DVAeDh+t}<Fc
zU@{(WPzof|G~qu=vxM_ZR##l(yi5*YtWV*G(wKZqRzeS^P*2wLI0&oDq*Vk%Ddsvm
zXv;{C8WVZZx5O^y);Ih%{+k#zi46kM>U!~LNgdjrkic&uq}o0vmg1Gnf;J4$%7MLN
zrjYA|%LP6qRxK7(Vvsp&#Hh6cdyc6Z$3)d(fEUyMcpgGNed6gjTB|*vex_<Va?%#e
zMsPlokrwvNWbm>l@rdWi*eAB`bAGx>11U(1E0yaQteDWb%GoPOW+8wtfsv~UE$WEd
zt4gC>Pd7aH=R24oO>RE~I@yeO3ppP_Pbc)wBSdXb26p}#CJ!=aU}CW~fUSWK#xRwj
zw1-;|v+ZlqLz?%(2U_Fx5tXgE45(Ls8K_S9%L~OfSV@8~r`aW1#G&&6tz<H)jl4@(
z6MlPOsXi)GluRHH*DrAH%R{@4O@YR@PwQo)y)F|IqZniY_@5DKIH7AgYdDgIp9>D#
z`64gCy&e0xFz2|$n>v8RO7w49aFK4W`xjcSr~(O`waK5sr&H(eUjDUC_PmOYr?nr$
z*>nHtdtO1!CH-73P#desy(MJ>TLR>EFTP=<yPzB@X}npA)gWQWWVq}!WE77|mxp}B
z2z!3{c}ka=mdNJBOAzOkrJ`U}Wr)N#?~5r>PL+Xw$J7O4LEIi(^`9APeMZ<6@!t6@
zLgwH2K!jK>K_>tcm4|&$@j+S41M+onR6BKdOV;zGRTu(*?3duxAr&>=a$@~90S7&T
z=Y;JhQ`|p?{otV$UPw8<7UlabPH{b%D%rxVIYTm?k#=gD6-0wC!h@}|WD+aQpx-P!
z*CR&k`lqX6_kZ9nc*g%2m{KhiQ{3R029s0vair)m-UoPy2RcxDh%Z4*&t-Xr?vBbr
zsK;M@L2C)pAT>)vrL5YzG0J@&Nu#PNrm7`Ikf9}tS4Zf3*$SvqpWZq+K}Ju9+2@hu
zJ5$ck>k$g=F53Kp|G1)5pNJf;pEw^dz%@284s$>HL+4RqWIDUeQB7Mx4WSTsYq@#x
z$Vr5zck`n{nf5C}bQh=Hw+P_l;J5pn4>B6awC95ipuY}@VaYp<aoOJpK`1H3Hqe!D
zP=n|!&~$D<yinzz%$Hbj8Ytn$A!ToY)~a}4b^1f4aB*u^kU7kz^yR#$Lbd$3oHe_{
z&EOCAVj$HoizsS_{xT5k{`$bfGPF#pw+P5liC0ERBjFT$u3#Zp_n9=(xf5r&5&x$Z
zv~P$Iyn2(e6a9&o2l>b^AP%Ce#U}vz88TVRusn^fxGD07NjuX>8Jc_wp-Z4w%M6ee
z0NqDW+zw}uQFJ@}?|#xS))D)x!Te6i;9N*epaO8D0beraG-+r2-X}AC9<tCV5(9kh
zWil2)?pCRO9t<c`i8Ya2G@+W>HB}Jf6)>6;0+o0f%_y>z7%SSy-A_>OUowIfRf$A^
z9%_b`xe!j%ppbEJbP@~GG4=u%dJhRDY`k(73RwNbX5R#N<8{BB4y*g`FoI<zj}h8Y
zQA!pblOQ99`#Z0~rT>x2R08?3b?>%_5t#~-cn%|jb`)n5c2QdWPDCa|sdHsiGK;um
z$H7S^gGsp`X$5VPS4tek@-VZ`<6%0m!10%RLMuI5H25)~nDy1{L*64KU@hTQThEnN
zi=t~@y|mM)Z(h_U;cY(+defGRM6!j};{~{ga0)H5bTEiO#Q{`#S#-#$PB(OEx>mFB
z9{>S~OqEvNuC5p+el-Bk3RP#FsZk-=_O$H=edzv6ssr4$CtktfCyz;-cCE+nARozj
z9_si@k_{HB^2?~%M70S9;)2hBP*q~tQ=7VpijZZWU+$(`m|!X_IrankjdF!qOG>+{
zkA33Y#FZ9vLySWq`u|x_os)a*xO<^LRVRRSva=Mf+<LyvLrYcoQxqteXZAiW+}Knf
zEZOQ^cL-Z|Y54^m2%u)2;<&>f)IS766zOu=dwlc?qT@*UG7W``*wT9H;X@anSfC#U
zlkQgQ6VxZwnT{VhLFRT%%;aFSpQKQR`9QD2mOE9$rdG7^1r*ziY%u9vVr_6bL@!+^
z)lBQob`Ag))<nil<8l+fNO#LO@W$(bz#Zv^^67<+h?ZhM9M&@_KVk`}owS*AQ`nPR
z%#;s3(j2rRx&|QY47&aLDG38H)rHvu_0Pqu%(u*`O8Z3EjmV_xsTHsJ8r;T=kV@$<
zXAy2MKcR-huS-;v&P0wzC;R3t)F$6b6MsdLe-;Wx+(3F%56Xm_bq355QVsI`YZh!(
z{xix?ACBI-u_cn2)|xFs@hRP?bxO9j<K!D({50<=mF4Zxski9ol9sFApZ&^BRa8Ec
zkr>^9a7gw*rS4~^?dC42*30E(ely};&C(Npr@P`g2ZDZOa!@zKk1CWy@8#}KH5iyu
z@<J3%9Z#HiOfrVNvgKq(WNY^AG*Z-@ei_fB^YB5V$Uf44ZUSdH5n%>-uU_X+o8UE$
z!aKXhycp-dCI{&ibOxS=uJh6GWnwi4{cX~qexDYP*XSF0z~RVm5=Qh}tc0-E^Vb{>
zuWa=<Vq6xb#k&^76Np_x+Un<-mSFaOM9Q~FIAUCW&!_qiVoa%BG!A(#%uWDPY^Tn9
zRF$}-+T2n@c`|m(L-e<7Fk1Alo$tSPFkr~`%@heF70xBO`0qLx6%958Z-2U||JPu>
zmcVRG_|749VH;G1^Ut>__%@u%c+>=Z$VJj(h>g(82R|S4`!CZO<!W0Zn9;#s+)e@!
z#G;>c|Hp05(@VTjFIv>B>}ez6sOwO5zr+!bU5xV0N-C%bd>Myh=khTDn4Q84UD~YQ
z<D=?A_)q!xi8JJ~(|8Cr|KB?mRy~*?pTBT1-@8ka@*d{bXnT84z)n2o!I!uT<!V*r
zuQWk88E^)g&C?Xxw8cV*|I;enk6d<TsCMaI4w?V)oq%$i(YI7`Qdr5S2gWPzQd?|(
zN%Ca1{Q7}_XISH(+6s{%_&nO{Df2*d3Oj!2x!^w)L;9R`b4$Ex#Bk@Ed9<x5_@U%j
zKrI`r%K(j}7=rlw%fNt1=^W5wIf8Z|xWmTN$6%B>VdgY%#T$*Z3g=YJkMV!{jjn@G
zkoy1ek#M9hwU-R;Sm{dpDhg4{q;l3s^DR4BKUrPX+FBq(-Y~QHNv42``yTi)ZZ#D*
zl(M%Tl@@-N9cT|Fz!A7aep1`3vrUUQUV3blOkUJWNUh9locFQBn|tch$ukOz(0|G}
zy7liUK|AwrE9yEE*_p0=<S4)lk#!64(VydOZ{5y6=UdL#Z_U<q$8M2skQX>AUcE);
zfQRzUU8F(~*PpCFi%I<3N<T`PJBx~I?i*&`K;TBYbYStd@n21E)H_u9Alo$0@am#e
zaXmSf+glvavHKh5Cy+DZQ028}V2iZN$xn`I5kUm^Utlip9WGEp&=Hd`Vn9T(m~BgS
zx&Qpg>n$X$el$-v5Hwb1I9c`yWQM5VclD7i-2L`Y!bnq?KbMc|R|HpCEbx1`8Te<?
zYa=u$V<GUr`ulr2d(bKE>Omt;Ni5I``Y-M3yrC$>U{=mW4|U~J|Nl6(f)#Ztq}l@=
zTp*Q9woH3<gei|%k_9|UtX8Qm`wg7q<dtIO*n98XPApmxcb`;x*G0YAew)51eHK=m
zBF%*aR+6n7D5Vg=E;?=&@tHjirdKdh?25NK9&<IW)YZkYRMHz3G=odAdwd*`ykl(k
zGq9?7Q;SNcdiLL&ru9D0Mz62UeE*7rt4YdJr}=SCqDR5!8HhDGXa6IXAt&-{m;<xn
zgDyd~$vM%&5Z#$IYQ)~xPO}lV_<Xn*!|K^6Ez<79^9RlTkgr^2qu@|3V7VgZ!Br9^
zmOHk@CR<DL<yhUl?1=+q7?*84k#Qw}a@X%#ICK%<cSV^IyD?t@ccMVbB8c7Ado5*(
zgm#dqO`^KNgI5Y6YJX0LjoN^+1f9th7g|k(ZEAb2q9*RGy);jggosT{@4~8Yh=$%=
zzs|nx#<=HC`7bveMMNCKN2e|dcv7u^frwaDuYZ_3$5l(>Zt|%*?Hb&(O%KLUo?bZR
zH=P=*97|UCM8pUB9RQQn0hTXe9`O-bTNtaNv?D#7Ox?-%N(25nh{G+Y{5>Zg7Vz*?
zL3aFxaGN^y`M21s0VyBA1dUjeMLO$dDyND_eJL`|<1$!ZhsV(j`0OZ>N-rx+_28}V
zi<L~zHMErM_D6K|LL`5c9<D%c?-0@jZ+-nTAtVi{eaW+B2~5?-Z~-bO#xJ)b0|Mcg
zhU)dv+fjS$EN<i@&;Fv1V+07~-ndG6PFBd&Iacgk(O;s=x(}EpvaXO_uKZl`^w}XS
zuS%YKW84k?2+9#aP9%&bOi;_DjoHGwi1rNaMgtFHmkR@+#3tQZdWlXwqm)>Ioemhm
zU%~(dg$R#Zo3WQUwmpmQCe8rEh#f=?bgC9JA<CqlV!5^fMgo^97_2olsD@`?@+0I;
zzs<T<1syC>2sCZTZw9)@{oc;3xMC@E4J8zF7~y+IC*Azr%8%ulV4?dzKam=*Zg=@M
z!ekCRwA3###)8IH)866NPl=J4154w`M5`axz5FOCraa{?Pw7{2?z66KNRRaXR;3@l
zn#?nD?*A@NU%pAXses&q@8D9MM)?5WB=ihq%4<58Pu0x9Eq)ft&Y?<5go@V+hS>tH
z3j7~0YU}btn3Ul*J|A0tn}=3OM5Cky8R}q~FGe<pmM`<C+KFi9DPZ>8vSB-rGR73O
znvbmHyWN*F@gZFIQhBeKG;I3;q=fB3y}Y6SC}c_h<)g-~QdSytLt4GsNdGyWeY4R1
zd^5%ZD)kq8Y`0w1P~GP>U))rCSrn)^!KIkW(eM|t)$n=f7K|hI#PR&=$5(dFBvi>F
zwAzs!N<hZ?4}rPq)|fn)H`S}zCRQE_sYsG1x0Eon-`9Ea-B>m?XZf?wLBL6@hitko
zC8SmMs}gg;(YXsWV+8kn4@w{V-gJLHqJ<CGBpVx$u{B%xb0xbT_$~>u!*|B4Ou^Dg
z6bSO$2k9a}*T!B2BnsNkBp3adPvZg$mTCka$u#t~)#0IcziH~Yh_xh0RZ*9zp;=f*
zGw!Q|*1Y`pe+);a^lUgOH|XE(STP%CFfw^$X(BP(b7_GZ_c-$~2n00aQwe8n0KWAQ
z7vA@b2yvm1k}FitH!zn4eLb8p1;P*k+YPA|RUFW@N)H$IJlO{!bQ9EB*T7Z7cx<`C
zs;KCB*_<V{3PJ~fLVh8EUvJ)0(~wa>?MPk@ERHS4ql0iD<4SUkRvhe0aL|zw94Nhg
zEtt*NXmx?{oYnJqH&_6w7J=S!yQ$1H^i4-k$7uFw2k1O=ZP*kR1@8aJVq7XzH$*pQ
zqenqH^}G10_797oqXNN?k8^;;#Qn#Umm^<`n(%dW+aNU0ptSm*f@WX56r&I@f-~|j
z*(<WK05b}t$6u!hY6jVLwx>ICW447!|Kwd<1}$5G{In4GwoSE1f6pUI<RjIk|Mh%d
ztnkP!7&g&$L6woQ7`Pf+KI?2qYohbqlJ|5vAc7FNvpbDb$~l_k;_ATA7r7eXBVV^2
zr{kaUN&xM)z9Bb)c$Tw9n>|lDw<b<H+eM1IrBY6bwWhZ<hC+f|6}bDadu<i(|L}(S
zw0gGPeB<&@Jucs4GdE0lg7SgY&R#VV6Dn2SJB!j>7eG(<qt0I|&QWQ;;?WEw;=(fK
z=rmbJk<pYa(97H~9>jG05T^RU<@RZ-lK4Og^$<REzCbbx-4zESGz(D0J%3fd8V*_E
z8_KdQ-gvbkfkkXrs-STqIK9@nk|m*>Xcf^ss+rb>1&GH)3K+v5pwi}}NWmVMGa`u5
zd}tOu`Z{hZ<VGfqcpN~bN}q}p^P;4SqWGibuzHK9X+4SU$^#mRO2;oFFXUFmGPyE6
zV&n}A3)A9nNSk!z=+<u~qX?INX>knt!;vrtx%E`Jhpu#-7#ONAzQ`KpObvwmCu(`@
zD)~@%|Dm+&vFpH4Y3tfh2S${Tvc3c<+wPdX=SHpXYp<qj)PBXfh?bnoL!a5A-|U-i
z9^{_`D3}RVdqCuc3~Fl!rB~0MYcG$r4yX);@Q7DW#?C_Wzd!BaQf#q23pGa$LJzk5
z>sk4ym?8r)D@`RbsnL7EoH*2mh8O;Z!s<bwX3rmHb64QWGFUE&g@&7{d;Q-u|8BV)
zW(uD-gRkwG-=qy88vC`~+?GB338{~T?^Q^zd)&p>OuHc#U!^58xc0(!gJr5&DgM{*
ziE++3#~XIF1d_C*7?e`)KO_(Xv@*?vAqqbWr~xNPW$|r6xVW@0l@0UlZ^=S$1fn7g
zZ=r}F!Qtb>_u&=M8W<R9Yp;!d<?rqZPVr+YmCZjcA;1ki&8w_D@z|TWF*loF`J2#E
zB`$2Y1j9$r9}mMyFQ4hz)fV4=v6RPt*B$^Wcb}mU!$PR;#A}~__XD@Vwa38|Thh|a
zCU;dSEI%pV+7{Cm&SVnI%!AR5={U#3(?X!^p0!t0wLt;pCGvi>Q|oH~#c_9>$+sSt
z0;``T?PPDK9iXjC)Y?-lK^hB<q5I(BBN<|`&nGGA=bCcIzwlr|T55~Is|otis&bF&
z;IV$3?U^h{ZmXB-w^e2~>X(2N2(Iv3&RzgP^<5aY)L=NFR+-5YKOSiI^0WJOfF>-i
z3w(M$9D+IU8W^Zp`l_TdYxdm8*245k))gb&p-QUDGNX4mBKa7{RjJB7B~rq=>@;2Y
zaFeY)AkSVN`)^as^L2_&^*R4g*{VC-rg)W5)^0T1uJd@x6gv9FQLSJGV~r_0a<bD~
z!SLp&xetgS=76<<@yTD+nrBqcrh9{fmvOKnac{iCX!L!1l5nR_tQ=nG61dz;f~f+)
zF7t=8ai-xMULb8}g`btTgC~m@@U44=^}N0Ani(@oPD4Erf={!9+BoLP1_zZ2qA+Cs
znYe%+Vuv*D{p^W5*l{Qd1yB34$9j{wfaCrgJcyK>RPQ_oOK*bV+DhO8r|awMZzEv&
z!pLDjZi5nPt%N7w@UNuWNfvoosQvkH{nkua?1~aDXi>$K4-hvbG2%;-sz{y`@41EG
zWJ;;Kvx~yP(`j|Y@cyYX`Ruf#?%Kb~v|L795^9mmtfz%vP?J|!xWak5+K`rS*4wn2
z=OfQLhj+*hGMo&s_Aq>l`D@htD`%w4k(~vR2fzH}J3C*f7J&py_KfYsJAggbdG*|j
z;=Gj?<v4>xrZ$4yXG90g&qJt8!=Wq~5^Y!3=LoGhpLq$BRHOl{7?fH_bv35*US_%6
z8#y}!B_~;|_@y=P%ch#WO7}Y7Vh$A9fXjB-bKB;ccLt(@3fi42;tLFHI0u7N0qnv>
zvWlp{=P$KVYrou~dHqmHgdgWyq!LA7I9aKgoE3p)&>%uY7M(CV230+fJlg#%uzdU8
zG~rXqNQ}ul|Ex({y0)%mFElqwIuiKY=uajsu2Sp)k$T;3n8v<?v!b8lE2H=hoK+Sb
zu?yTf6*GO{3MyIG{i%vip@6g8|G3`g?x9mx)V3Dt*x`gCN9swJ^hdAMh&IUxb!4<p
zWs572-VRv$XaxiBi}c#1Wc8Y@x||1<qqRNqAr0%zFZlR|jjXp{_XfA~943#A`?K!S
z9b$b_+_SXx?DrOJ4+P^yY8h2Y^M*j8Ad;i~FJ+<OfjOEtv_|CM-R!%HGU7I&;Q3Ti
zk^7+@f-~Op-g{2Wmws-*jp7B*lp37)!MhLS<84p#{bB5S+H#LBa%XlPk+xY}GZ|cE
z;U`X713O8#0TbwKx>hc7!;X8Yz>q14UNGT}Q90v$A+E5O+>-^Qqtfb6EWY0-k`~L0
zgt=D>@N2~b<+hYCGC7vy*^2{r`~J5FPa+G2v9M}}kF=yai?XYIfnOjH20%Py_VV2H
z7Y!eY0N@09LUZ0H7D)^71gK%s=b<7C&sBnG@cWsLpT0A+QFXCW&3~cLadXA8_B|sj
zklWjI-mhPUL5|Z73Qz6ptyW0FEzuPtl!)#6@ps}&W-iiS*<*&V4(efz0pPJ=l-wT&
zN>@b|PAGGx3d66GlU9r_d>p1-%uH}AQ)k>*V5OtiOPzy?Vf<yGq7A8|mL8J7>BdJ3
zo9;ev%MkDOrKo;$7EaHZ$s9Bz5=}NOmmiwPs6|zc`;P9*{XRsXUh}!vJt6`)f(1a-
zdl@D~1DK>Kn8pBw57>>qzI$ct_cnOVc)+gisvCi*?No5m$`7w`o@vZ6mI3^}K};BC
zru7ETJL-VaYhB5yLD2Ux^%buqAgN*+fblq}@nj^iMddOM66THCS7@tA<hu_$byEMW
zBv-{O-+ccW{Ar^P9E3(-s2a*uR{|WdmO`?WyU^WGpHc!tTHCS4GZC6^oS-^ZsAc&j
zqo)YGPWcd<RYYcSxy7*QUZxfkE7%q&cPC(=@i2x<o4p|Tr+<B+<=&of$&a>yAF2Ao
zNL3$FRm_?uso@aFePZEV`H@03bK1<oT-}Vn#FDBic%DyJDDhvrYka;5aVE9U!;xMw
zs;r6<dm+_+jN)b^7yo5o-c``O+GuJmzCKDhU+7KtlaXuaE=SP92SdJlDD=t*;3_TG
z7{_gwPrCH+IWfCf$z-&yqqolKr0QsX_9!*3+eFDUhV}hE=M)O5K0D1Et1aq}#Pe$1
zS;>;|u@WP;#CacBa<To}zDuE$zB7_@cc<~jFnG=QqFJ0E6OAYghIKn)=0VmZ!%sN@
z_O~JQu){}UG}u=-S$NHfCu<3vm^82rohepaDV0x-0<UEzA0jFUr9OWwl`$1IaP$%J
zTIjOj_clSJAp}HW4n&YhK<8ua@R)?I+`5w0mqwBE=$1i6Ik-h}y%u&#QB79loxL<F
z-p%)a#1XSTRY!U#dV_Vvb8&yZN@S#g+doV>G~Y|Fj`RrCV;FOv;TymD3Qh6^MA_1d
z@59CT+LL^-`Pc$4oo8)8tNR`IG}0CNAU0sWkgES-V+-u2uLg@~?!4-tHp$`om(Q`)
z_AP{noBhAWI~QvH-b>^F#7WZ6Dr?&JA7jV*;5#dLYHo%FO)IgU>bAB&{D4J3L!8?w
zR!+3;>}s@*;`K%A#k7P-G#E-%4v|4ANJ09C2eu#n>#yd(&!&_TYUcRWS>ui*qj`GB
zS!?c>E>4;G;d_23@hXB}v=hx!sIa67%om&4=`b&LIc5&m?S~l)AN6kE$<Fd@@Z`L6
z(9xtSRHRY?r<B<H#^3KHni=8fQ6YbKuNs~@WesB&c95h8n9U~j_ns+Rjm1tWeH=uK
zk=jYvp}H6`mA;aw={;ZW$t0QwJo$X|Sk?2*OIi>H4??x|Ix%LKi$`B@0({z1e#{NP
zt;mm>hXpffEQRW>7QVAUzbXx_kjRl>AW#>%=p(;??Kfqx+tnX~G&9&@Kc4^SOc5P^
z%D1q%tYCS8WVMR0OsUq^fKIf;jI`%09pz$KQfjQ5fRxrYvTC1w7B`AL>A=ysseNNg
z+4B(flZm5Rrt6&*InXNNc6;9M7xvB(Wa?7WLKP+gC|^A&pJci!_f4MkUH)#qXlu4*
z{&Tu-@~Z3?oI=(p4F<5RWz)&8^4Oph^zX+Y8)I-IOXXJS)9V%9b=|p<?4l{p11>az
ztMH^aNc;x`3(l(Uq>&o$e=Sby3AkXpw2>(`1~3{(YH@F-Q*z}LT7$pPsmPy=g5>rs
z71sZ2N)#%ns0#Kkx=JcjqYQ8S-8bp!?!1T_yNk4d?u}KGDO4<oxRKWoM#RJL?X`L6
zNS0l^dV}({2=5np#6{p8p!wISWzuw*UHM1#>4lw7>MroUl;h~%Dp=KJ$ij0-pYjV-
zm^weQv$Mwnm6Y~n|3=)}{g|mo|0?D1g^ME<VeAA)j}qq#JWD%U(1QylJ|%)-XNIf&
z0<ZNin!+<a@}|K}z_l$hRw$1qzU0|$<DIPtyhH*Yb2XkmU*h*k|0t-d+IpkPUII*p
zT_?a<ANN?8J9o4Cmpu4HHA|1=JCy2pu7@N=RA^$cm9HFC1$q4$KlU~{8uf#HYX~?$
zpeY4ai<3^G4Z~u+d$~k~yM`h0J=+f507l2#AK1t#qPYQ{dEo8!en3MhH0AVar3}6B
zB|hv{xF?7UJ{d1omNMVAg#D@$lFH1&;w2tF+FR9UlHoy`pgR)WyC9rUDqCFIE{2rl
zu?y`<e$Iwa$cAx*MRokd$+IUT2S47@pHv@hBZJ&3H67sR`Tk*n_Tx2wZbIw+_Lsc(
zu3j2`2&Bb2t^h5(Wp-0b8P2)Q1d|`Etx_KzHl%JQW_{C~9zB_L#2Gvq8YdC_;R4QG
z7%x|(YvZ~|*YykYO{>E<o9d1h!@LlpSQ&&XQ26sl`xT7@vqkL|aG7<<rpH|(HKXF9
zO;jwzVT=d>lK6cJ$Od1YT!sts`gcnj_KR&op-O)>nLn`{lK|POTayaE9FWrYOs5vn
zHT5z=D0%;WuDi`Ye5L_o?LOHA0DfRkvh`{Y_}H|=c~@XJQ*3|(Po4p>9wyzIX>+1I
zinW+4X1DN?VJxH)i}1~(>+-RKU<B+Zi8Cq%A&Fx22OYSMS|_N5;<v^n6GNR<IQ2ua
zWiS>Yj<2o77f6N}ygnZfmpmzRWR0T$U~7z*s|>|}4In^^MUws?qt;azbWRjfe?I6D
zd3h7ySnl5w0<~VC<!rwhVt7oZjVq3w9f|#RuGX5hR%Xf=0kI=LND^*D&X@D%5&>bY
z-`^8M?eRaBlE1IdVAj<G&)On?BV$2m(-$>l)_%b7za1Sk1|hv?fipJ>98Z_XR3`4Z
zUeyB$M#tJLA-Pgwu4uXdKI_aHYPR~q1E*2s?#38Ei|p_qQ@|I!A?S_+pq7S9C_Du3
z;4+%0Dhy(DyLS3www_SbKPyw6UgHi@od^5Hf|QM3KA8VBWd1{C92d~$ghKE3^90;X
z+ul=*hjt(3CTj>U>wi;F2h@Hs#MVb~u7A=aRU31eoe_qzI>hY6a5+?S^?11qp=bob
ztFK-FP9B4uWl2M51$Z$e(D}UmUfPeM@w#~a^1;p2xbE?a#kqnSbwTs)1uEA;zZzs0
z@4Bf8Sbmdq3R%^ZfV9Mc!g|>I=fN33<uzXUKsIB(XWN9@V9_ZZrPKRVG|(4J4aPP#
z&JFU@Z&i+MkeK~ZxVu<g5a~6c2(`|g7NVArhM6`IKULU(3lJgi25qBK^x^}^QOxq@
zFJ{+T@bCLq@aAA}ozSI?onm+T{C!xbyTG*&r&(wh8YcA`1!vk5z?E!tnGdx59DG6P
zd|xg0-FvZ+*HAx=j61JVq?SdNNOvU@0%}Tdf2x<_onQQsh6chCzEouYJ^Rf_it7Cb
zP|{;+ts|O~f8S04*>AkrOWUp)dOD<Z=AKMZxKNU}{y%p|J*y(!Wo~ZWdbhgfCj9$I
z42FX6v8C~RiJ_4}+wc-eYMi9wP=V4_?&-D688B+MHqKh@JHs?n?J`X~*MMNz-Kd}`
zcmvayDE_YNP~~T#c)z<Gt-^^yVCX#e6vO!<(o*S>kIcy@9%*ywtoh`t+W|p<P_w}E
zC$KQkA8uS??UY9@kd`qC;(ovoeBC+a)DPlli{<V98$mdqYHH3&r7o7$_TbAj24>3&
zIvH+U%y_VdY4l*#d1Lm8cJ5mUIV^sXK0j*8l(thE_I|zS8Pl3YSETYq0c*{I^ywkk
zYa`{){LlS8BVAg1Ku?!M<`7~LPRHUnoS-2klFij20mqk!;<V}O+)p>!w9xzLoEC_r
zE;#xCuDfVyxTg|>isd^f@XOG`-{~v$0gTJhRQkv5XDG4^_QrbUm7tWCw{P|Fwmu*s
zzVJh6?9Jya{ji`mP}OTOHG!YxuSZimy^*HPsQ!n{e@DMNAT!sNY!34__=`?1WsZ|4
zbi_F*c+kF}*mK#NR|CZ@8b(=zuN+1VWe(6HsOP^k{Oo!Qgzhw$<SQfWR;L92Gn~Qg
zX`6LvpF?i<U4)fiJ>wgx<IeS~25aglxK$Ukfv{!DRQ@s5T@%O-)8NN=LE>}f$-kQ7
zc{atpYZUeLtfN`MNb@81WLEiu!&3!guw>SM#a0@5NE%0yOa3#K->PD|sly<*t{`z)
zn4O-ZeF}ad0LknAXG&>w34_GLg=^;ASxXoE*VaDK)52P5r7|m+VP)?Z4Tk^fZ&v5d
zJ_NH%9|7wl$!A+~jp0%yGPxBM@4m-u>mTbc&7AQ3TNHmPXZ&b0)nsofBwClQi(EUV
za=%7tJ=N88HrlWO6tP3o*RbCtX4>9V*V|IpdwRpv9}sl~iI|Y;x0*EFU>~ZZ;ph*f
zQZC$Lk2qcuzmjflLVrL!z9QRUA$UbI2PI7-y>4E(f?nFEB(GHKVp*O+rtJ{BM!}|Z
zp0yh}LUY6F?t`7f*Nxx#&o2dPBGLuvfBTz(6Cngtu!~2$4VE~-s<GmB-13Cm7d>VZ
z+IFhb+bByleItWB{7(w0e1E-wwao{={>MfCJnINC^oSi4%fPEtPkiHjkS>~pzHOa0
z^;Nw(J&;Mw66Gdt6Uy#F*^~oUV06{|ZgSF<RO}jQeKh&qC*0#&)8iV+2>H4UQRFoF
zGQ0BDVpp3iRXE8Fh8r0yiSf51O#@bj-9Ax+e^bx^(>7h*l%FaQP)I}~0V3&<(T94H
z<3<~hR{qp?B;F6>*B_|!l4G@*Qx~<4{=80RHkP=c>w8xds*A`=DA)B3>{i4_md=gs
zUw=7g>eS)tneeyL@1){cP_((+JNJ@MAhR`Mn-^S)cN%drT%Gt1ODi|ZyCGP+z=&9R
zXuqDw4fx3T4CR|G%8OIY@%4mi(D+4Y((1NWX!>^-#NgLxkP&x)v8OiEY}*dAds79G
z3e=w}5&%gz#X@8}NTEhO_u36>3oR9vDZYr%L>fC(e<FeTfKAkCY7d6+iaB&Q<*&-l
z4bi7?ZmLMG#mfJYx<aiu(#K=&q&=qAoU^A!=Eee<FHQ)&2pCPM2+9VWe)$&m?59<n
z+BwcsHnq$`eL$Y-28y$KFKwwpUcgVQJ0L{BOOu}9wCDy`xG|*mh48yEC`eH)=e+h|
z-1K|H@IoTDls5MW?*~*GbpTle4~54Bf*NC)V0Dr7fU5S0sc;!vR~f4`d#IH=S$sk;
z7Oxf(#-27{TX2b})r|vSfZmWCTO0(+W;TY4Cu@Pk6f>HZ)i~LIE$BWQ()-l!T~e&P
zbrLWh=y#A-y1&rd4cRsIo6FGhH_e|h&EjSv9O<?3;%ZE=o4qT$rV8?Sm`$kjLQ_Su
zsKR6ULr0o!SyuVa>oc0dh^*!$cSW~G8ssEca{Q#`A;Hw}i*8D@>873shX~Hkur@O|
ziSL+H&ql--$mNG6akD0G0YN3oj~ST(s*71UO|7-ApUZt#8qh`xur0WHm;`;2Q`4uS
zP(Rns%tL$ar!1gLcr41W?fWv5aH*4p#O59?@$ZCxq1W+UAOA{O;})MYni#4$1FV~&
z;pI&LQFrW7)&g~z5*dI$5^VjuX8{DDV=?tUKjkM1P5})gJ(V3n{TU3TJ$dk%yspOp
zIec72q6!={?MTSYk8ZkJixqvn5Iwy?34PhGCUm1qnFtQGQ9+<(d;RZdhWIUzFqE1c
z_IKC#iPqyRiP{^s;Z%?Beo+`4AKeOT4W#PU3ctVd$fSv}WY&m$B)yO^d7u3el)S3R
z$0$Z&c~bY_Kd)g3FQKB0E!(A&hLUB|b19MH#iF4{uRpq74dt?tX;WmQ;D@Rhjz#W6
zEN+l1Mj%)gyozO^)|BSJWj$?5siVD*mY`^F*GOF+L~+^)3t=sFWN7@@-0aP>=0Gai
zE(GwBa}NrD8kNvWtZAp=zKWc&1pGYKBh^aHtO=CP{2*F*K@EnRty1oDUFqT=kahd%
zXFf-PkfH30-;W>K>Wx{_EwZ|MC_A5Pf226!F?ljSJAFkCT~aGJk$czw(y0T}aDuuw
zR_1k_c`{WsdW-$so}pFU)hfIJj=t3pISeLSa~9)6!k!0pXlNY~&Y+9A)o2#5Ktw7r
zCP(g&nf-}Wp{QZntNcjL>y)pKKeQ&AxSj;bf)gn;xM6**n5#}@w*KldGcGMBF0Rxz
z#C*=AK!!PLSrLx*F(PJ7Al(y5_gB;>SB3hPr#NAz(>jBZ#*q8epLL?O<R2}twp4A<
z7Hkd))GZ5Em;oj!!4m_SGm&L;-`DBWhEkB(ZEU0#yww+hc5t(i8L1e{jZW=#v$Ba*
zb`9(4k`2h5sWahQUjwtptFZxKhT)g))rPFo&M}^<*A9gllR@Mu89~49r^19Obi2@g
z4|}`fA@9#7B=l6(wLZnMEFP%eu`1F|DCJ)k6TTj(yL(C1G>dEF$(2n*c4Ecr$s&d<
zq7^&5o7YSmk1Y;**w4M_BS-)jD;6?QXni7obF4#S>4?9uJeCYB^>}?2mIq_g7_D<y
zmq!RZJIOXYRO8}UO3gWk{T&iR;97q?`Gz6EI-(YJ6W{O=bg}SbnYX46T4q;YtaZ0A
z$<v8au+`v#8Z*QD_YR$PHVmjP+e$YpM6<;EhzAmR@rdnW`9%k(KhWv^KMO|m>LlU&
zgRAaDe;qmE8_PDdwG9sMZD9qu1grpgSCM1@+3oDhpSi%8>Qef{+G#b+Lea38)E7Ar
z_17hR5!q&XWpiKgkNJ~D^{*zb;2n3+B5ko}*Q0cX*26pcKh+P@Ga0}U%_q%F3TGxD
zil-fSYH6k0FU?j{Yw|$f9KH(Qv@05QoF5REuhi`(hM%_7S^jW952%+rFBBO6ePDk`
zk=ku;(j{U?A=fB9esj^1#6LW*js5T1(rLkJpV)e^>1+^XN^k{cG%prg9C@iUpT~RC
z;0&-7D0_Fq<c>$I{HA!n72}3qe;~o64)eGr*NlT7n#?-b4UtQ?w!Y+mVuKgIxn@hx
zGbu}lO+D9f)Zj8Nvy&?-D*1;rP5sThPh#1YXMbQBcx&`=>9lh&CJCJf+<0(hxI&QJ
z6qtpPE1W`c;b~A<(<C%!vrTRoI1>2{U1Vt%##=H4)ZFkv>Ch=7-TiuKQ(B}v^L&0_
zsLHQ^{s$iwfj94QFei@@2l0YhZM+`KGCp7Q6ZGYlZDg^};d6Ef!^eNt^#80&3n1;!
zd62)<r^Djuryu4ll=-(Qa{Rt_bA_ySTAl<Lo4!p#VKX?IkNn{uEWQ1Q0@jde<DbQj
zcA1m2o)hM#M2q&F=PcLrKe3eNvS2g(1G2&MzVjZWTE<!{8wd)c@Z^ysXWdR4nJY@O
zcjmY6stF*!*$YYIf}xM?O{lN_e<jrq$l1_H^6@6L_Y|6X=?dy%fnlD@Iv5Dr!}RR$
z226~_$Z?ev|Gr#*4EOO+<GxXNo2}z>z}-qZ&Rc`qr<Bd4ekE}rkmcxAHzao92sMyO
z$5;1N6JxQVgFPHJ^_^)^ptvWQneU&Nh3Sj%yC_Dc&I@8$I>AedpWsb~(_0p04dnl{
z^Hj_qzuu?*23GYn{`+pa-#Y(3@pzBp&Dg0(x}b{xN^T#NqjWv>GZ2RZYf>d|6+p15
zER+97S%nnruY?W|x{)(9c5Yj~^8aKv9I*RA1|(2;exd!y26h8w!cUY6ILj59Eh@+Y
zc6(92BE1rKjxqZV(;|X24sqiH(LHw=LwHh|K3?4qS0Me+=`7wH3p1UZHaH*z&>IF6
zpXi`8n3yD?-$Z;&ZLxdG25v)(uoKHy+8rb7F*Tx@Nj0n6Waw~YzDH(|O4Db(YUoY(
zIr3{faGToLP8zD~6zeLsKZdj7)xyO_4@6pu!<Cj5-B{IBRVnO77JewA25JM*Ry2pH
zSI^j2Rq#SZ;&XEl%F(+#o#9l3B#aKc)|gmO6}TN~0orJ$i_tX3`YMa1S6$kP9YnWM
z{+)E7(*lm;w^Bdxhv|feTT&e|)Q(&XV^Vj|G-YNb`ngF;iWu%T+G>L{CgCY+dn7}w
zcmLMwY9u?9zI16%)F+!W`a&3|c>p{^W3weLJ$>w0wTM%ckNG<CL`RCHFG-MhUWUt&
z;_wnDAc9Kx4L{Z6Ki{1YNESmYsekI~@nU+j@E5;pI*+JvWFl79y@V%J`j7y`NlQx1
z0WvHn5i809D_-Bmx~NSu>5_|z<a#eArh2s*z8Co-y~vstxBz(F@qFNh7zNff_0lMM
zn2^!M&z2(REfV1-rb4xL2krI~b>6#SRA?7i07NldhY{so6Tp%L?v>cQFK#1D$B5uY
zBxuo-5-dJ@vzAME`qeo9eVON$7mX+A4pD(Ty@DzJ{9mY2PuH;bPH*GwA>^{@I=JD4
z_x#($Q?p?!S$4m<o@LVt8&NA(R<-V-BtaH~{$9W85~<B|h^K<HE!4m6cpyL33(-PV
zzsF7wZy1y5cmf&w7XB->kLXn@2<`*@@j`zcL`Z`$w6psIw4>Zn(P?9IPa0GuafKjj
zCRb4LLqhAFtk=HDoopcD`{<&_q8g!IVdxxjPez*UamfuD4a<lL{K4{Ckl{FdDCcq#
zWEXgBFui+W6LS3(RPQGqF}$E}jos|kVDNB4SX#rUhhmh@5->(7TlNkv4UBUi!&$(Z
zfV?V<YYrUg$leKuaki=(S)Ag~_z=&fYFvz~Q5IBzX~14T^vc<~%NOb~??w66jrk*m
z9j?b~6&zyjee>d-ert@w%gpI=X(jprbm#>JDkG(uT8n__=e>?;fX}{b36Zz~dULG0
zF~2R%GqdBkeD_UA0mpa`YKjN48!_GAs#q<NFDb|+RIr(=nbaL0bU_JN1x#?Y`p~H>
z(TQbc(hF6iuGl)QFz)zsF)LOY=eVh%b)-B__7&Bq@MFYIHD-zQG>gfTL@?Wq{Q2eg
zSFin;M!(~vv8GQUpj|O#mju{IcopsgmE-jUo$#3MQrYC@S99+5>rR%2(zE2=l|y}P
zRdJIc2X`s8_5{S939`L0TcI64(y2ScP5P2mHVn)G5=szdzh_zl62bm-t|i9$4l%`a
zDug);&A&2X?H^fH5s{DV(EwOgpE%o!K4v9Z$-X&E4$EEU4!G8RtysTHh2Zms?k@25
zZ;g;}XWc#}&WVe0WZNn%KS};{&iJ<v`l+R!3Ev|-{S<F8pO^QRV+TJh=kz`V+u;{q
zlX+b%;>VKxa%y)6gA)01F9xoYzYFmuGRBK5TyH5*Er+zAy;t11eg33c$eK9$r);-!
zgCR&pweCt_!x$8!4ztIT9jHnuA*p`?8d`s~?U2!-!aJ0v?eH;*nVgVKjbaw1om`ek
zDShNIc=+4X81}76-!!@c1+a|FMvJ5noC-E3)ZpG44M<kcHzeP|YSdlpCY7tgn>=Jo
z+HAw}l>Hvn-TODjAMRpM-tscFu;_b=@Qy9H$9^ZZUxP|itUlEirQ#7sSR=bl2dc8+
znNjkdI^Znsbi59A{T(O~c?41))rP_$3{ibH;e<IkzcrZcju(-GfPCZMQc>SEd1U6_
zG#KtcAAAtt#LiJh#I9kXaMkvllLnbz{*pDfn0HmHsMft?lC{GC$CaPB!Cl6^RW&Q4
z3R%8$oA}m`1LAY$5e&SVBR-hDBtHuqf8Jy<&deOpFld-9Y+V2nbs>-FJ{vVSvy4$&
zM8gc?zn>Q`2|+?10D#a>B*@Mi|7O(&dbHp`q#AOI`qSZD(7G(9VimAgJvf?6AIjuW
zs`Bal#G7oeP{4U`7OVqqZoDp(iemZ7Q}`jGSHgJdQ9?ErH;bHo&)Phcf3voA)!MgC
z)Vo8~ICL?$w(lc!GS*5h{;e_LB_)o0X8kQYHO57cnTdtzxl{)Oiq68s@?QZ(r-J=p
z6(-?biy5C`x@p0bdA!}9smAS{V=ULc(4jZJiDw@-@I0!b$`x`=L5;zPx1oWu*At=w
zG&bw1IJ>>G>|~7c4-(WL{K7)Mk}R9z-_E(a7-=P$SuTY^m1sT>BLKasteJ*3fs=Hh
zd2TEY1(+w|8pwAUkm+!UxpqrIO)r)^8JE8>4FXj%**=^G0W2=mwF=YLN})tG3%sgY
zV!TUnphG@$GTbt{D?&trwr@gTDhw3@(eAc8Yf<;O20^8ypp*k!oSZby$qvdt@%Q<~
zZaa(cTyQNzJLF4B#dwb+-bcZ{)e_(w!+2F~I4|245GN_Q$Wdve!Jzz=H0(TO8d)O|
z9r~bK;K5N`bBMyO3;MD^9q`X-k1v!ffu*t+LDDept2<DVzgdf+ZSNtBUPh?GusTej
z@J}<gkkECw0K;_W4@e1?KW(dV3{7kA4Up7hw?mpyE5fu?+K7&ZTZEcP_%x%0!WHLr
z+DmJz%s4<8?8Q%EgHVWOved%+H!>-%v?V`2bAw@Npf+E3l^EV3r@hKBKS8{2YAhmT
zJWrlMx06f?r(1R`FHB=i%^+%AU-!Dptc|UF)szmzFyL#WC*D^3rbf!j;K+1mPQ@wx
z1KNTab%}ys&i;#`1i=;rz}M|)nvN-ReYPraB;J5!p*Izal!z($a*DeJM`}m<@ib9c
zV#{dQJGR}sOumA7>2HTOu{2T-%JL^r1#`q<CXwQ3O*zaR<-X1KYUYja1s0BoH_H96
zWH9fTVK7*Ce3xJ7D1((jQ=6~jbgmo*3V6FMe_k3ifC<sOi55*1E<WG^t;^lcbz+oV
zuh|<@)RFQNZqU5t1o@e%WB+NGHQP7Zv+^-3pBt|M(f@m2hk^nAJs!>B#wAb+&R%ra
zCQ8rYic06(0js-1AwBHsXbY7En3qDAfpU-ZXc_fB(Iu?&W+kJ$oe+o2r;Yefug?}-
z_>`0!T72hE>^|$bAhd40^kgSmo`%ljEG(k0;g`hle#eZu*{t{aa}uOn4Pw+Gt}Fn-
z$f7yuJl9-Sk<$0VD7u+HqTsd^{$gF|Ro9#>rpKJ(BE8xh4v5fMMAguCjH&a?yR#)0
zGHt_T1$QhWooO-xT_KsoroE)cwZXJ)P1{`(FBv@vPi0tn7G)B4{%bv|p%VSYON#--
zS#2M1+e$<aTVu^kL}S)!hA!IG*n03GHA^y1FK))%O7dw<#6`>l2C<9-0`WP)#D!nf
z<2~>S?jN?3ta$5a%T4{mAf=b<E%vmW-~Q!5IPwc09+3LI@M>8#`Mi-6m`1M29ta{v
zZ?*U2@APPyguPkw9;fe(zN^d9(gkS$Y%m_r<Age;XZ+?YNLGS|CNCRofx|<rD;G9I
zG4;J;716=J6P(oyI~uG+Kq%p{Jp|Rceu}Xuw7~9P^zC=dplaa(a-meCn~lqOP4Cw@
z&AU&>%FVmQ?{IL@U2Xn?3pxKC0OSAE@f*>XIH;yBA)^y9RktjOVt4oVP**C29pZ_p
zDXrIh_H_^Mcunz)=@I%Ner9NcntOwDQ_GhX+~_im6)>%vGk^hRi+j_MA<IhEo?HL6
zrENYpZd6PUwLo}|f{4^l-LcQ9W;$Pgq5CWSt}XStX5RKq!%nDMWoN?e$7IOSHRSz{
zL~QTM`@1j!y?aY%+^_2igg^48D5Wf?F_In8pZ_g5L}V@l>_vhT8ESOY7NR^2eB{0x
z)42sdNgDhie`YAa^+#A@HGh<3&P(O#(yGfnR6WF+7*@V0@eLH(Q_A{WPEF|TbGoWJ
zMCc)!GC$(_)TqoXqOxzw;dYCqIoFn{bV<0hOi><Elf?CS%8KgMfASMh;BO$y>o$Ei
z`8f;{iKHRx3A_N!27u8h<TvcjX*_$bD2wB<T!WDjwIaGD`|gOE2N+a(|0#pnVnZst
z3Pl#>1VPWqT$=A?zfaPFI<6a(7PZWNlVvqJL!TH?Rg1Z0C?$fmrD|fn`fd|^xT@ni
zYMa{lKLKeMmgwr~cRvhK4H=CmgiHstQRjTB_&Q@D8BxxMq%@#3PGytHEoEU@x>abL
zl3s)Q0qH=UfKvidj%kve<nLyWVI80PsX&=2_dEz<cvg*&nn~RSZK3DEfBgoE8gUt$
zES}^CJ_uIG=$1Bm5O~iyBP68$KV7dutD84OnV35#ePkSByN$!OQMRqBYHdc@mqUae
zFfeCU9SxgHlzqVf4&Gh~d|7oySrt5rwRNtdMao=+jzFf~w=}zzQ5N7!!Q+dR<>NmV
zfz4Joqs-AXb73tDT7L-YG_9Srt86j!n6f;tP=vtcx+$w3XOEy3l0yWGMI!C3jIxcx
zG|*q^Jw~=e3R|J<=%bkGn@|Kng4Olz7*{CkV84X470RkA*mx=MhpIVcsODd2^a;u~
zHy;rH50vHO-#UZhF#94GA3^8R`R`?@w6|A+>Vv9!S-UGSx3L|qs^)pETgPZ!luaKd
zPf&IoWt81OU)@)yY((|b6<0x->C_O~`ylIQR%NY}h5j-;=Mh4Iy-XQ%y5jP)l*JuT
zvK~`5wS6G$-&2;4e<YV47iKKA)pYrc5p6^P>MfprbXBm{k+Gth=e4%AKSiDTdh%Ma
zW1)-|c8mc1*Ut#7m*wrXDXVbK!xdjaS*g`m7!FYotER@+`+3>XSL9(IRoYu6q~bf|
z+wox2rYHa>I?s?!t^J#%?=1(04e{^ZUs3H^XR?hHs2oKTVCPWZIoT9IB>@7v!o5r;
z`V^i!R=4s;b%I<RqtV4-DVJTM>;QLAK7JM7WbW=czNuu@z}c3cmofX&buZc|ZzpA1
z?O-_rd0waemg>4Ke@%J)S`Cv3p08uNBTql<4?4y1>;t+`-)@RTR!SeL<DdhS{TfzK
zhSWVR^5Ig|tZ!(|?|yt$p=j)5+|r@qmp(XAr==NCXWJNSvVpQ`TN~+e;8`Trz4qRU
zD%{!@6)ErmV+UHFSk7kyT#?qyPV#XmWKa@m?FOdrF}8Z(G(Te;t;(vhgb;j^kH3Rw
z$Z+sKEWG=XAQUaW(UP?T(`K$*HP5SurLBvy@q5dXZ<6?t>=4~NbDZO;ychHx?XxNp
z$shyP6)q?%-eY{js>NS%7FJMJP*#$Zeg95g(F*(iefV2bR#4Ur9(HQ)$SYc5?<V?h
zOIbnLG^0#a|N4`>niclvU&ilCS)rOy=D9oWU+-RN(^%N|cYm{E_b3p;Kp2K$wwkR>
z;Om4mgJdCCYq1vyUfRE$OFJv=L>T8ez}LMn%;fl|xiYJ4?6Nr2ef60hVb+mZ6jx@I
zefP>1Dd0qZdWK1msQY5H%Bm$w0M`Tg3yr@+^X^g4J0KLvI)99|dOR7kl)&7lfzGvb
zhxXm0!~lXT5<AjcnN>D#?A@Ub!ypjGaZ!?HQy}J=VoqF>sfy*ARFX8Q?F&P=?YVYj
znmG*cdw|dVgFu{9ff&Kqhjg}0S=Us;)$*vY)<|Q3te^E>R_d$iw$d1XrNKv-jNO#5
zw>@e=csxm~t?J5pW$$5@%rr5g_Z~71DB);(6cF)b(m~yC6<5|*(_{9Uqf4C;an{Nu
ztZk1Hk=D|fi*wbL^~zFa!<c@~RQO{FB`j@^1|`1i+o1}?Fbu}=V}T1a>IP>sn4T`w
z_q`)V-fIojovUf`dw{<QpG-|RBm7rhSyXLi3K{uxr<Cf*UT4^SN2`4zF=hZgR$f_D
z`^*#|XGG+Yjc3?=M>&I06<K^`Q6&=52HspS!{s{)YD9>>ztvY3RkERpz9~9G@f}q%
z?EvTLU}aIIPKJuiP<%(F?W=oOHKU!>oi&Q@56n_1FiL@G${xW4D8RoUpeleOjVBOe
zyy3zdU%)rnZ^fUZ?Q<R4QgIBpIi%~tF=S@tW6k%Q7RrB}6=fQ^D&wjx#Pl?zZm@k<
zYG9+TXo4;o`?^}LL3o{OtERXp0mvEb{$@QM92xxR9V?|%S}FoLCd`s7(F<0{u6Um=
zXCgwLUc#fSLPS~kDr$T;*d|0JWrJewADf8sN(ho%6)#t3`H#L#=OY|buLZPZML7^u
z39VAasZ`y+W5$=Jg|RSHMY$kP!OS8>6y?|w#=!oo>k=<tYs!YzYROcQH)?6K9aiUF
z@#|`}dbx!%J5vIR#-6TI%QKL5SKqr<l~1zcq>UPD#x6$q9LI^Lh%tq)196@0intt8
zjdF!He+X8{JdBgISO*bpS#g?0!lMjl^e+PG9J}e3O50rz__d7<GBN-{$~DBsD&@2x
zz$&4gvlh{BV8Q#b+a4bHG&|E~xacl-&Kkp4l~Dw!r+RNp$KT}gB~vzh0Mx|X)+#_p
zdk?D{cm_aD#dfL2o~{!epPGk2nW+K=Dxlv0ob~ufy)Rv?8w%!zF2~fq08Z&VSRuKl
z{R__G4}kGCLZs}R9{zvIepBh^T(?pdR<2<-R-+s$Whl9nz3?aiG({BX_mOshCvgK`
z{SifAYiB^?Vh_py9E&VU6ayY98zN<X`J^XVx!Pc68i;00mK?e-PS%ta$}H89Chp=o
zL4z*3tQ5_2F?vFZh&})eiH4@I-hpggh|2?aR$lO2%@jw`Ev~~ycyB-Hnrp*gJUF`o
zDgwUR_W;IiIVRf%?DY!CbzmS+$2bg~vNKa_-MW+wM1V;Ajv1;V2ic;dIx}m#Q5NhE
z{7@f=$i${l4eCBK^PddoIsojJYnY9dF#t9gvBU{5zvis|@uf)Rh4V&SDm+JJz@W^J
zl$9HB<#g7Eb}I*8H4&o-4rv0k%llE*y#(Ua0TxsXnKB&H#c@ms{h)2w_{}Dcu9dL(
zLfNVDT2Qr;H1Th)o`I65>$I&Jx3T9j-mUS2;>AJMae==8V=LQh9YJYpf8uQ|QH_b*
z51spB*zj6L<UU7N<xBXoEC7A$drYu%{2^E&xi(U5Gu4EvL=It7*3&ZCJiCETN3kNM
z;i{jAO+-@;(tZ}{#Z<(1qb%4TKUC$xD86G;sCLRI+_XgFph+hHcFQ%)#>yuEW^|rX
zWgsK|^vW$|U8K&>e<xq%!e7fkl>bbbaoIP@t6ONl-1DFnqo#gC5uiVMIN`ayD6>Tk
zKsj+7Z-+>k#u&Q=eo+UwG=Ty+ICKDLXSn)8S&rL)nUN;`?0OF0;5xawcgX6WhmYS)
zNB2PK0Z>h#+X5VF0{zSF-4J6#thzxDMz1(lk}3Nh(@3NBO3BsI^b?(Gp82oPDYM3a
zVh*4SJ2(nXC<{#MQjq?;J{6VSC=2!{2-V1J3ANP)a`YI$vs13&QKsCwZY&2hnF6i?
zRZ6dm<T@}DQsMpu$oQEu&O3#MSO9>P^6;t;;7e`IJ<1j)(H-`ptj3}MN(s_lDH0lG
zUB___SCq9Azz5v|m6TxAlqhQxV~o1e#GhSH1IkhWV!Ty!@RhQvf7Cr*a;6zRWeYk7
zUPs$pu`t`fhHKdqdjr53YX+2k#?)u*2*6{u$VwZpWmX@+_!7WigwQEV4gg$Q&^i3K
z2n^jIwU!j5PZ?U*yHOVGPY|k+*%+!%*^sUPvj||fT*EuQHLYv=pUXOQ>fcy!{j<np
z(iqbyE)~8LWjYY`>dXj_jbCili60^cm$G?z{LjQt_oA#jT4l1hB`4fFMv1aCYBaOK
zzEPHPmO7Ru{?&C&8BWjcSY`ud<?;tqiL&xGSGzp2f$yKofJZ(6fMt2{;B(n$OdpPO
z15c*9@pD=J0;|n!y@W=Y@)30@i_4S^l0IeGWu;X)NT0GqemBa3{Ru)fGMhq`D0>00
zTdv_9-z6IeDEqa6vWm0RTq-=D_&a4O74U*R*UsUpquKCu?Oh49c8wOmOpESA*+m;G
zz3_Rcltr5Io)=BvK$9uc#XA7(OW9iu$Z((`QwBww0o~mwo0<nG#8%&hWk1KmR$oE-
zQTmQij;WpZ_AXc<xmNfgFlqoc8-_;NTzdm50<fOH+%eYg0ifXL_MI{dxDuINC>y>9
z`xAs}WVVDVQ?^^KAs^oXWx*ZenkesVlnaMy0(1T|WsQ5_P*mUu9ro}LP?PX=yrecl
zvzrdZfj@M(4`qJuEQU^#We^f&QSZrCzK$E3VSdz#;)FVwX>Xl`mcfFWK+*7eHO{SU
z+PaY^Mk;v9;rvW?#XBE(YdnK_J{bk3iC%V`aQFUp;UC8Z%%@8mvmPl5<JY{MisD}y
z!oEvH16$cu*UqceyI1eI&bW+VR4n79S~Z)&^RjNXD<0&SDr|N#U{0_?PQb8YIz5Hb
zU~AWU>gm=z0(!Li_Css*GW523Ko{wCCvuSJvst#mZV(}7o$AKjDv%TGPY`PB{%O7`
z)M%<bM}-zRG`foHx;@vBDLZ7kOZqV~;y<xbOIa7GinDC)1ybP=%Jq&|{%UgvAWJpi
zL{qazJbqx-tJNNT0C#-=ZfXGR8dVM#Zm%ZC*n={gh5+ECn4u!%-+t5`BFgoNj2;oq
z?1nIF1}hPf-X&uGg0jY^5Awtp6GaoR7vEfos0#p|L4!yDv1cysOF${fCU9dX`p30F
z+GLF!W$uW0fru*iL}p+sXPhKguYhi_@gsuqm^?MV{PrXLJ*JGwTD4$>><VYZl4ssd
zFT$hj_>S}GJAj$#m7^zKgt%7VYH=yp{fQhT`p=dyhUgG!Pr=*Ywp-3>e?CK<aDKBX
zRD;OUiKxVgrjqOSTtjLsPFfKF4v0vt5YdT!&Ds6qmm-%hXsr&U!l81WYf)!&2Q-N2
z8R#TgB8xZj^Z}If05CB9t0GYoy^-LJ#sAo~LnHtoD2k#ZX?CH=2=>1)Q5KBwj|JQj
z-f90vS%f=A)ir+nHBgi?)!f^p>}8ZP)sj-Bj51YK%B(?^GRjmbqfAwmGHXz!j51YK
z%FeR}RmwK|bVvXI0Kz~JOaD&f!ma>{TYy))R5o?7R#p8SEy{ALlu@RtN|`sPQbw67
zWt6F^Qsxb+lu@Qi8D*-flx=Hgx0Af`h4E*R9$3;cNL!`>Ctw1UQU?lP_<a2eMzdo)
z@xT*rz&r7>BoMFbzuVHl{5QJYq$_XIlks_<K5PAkX(iF6sG2^-HO|KOGEt=f2Wv&=
zsFHWAY9|uNFk`)Z3an0q2#QE^XeL)ArM(H!*8qH&Oy;mH+hey4MCH?5QQhSVZOFxC
zJHvS>6VVxxn!6mos&^vi`&@~rpXuyoJz44qxU!6mpmJtTm#<PZ9iTB2-cSQXWVi(@
zx=2Lz#}KjwJW*6%M^M(S)g1ZiyR}*^WPV1zSA6)l-s~I7mX-s|y(3kFPAs6-<@3>A
zXGA5S?+rqEf;5L_HS#|~QbR8scw-=R$J6r$k{Ym9?z$P@kEu|$pDQeJ?#oVwl%Yj6
zq-AmZuEHNELt6|r94OB^(rea}rH+7OyIp`~N7*YsKO0F6hVkctzoJO?{e7W@YVZEu
zl1V^s(+c_caPRN$-6JULZh%)K-+low8fN7i+yY?N@qN|3BUR6Bq=kG2l)<4AG`9>a
z+)0{4r~|%8ff8k)B7o)%cGbB8beaONJ*iT*%N526P-e>vDZ8i5f2XWO&d<4KN?G=k
zrCPmeW=ff?15=td$Oo!D)Tjn*R=5g)WjnqVrUAVw${v7r{;21&hDc5z?_M!m$^eJR
zHd*sDU@`=>x@58ohv<Aq+0<a>LRt75)pG=NU1g5e=c4Ju+<uUEiMmqECcwpw5O-}{
z(sf_(%7YYv_7i0e{Q*wtVn2sMit{Ix0H4|`H|=cN8@OHIQ!Et9lw5oL$ki%q%39Jb
zgb`&!!Rv@?bIH8=S`D&C!84N&zoRVe50X$fL?m%2RFA47CVh)@`>@ySCrh<zc>o(X
zSx*P@L(ZPR_DrtFoCh1lc!dnWBkQ~U;1qBuw_dG*v4l^nHI%JkxdPC9#A^@hW8Z#M
zXUVl!LDrPvldc;%A?&|SJPjpwu3N5y_8w(LFPPR{M_Kr9s<FVjPs31q<>8;v2$R`p
z6W{-UW}j0&21c`sQX#<l(S6yRxJE4}DnC)Sp+zJ+*vsJ~H4^P}z!Oa!8XDg06G_D*
zCTt++K%q>@)#CbJ#ta!#R&whXeb#_QL2jP7MyHoM7cDi&<w%6>6S4UnWodtsP}dti
zpWslan#re)p15q6Nk6EX{bZ?D`#pep@4ndtEb&rm_SQR-HvcG@foeQ30F7Heq&oJN
z0ML+gytsyImH@N2eH!0}6QVCAyKNjrnI@7h{b7>~DRUWPAuyo@aOVM2(oxg_xGk*q
zD64P{ke0IWXVniR0z(NPXaX1Z?!*{RQ*aA78{ky})dHYX8R*P@q6|&rbMr=ny&Rs0
zfJ)hvj8ds!AStn>tR%iv%9LE!9U{5OnzGplSj#aiu>_2}cmZ7iCpT)4xS0V(QT`oe
zX@AmCJ?ooL3ss;(r+}y&RLy#_RIBSSik?+~GFyY!ff|K7liUVcLN)$q0PudGEXTS4
zLPot6Y*Bu=<`eKvXtVkjLYMl$%v`*9qoXJruvL5=HjyD^RbAJMTgo1Lz!!x;*%XY*
z3T2PP7^AAX@MqPOG7~_K7v?(lDXS%0-RX=o*GnjC(|yF_kCZt*as%>rIZRjxz+y}y
zr7Rxg66ra{n3H~%<qI}c%9LD(a^W{e>x?NghX7_BSRBbM0^KM`x1a=>P=?;{ca)|5
zNkjFlgP|srb!nyEx2md6R;Jsx>sGboxoknj<ov<y)FeKWzxlFJjQ<;DZSO7FW(MX7
zHV*JfUFZ|w(cCW3TcU-p2GFJUF_cxe85G9I{KTmCd_XM??Efe;^}p*=UHG%=K4mDj
z?wECeGQ;>n{Xb9!wA&%+yBrp-7N@H_NT16#<VRxRO4oHQeJ*>5w+UCOs%fr8w;SJ5
zHo309RtG2>%7n5fUDs=eC^P!Mqb%)D8mea<3RR(Od{8y($x=63M*~vICI=|1L=%-)
zjQ0%y-=7$@W>Z5_WjDV&sQ|V>TEX@9U~_g1W!I1KWy!i(Q|7nSp3f!Fc2&x3F#&*M
zDeDe^I{znSYqted{}W}vnCEvnL}sDBV}w@+LaOb*V?3KgJ@xvIs+#6Hq4)Uz&nDM=
z#B5n$@5HD9&$n^+liUtawi4EFDC<tr{-mLL)}c^;L0R^brCQyi47?7U(qKoFh&(CA
zlV;|>r|d+wG0i#NX_W4IJ_9!Nvf??QY63v32jJ=%BO7=>>a#5Mt<w{LOn+b2e@25^
z$Oi=tI=!Rib{B?gp-nAd&hr4U;hm>&)=(GvmFV9`OI5{$kK><-1@e8l0)k%YY-B{e
z596DEplKo_Z~VF7ll+^xLXEmu-u1LpR#n9X4S;r6FSGhy4tF<53qvmqy>UcZx752v
zdLAn0u8l;<3wF`#O0G>ae7fPAjB<T8{ZIx#iLA5u?T33+vM3qik@QQ|b2UiZu-4fX
zYrrRcm+HZ8O-WDtlZINj_49{9U72rrl&OuPS6zRjs!mq+logpBk={UEy!>ev);p80
zqIG7Ep&0k6QVLjqb$9}7_r6xU3pDTMDN1+Ymd|Fqh+gyQMOe`fOx>a0Wo1;ofp0$J
z2+Fi=lLj%P*?#xqStp`OhsY}tQA@iLa^NwCi0mp6LsU7#<tH0;;aFlW5j3Hy5>XWZ
zVu33n0pvzp+&X|G**T)viTp?gp=3dYtV<$3C8F{zkxXUPj4z331l?r(bw3BAkF?@1
zhCFHj>Rim)d*A&Sogykm6zsezxi&7jXy&pSvd>G-g9+j{`DO@M##o5q8d0@pg5BzB
zkhmGmgxAG8k?SkBf1q3Xwm)g8IOn`^C{#`~vx&%IL??$-vz#o|DkpcI?6XAka%6fr
z<m~oz&*XB*-NHK*<Ct@97ie*K0whHhKpzSIIEfl7j3>akjlT*ql)$3Eqy-e;lK%gm
avi|{S`j5n<bh}gl0000<MNUMnLSTYNi9Edk

literal 32298
zcmZ^KWmH^E5G4r-uECuE!F6!A;1V1L3pThD+#&ejFt`(32M_Krg9bt%xVs0p<y$#t
zf9(Ex-TiKLS9Ra2?o+Q{l$xp>CK@Rk0s;c2g8XL<1O!A70>WEb6vWq-#ue$}*UF8W
zvX;!t%gg2E<@NRT<KyG;@$vor{psoH5joxL?CkmZIX5?VP*BkF^77%~;n>)iiHXVX
z?(WFQh_<$NX=&-y)YSI&_Uh`YmzP&#V`FY^Zd6oMMn;CNuCAq}rM<oV=H_NK7C~`w
zv5k!l5D08;Zf<L9tFEr@=;$yvH!mnCaB^}=N=nMk&USZq@9*zlUtf21b*-zbo0ypR
z`}eP`tn9+VJQxiA{{6d%h{)jJU{_a{y1KfLkB@+WKu=FkMMVW4A76f6etdj93<k@}
z%2HKTH8L`imX<a!Fc23Lla!PUkBCrIRQ&VjPg+`9YHI4M`7bY5A)%puetvvBJlC)G
zKkqCpEy?fz<mKgi&K~^z{Zp`m0|Nt5QBYqldv~8+iqzOySXdtR>W?WnPEWSExh<})
zmZ+$xEV%*PU+YfCg7tX0=jP^kxYb{7riH%^1q1|e3psg;a}VubZl0V%AP^lLopbh2
z3Q;rLHTttRFBU%!xw)m9Hm-NCua*WEs;a66T?N&0RzBe3z1;q-HRGE*{dcq5b9HsJ
zzgTmopn6QBd<I#n%*<NKRRTx_9Q8ZiY`r{A`Bnej;Z~2k)nWG<Je!S>`$@|bHg$Hq
zId$hMJveie-?x40+Fd+*e$bF(P_=n}u#j2Meym+KtsZjU0V?s(Xc^x;<^YAuaTjgA
z-0beATTI^^jHlovil{qFa?6>?^Cr~IUBC*xbtD$IPTZ@eo>mGFS)&F;gH^&iwx-*=
z;()&gs#4j@PHE}r-w<VGRreI8-~0yGe+-#*{$?O-fP%rhPV#XKpPWIxbPAf6EH$X*
zQF1IA+?e7y9m#xAz1P|^{ZwMWD{GXZ#yj5@`H-r|E2?-o^&Gc&-?P*EulKyUaeJ?~
zc+(qV>KFM?<{9avnO4(lk=TVsYWgF7-Z!asw$jZ)+;TO$R>>@EV&|Z<K19XTH8Iph
zmk9CS`PG@8kG+X{900M<pt-ErnuiD}k36V;toA0*qwI?pG6I6niNa?|Ew8^v*(%m}
zlEmRz=cNLFHzd`+<ccu8LyJZZt3Xyi&({KWmX%>Pb3y2$WoIN}6m$nF^zdKmk!dNV
zl*<Wz`Ai%9i2fDrm7wuHRX+wLglKi4+r|^mT%$$jzLUz{!^*0g1SZne&~OMz9naj)
zk)ugmO7z@em1uUxqD_={w3+Awf(YIMu;0HAQ{pNu9AhDVZLS$L;EU(F0HC4Wgp9Lf
z$r;Piv@v35@nd${KhIUb#Q)4i+g$G!+`_roI_(3r2f<LoiZP(Rx^}O8u?DY7=h2yc
zon|^9YGHl<>ZTz70UCwQNg%TTW|pN%tx&iI$e$3}YnH9;YZU6i)KraS&-1(%0s@W3
z=BstRoIQgNYPmc3dtDVQ85Mr0{U_34T%ttT)ID;x_XjJ!+N;VUiY@*JWWpX}(VU;;
zQWOhSHrqi_^a8or0P}CEa41`%^(ekK$)MIH^rT??aePfw&{+EM5HgK{{0UE|^VixN
zx(75t_3x{ucb~ZNZAY2NK&8q445M&)N5<|nuBOs(bnU>Yg1(?QkM&Z~(X1HxY)w8`
zRd`PHnI>yDnpK@9WD@Q%Bv>nKQ*@LyZ*RT@!gurrzDw9oCvkL@w3+f!TE^y+6%b!n
zuZ?6wOG5kN$vFq}?nnQDCH@%aM70|%i?q&p25FMzXh3l_8ZtFfj&)O4;Jj!^?!HUl
zOptWZg_dNLx01zNf6`+fD9~(uuq7LP*FSdCH~Aph*^Kw(lf1y)NWKxAmor^iC0tG|
zole3ws;a0R=MHtFMf2(QBnNrIo!)dGA{!^AC1z)~rrFF$YZ9*{{QcfrS>AN7<r}|t
zsQ(4CxjcAp?H+^iCtsn~P3^w?CqV>3yl4*_HRBk$_PDn{k+9ie^lGS;RLkNi&4Z@l
z8_1zWByjR}-`l_ct{0Dzlf?M6X0p_%7`#1PTAzR@nt$TCnxe2dPG|2Zjkzjjg#cyC
zh<;ap;@7npIa|%jO4v*r>4ZmJy%+Xxv$}8xWfz_=E-nm&Ytz#5I^~UJ*Q3*n%216D
z-k2?L7uYKQ{cbnQmszT6LW$J%&!LjD{VBr7Tu)4dBghXeu5J~=S((>2J#>kBrMTAI
zHN<EI4zAqL*Wx|gphJ<Pyh2N#vIJX-x~MC+>e4dPIb%^ktq=nHd^B^$@4L}0m7vKG
z=3xoQ$(B2<|E3lAZQaG@Tiy#mlCbd{jLeLA3>m4AEX+ybh9`$jA;T&EI_Zxurj>gT
z1|@AtXMSTZ9_k~C;nguBIv*^geb^ESH2#<G@XANrbhQxU>K(5UBeHP$L(^K5zvNi%
zo0MQyX^BJVbkb3AM`$HEG6{XR@YX@Z07k;JKS`LdF%s;ZX=(`1Sz!Yngi#(gR{m)O
zz=($)+9lwJ{)=7Y<wyB3xVvpTff}wr0x)M`l3x6&I?ot$IC$3Tvy|uGF?>Wq2Vzrr
z&&viI`u0m>+iT=NRY>O|9)-bIEM|^a;lzt%sJ#b_Ix+yB_V~&d2VHvrO;x7LkOWME
zGqRhI0>V|v!8cvTF`olvfgTxCn8Kjxczy4;V%HY(k}ntPke^gfbp;g)8uY?UonvoE
zUcJl0C!_v8QFPwCIv@xk=tryxzC3Kpe!Lh%VD>p-vQ5qItXG>56{A-HP1Dlh<1qLf
z+WQL@fxs5mZ<qq~Pvw?fP$*Y4NA3PvPhIo?Nb+?R<KWi0;D@LBVSpSG9x@Sd^!8Ou
z<YQ80{W}XpKQi@G>%LPLYC&iwHGfR(M!L4t@wpd<>0?D{<GVAFeIW_LO{>6u<0Gwn
zIs2HxOFQnbR?<QzZ1dH8>C)S?(@av<4ve3?2bw7Tb>B`^3HB8Ca}nz}c#zHQis~h&
zFyucFU+<-Py3^$<qW7u;3<~C-E$@a<cYcsS!|)*YBS-s2_m2(lU`RU1mvnvKd|1ei
zOuIVB^pp6RSiu{y4hx<ch<w55ry@m8`^kJU%Ccu?L`K>tniu+9Uj3vTixOoR*6;95
zLW_SL;H}JT<M19bo#^NA)QP-v%1!p%uJOjwj_I!901>2q_JgaiCK+`9acPH;apHYy
z;-r0m<)<q=_G(>cXSr1X%APA0*-WmYZmF;cb@JuPckO-#Ax^*KdQWLI57G=a#Du=w
zGj_VY(`b_>X(uL2k6-(s*1iz%%2WQW)#+A|VeUX#tVjJSTI;+19fDkOIsd{7eD#Qx
zQHHjvjg?V;Sak@u8yWf*9t)PGkatc%wt<d}f&WN=i#kbmq2~otNlKbGt1_K^Gglr9
z5N<82O#UN{;8{EkN?_nmb~%BrH-X{3NW($fRvqP=@tM85eU{~k0=#_<*!|S3Lw<E>
z$ZGS=GGX_*ByGHmhNCDQpTq5h#<gGYcW2f2+QS6-<TX&-atBFiSK4@OrqNQ`#$G5G
zUvn6a^Aeg?e-{yRcd&1IPfi=vQ6<kIjnCi;K)Zm_bX4KnmMr{XRsbiaC8nuAI!PK|
z$8`F%yl$+0jP{?JLD9u-b0!2FNv75$5pmVd>qyAoKRa$29{TBS;9AEAG-L2KgV{D4
zDGY`n>(=%QPWB`%m-B#T3^{lz$r?u0K0)d_YZ=CTK7ctCpoL_wMv^K?5(P8Fa6|kk
z#2kvGLk)G$V8>L3ACzIBp=c4N_40BV(xytj35JDGso)i1YazXV$DEa8GcX>gtrJiO
z)qqOG!u+}(^;h7?Uqx?yoL+AAGHSaamAN{TqJO5<GpIAz+C_g|I!-4#GW*th27FYJ
zaUt*Sm9)jj|BQcA(BiZIgQhH^P~11l*rT^yYJkr|5=7thkWl^Uua;lB3?}VDx{6Mb
zzU>ag&5?%zBXgaYz*L~WJc@+))79*0_VRI!Z_8|*c;!RqPq)doU2?y(&6}HtWtN-u
zv(kRFunpdhO%V>zmo*S}IC;uA1_Xj6mpB28<_-HRRAtfOj&mby=)CiY#-0BK|6e}%
zaIaC~-RfY<Io@AqZl}Sl!0nq{DeJ1)@h_h^gXv@CD`vhpp&INYA9;qk$upX-HPO`0
za6A&qIx*K6KV~t}p2{h73-WP*<ixbL@>$gnYRXCa>E78%<g&t+yiC|!Z$r*AGs=0V
z&M|wD6<HqrbbJqgzl)F%0cm=n+H>i1a}#f=kzs({TZ5n<C)N>RktiVlhwI}QH$kT4
z&ZE4fbU;a)t66E_5{+bqw<Jv2GP5j;HzQUn#PN@s5)SZeCZ+f58%d)14&2T2WI;wU
zL}Cev1suWALVGk+#Lq%h8Oa``MuC6QmLX)t*nKInXe>iQmcfQtf=J6kyN~(5ss6Ry
zm8Sy`Z7OMpX`FUxknpqozJ2;N{4p2MaYv**0N-!y#Pk~L!R|>NX3z!^b{<)VxkfVi
zN<Gg<2rEHUNq$h)_4;pE^TV@)+fajcTHC(sH&XV8eBB4#F(J}iXwn<vFyfgB$~i0)
z=VU7|*<HOCL||=N5!}iqb;E-tL;`C0l4iv=%_^8re$Onv1^w2&$6D?*3nH1SKK^}Z
zhI<<KERSxR*)V|UHYq??$w^lTZc{MPaQa;qvF`t53^6OwbhvC=H&3Gv3T^O=`6_J}
z@@K~VZ`on$O#@>?Kd>k!Vy8vpI$wJP9u1KtnNlPDj?bKv);8e1U_aDDXG~a=?&W=E
zLB-$&6l$XN&OmPNF%%4?EEUEj|K`67VpikO6}7R?ogMZeGfO0szag1lXQHLN#z7+8
zt-$d^G;|NT+SN3)*NES5q+J+C+r-h-`8pV3@TdK!?u~AoT4{TSaA~`2ndRUm_l&Or
zqkHGzRmBFM+hz9n*FeP?;-8F#;J3}zw816h8**gMRwavMW%NS$oXM!lCS=Ps5ktxm
zRZ}<8hi0adJynhMbzijFQ11_dK9sQWnH~8N-PGZ4>hUi%3`F8;ac?35ZaeI&1vsQX
zz);MJ4}%XPP8c^*C*UG<9BzGlz?3E|B>sm1?+kQPm=7wbCk~4>rWAaZBE>dW*-c0i
zY)UZK(4_y%EP_P}iY7q=n<atGQWCzXRs_=>9NnTs5v$y=*O^ksm5aK!5e~uI&n-|v
zvp~0KK*f1bVi<NBJ6aCCP+M@THVqFtB0?F#o%AQIO?sjWC*}gmne(QAABkp#3SU^x
zP-e(+0-o<96RlbIfKmgIfZ}~5aO-GlnxNlFRpfN6rL6m}4tuVYusqU(0>SaxAg*Vt
zgw86?xDMkXW$}<zz0iH*oP@OUCZc&^G}ah;!|A&T<_UODI|3eB1(npm2m;ghcGs2b
z+4<=S$3L8vXKjgwx=ySg(W#pdnLm!%A6l-*_*jDi{Rc+Ox!P|JBAmsAVg$HAThe6#
zCS9DMe32AMl)W|DA;J^_uD-JO$T2E7ydMTNt^yZeXfhiIa`YVS9;0wd@r+O?jEs!W
z{%oz)vEw#m1a4ne7D=FvbYzN}V!BaB+)XlcT&Fk!C*BkJc$sWp)FwKi4kzKN@I+!h
z@NhAMS?+AAxl?UGneU2jzi-mYvKx)cX>cH-4n#49UP*oWnvXGw&wTe5#H`fOeZ<!%
zkfVR9pe=4&YeJZ%ozU>HEI|1vGLCHR{`c+A^&3q2YiOUq#T>Dd6jyi0TRHF=PyY}G
zXuZb3!i=m=-T%ZaE>c~Jqz?H^fstj^(&G0Hc}ke2>9ZVoUG|5ApdHv0P_C++v3k?g
zXugtM6JP?+(8hjd|7U$)!;|DiB6#6Vf-24J`QwMNY!ZvF&cM88ILlrBK1YSIe}%Jn
zFYZy@U}7+o$fImaoLRG1YheuSv*1aZw=S}=KD&;t529$a(6xS=r|wJCL-DP(4{)%e
zN!O@7+#p>)2L9D)RQqX!fGeaco>;|~UAmkqvLX3!>}NOzC}AwoyV?gj3Xlcg<(V7)
zEH<VW8P;o@>dWRfL(rj=G=sPq{!hb29@O8a-fs|%UhXP7f9jrzJH5~Kax*R{GoxBz
zTX|WL7d>Ipk)YMO;4<XXajwMw%ok_7v?EpRQWo%?l#xbbPTn?CYR(9;kwRB|?p;vN
zXJ!ub4C{6|B2Enl1?*f){c=+szBLR75?k;G7$J7b6yehd97mj5+lPI1R4xNwGUuys
z`D9b!f6Fj+Av!Fd`&4(H>i0J{e?K8yfiao1OJnu(%%UP=S5kb$Z*`%@fczZ8h-gvc
z9?S$exk5$Iht|J^#RYHB-#HCnuuI3PD*~r!u+&Pnv)*^Ip7KdWWTV3lLBBzXgiDJ2
zDgU{{%Yn_h{piaIlbz}4%OH~)eUfrURks=d1hr83q)D$yUs>0Pky33a`Sh_`k4f`7
zHpf@jff%KbvqP9+xC$*Q{j;MCHIYV}Jk<!|t@`!TqMqV<-B1jAu<V{+6>_N3q1C&$
zqiADlK=JQk|2mnI3DOGyxjyLVb<dx*=~?PV`MoLu<!*WzUn-P4)Id;b1c#GrsFaao
zFo*UI1;P$Fu0<G*TM4rpwh8H5YpB#`;v{A_ECede{G93S1reQiuHsJSOU!&x{xe@u
zH<88X*(r?iN@h1qY6tAUBx$P#x(rz5i_JWrD>-~Nj8fZ>er<T(J`6vOr(-)bI2?Z*
z{>xkh^wI#;1Y3@`L2=V+E7HlahN>Gj>wk`EWq2a6orn#=_C8jc>Hpn-gXl5JY)xpa
zm0?9)?VZm2Kt&oWW;slK<ah#gDFfVC&ow%&yV^{;>{@eI1GGKLtA4`8v@@(`FgNvp
z^<&ZMDg5ilryVV`wljZ0o<~rHTXj&b5%Z>(Vzw;FTc;<*O~Xp%_M^0%%Lf(G=#AU`
zV5PDUJEP$qVIyaIf(U7eS&z39>$QScvMOM%73AFejZltz`(NTqGs3o>hi6vGLdOg}
z$NM%PN$!q(eUTs9Sj|UV;ZRiz;IyseMM*W93o@-Sip$gI!^kT&XTwN$#<Tzr**m?e
zA8weyT#z^9NgDqHJpCtA`&-@#nnkk^1|~eZFKD*59@&@%{b6a=HjY*6W+@<+$4J08
z!G5Mdv_Pr&2WQw+gZ+Ejxy__)2q<R+pb3Q3QH)TK+LIVgvCbqK+Sv-RQRJ&G-Fd%u
zJ=stNvn#Ptg_#JwXH*`AmQvgTmk9S}{j0sv<e^~a4m8oOYx2#*)21b;eX<@zS+Y6w
zAr0sJejUu(FlTZ0mhI!`pq*riiA-XUwwL5Z>s;1Aq{qK0%25-t^F)DGX1M(93EbH;
zXgQ{!!-1AmS6a|@6uSJ0M46^Q*0niIKp>^}FM_Vss5>x9cfY<HEBa*?@Lsy;)SDn_
z?2t@;i+f13XB5bXiaB5+2!lS<c0}3d6LeH==&5F-7lK94`xbFTYOd(jms=f3emjq^
zKrF^VE)Fvi8*9Jk#oZz~EFA-BCH%-#k1!P=qggBZ)~39ALY_)p!x8Z}hjh&k<hVdH
z;6Q>~3G1c6E@F_|ar0!)uDh@8+s|aiMNa|#ZCvaKS_ZMAM5a;^q-u+qd^d0M+m}dg
z{L=sEbN5>V32k-~EWGTgcwFHCWi#MKu0w?7eE)deXUJ7ptk?FLwT9leGM2?w-ymyw
z%0E8bxm=DBG6TphOn6ph^F)+p=9uk>pTEv>(nY}E06i#O7>B0eG{2OPJaMu_vx|U8
zwY!jd*9d|h8_()dVv~r|+60|x*|1|}U&|(Sq_TGH!%O0HgZPPkqW0L2Cc{nSC&NuE
zW9-8X$~J@wke*kz#qybFH?W;=Ah8=oiCCjC{Sa`Y`yiI3-s|F_Vw5)6v;t;p7t!dW
zhwz3K$X@zLAY<8?aksHoMzeL486SLb_1-ilA~>o9X@)fa%%;fVYNiO1MC{4!!M)b`
z#(D?cf18ex-z}1dPl#T;jFphhbMI2k-=LHPIuN6VBQ>U4EiirLHMeks5B)-WqJ`TJ
zyM)WpG!5k9E)aBRK?a-V9_vAs1reLctXYEZ?zG!(1ionkb(m5W6QCHWzS;YbkEd_J
zhX~<|Cuci`bd@dq+hSg#YLG$qUt|CJin@m6lbFWM^c;ah3am<V>!SdS31nhmv$&^Z
z)W6{CYLqzS_Q;a0SC!1+Ct*MSfv4$-oYs3kY<$M}4R&#!ezaXs0S-ns^jpikLxtpW
zm9Y7KrCFnckCYfs20bqU?2kC1O2?&KHSzr1PC#-ax(sunVqk!ghyGLp*mQu+W*ylD
zxpLiLAcEU-_u$?67#Y=#h1+3GYY?W3+ir(m(}up?hoRC$1uLg{s_pMm4mwhU$J4C@
zRT~;p8^LF1s;!_^;)3&Z@If@a{(4Pkn}MYMJC{&9+pEn7NbiamP7LcNB0@atr2<_H
z8A6`_UJH709@U@Wtc(`>I2+$M{mB*;tt|+Mav3mKfzwAC?HV<7*cxCA<M$s>Al3oP
z0m_h*HkdFbkv5DB4^;q~ktBtlx=1&LseZVHg;Fece=x|GAVn(YNaISTtp@!WILM%n
z1AIQTB*yi1-r?be%}Gw@?rK^zA4|67TQX6fh7Qf{$a^M=^MBg*ZxJB0=mu>=+-s&S
zO!#%D@~?1P8i1U4^9PqdasTMg>~Zg<_4r@|r47tkA-i8u`{>qQog}vT{+$0M4fCNo
zIcYLrc8BF|DbsZ&LaNgo@u;QLfcH#rH54W7(jyUzq*>r=c<OfC*Bz|%WL3=U>)cIE
z8mvr;Kf710bD4+a=#RWd9MW?FT|*;hWEZS(0fwr8I`L$S^Zg@T3b;61_5|QKf&>}p
z2%J->4PZ?7m=Ob0c`mfLyT(7TK9Rc+oc?lh5vRQPf_?~u-0P@4-#|4(b3uw=8Np%(
zBaOqs4NmiY^X}ju!y<4(OOhR+htCw$UNU9y`jHd%PkLrG^9yNAk+Lw;OTdC>18X<C
zFt4AlQ3-kxOL70N_ev2N*E}&<_bmI%&^LmzcfLT#Byqz8(e*?fg99~q^Mz26v&L}W
z1O%R&!YY6NhF3~95}y5$URHSt{VQw0yVZ$54?DuP1@lT);ft;*yF~x~{#{AtdG@OL
zbIm?IoPIC14M4Uz?}N#5;SKr5IP|s_rpU8A<@owiSNOv~S;{nN)vd{US2zm$7id8K
zj3`tM@3M+{$Jv8WsXwo;?K7rGh-<1~9|JbNN--OxNbyE>mwt}cO?jS+Ut8AIIxt_d
zFoS=9L}H4viN=IQdNvK({z}duR#@5RdKQnWtr|crHKZ4;2dSgs((B_sU=89{>tN&8
zE^XNA@DDwpBtqJ!W)(m7jYfUJ7ESI(MQKyM?N+4-P?R_9tKZPK1t?A0U#M1Izlbqa
z%NS2_-(~gu0_4n_i~!1`SQjI?EF-mEiyz_8H$ZxB3m_jg+#;M6j%1NBM#rWZi}N1$
zXMIpELkbC0+42MA%ARGwk!GvMaAO)V?1MC9RJjiz9FN1s*>@2D`03w4Avj@ek;YZ(
z$hWig(Ij2)5puERDs=b#og^w3M|l3e(<pSD)|5ngVDH~3G@$i(kAX9W)sALslAugw
zxZ-bC+}Ao*EJ;WrXn}eeDv%btVMxf5?|W?2?%sG9AobwE=dxJvR)&Rmc8dJ1fRh=y
zz#LPdR8Dg*ld*KW%TeW9HMe7*H<IIK$s&!2!`pVXC?79zmT*48n<I)r-Ymt?x++gk
zfN0y{#K}obt^tJKfYIALg#t`fJMFY*_$Dkhzd`Z0X1>Zz9FPEH5EETkz8>`y1!?z>
zkFMUelz!K4d-BYdiAS)!UMXdNY4H10fj&w>AHprw_>a)Me{)uy)6*oK`96&p86kvH
zH(%syrhjj7SS_I?Kf;okRtgCX4}k#bdcRC7arT>WtZXh5o<`lUo@*({=$I1QP>&D~
zHrrwg>$ON4fpRPh!jU(z0YVj_?%Jj&!nj)_L+f4SM>nIitHQz&fA2nl7FN&i>x<j&
z3yWCk;;Tx5DkvGd$w4T<X5M0bj|+<h-6ElCQe>lk;u3?Mx6V3vu|77p!d|QIt<=k^
zkW}|kLM^c#Xbr~@nts;V<LGn;M6yM&CTDyHsZxE6&I+6}d&GE@Z>du6wJE|e=An?b
z!j`aV!7^MF0vC`HSbOuzh?;}uMTsL;)z4!MJzMLJc5A|a5m`Hcj&T-6T>m2zDx4V$
zz1{#ta05<no;#dgS-UcJP|bXHOmnu0)x#H52t=Bq6bM}@L^(~Xf1sIiL*D&U6%*ea
znEF+(ghF6q3oClzEB^o=Kk?`*SJ@26Y7mmN{4o{iu<I8RK!8gp)|rF6kJR3g<KeH0
z2bwL^uTE3{?<$GU<tMUNkTe8@_{!W7sJ#Sa6axss906qfzqp<NS_pdW#GSQa;w;;<
zWY>E_OaQhmDp0lVofb;D!b6Ig{6hSR%DrnOO}I{9Vm<Wuj>0^^9#3Lo>5Z?wb4XqD
zelcbUD;y<o6oA==7ueM{cV3Aw&y~R9?Sw#VNF8J_wEwMl9dg`yWvhC6E3|S~MFsRA
z|I<9<*76J@gtBr!nw_5b{0iR?h<n)!?XC0CFA15Nn~FBUWn?v@`~F@JV{}7)={?Z^
zQGM32&#B*WxfHP4v|SHt`Ng;BdadnY3twMVGvyQCN>A8s>P64gHGMDuWFm}5f@IK+
zK(C3gB7KbJMYzfTE9JKziR5b6<}6cu5-g}jL3kz+3Qj3xtv?^)nwjdJWIFi}E;zKO
z50Q0eY)Fr3AWU$QZkUzN@Y&5RWnwX+sWM%kO<KItO({@0B~Sp$`X~YYHe(N&5f=Bn
z5UT2RW`l;%BHM1Kvfs=gskI@_XLB{o%4GtqBI}+-)WSb$wgP4_4>}ori+xXCr4`j7
zw@|=pvAI_FEn2?l2_eOB{bMX(D18+HYuR^bC(N47P9?FT3Ln>2pa77KX$*vm+&_k<
zfQpS$W$YI0k1m^7b;p8RTzU_i5JtxF*k#+qRkg2Il5ziOZNume84Hf)XC)G$%8iv!
zkr__&BuDmXu3^Sm>YH%2`$gCh?K^F|#B`-EDw>c6CwFUfrFEEzz)3S;)cO}HEfZX^
zomqPMMiyIQ*`+4$)(U*RhJ&=I;W4K5TvYDLRpFme2>!~!T^$xKPr`h72rB8T?P`k1
zmj}ug`ietwneYTPAhGfpBZ*M^&#=#NDIi)6de>rBvP3{4EodP!C5oI0E?ca;B@g7I
z0ny08{u60yNnKzaYqZQCh)aoDT7(hzIUcH*(R;nZJT#j9tSC&Mv}~*j5Vv=&z~{6_
z=j4)2W|ZrZhd;;XdM)#?cRtGhe2Jdf>#pdHxkrfeuUnH5S8n)7F!?9>(l5dZ=R$*G
zpQOt>`bFeb6=5baj>EZo)5e3lkbTCqI<dF=$!v(Wmlf|n(;}zBc1}!oL=cD#3IGHh
zPzNPLU;+pMCB=b$7j_@Xq{-SrKnp5XUh`1r>+0;}<u3`^|7JkK|N32w9KP<oH4)8J
z8GwA&8p&dyWs3?r95sg#HtmnVmE!jI4^oe{u$*tLQ`zlJ_7;AS?Ojr8%9AkR{iH}a
z*E3uIKb5fR&IU6euEHl_jeR^Ntdn%sIt(>cKk82oVV0~lRR`pSEq(RxXJtRsAeZTE
z87|7^<@D5P{kTn@18C5D1^zhNk^6T72XR<|9o7aC&nFOJ^yZg$pQD*(#Iaojxuigj
z^AQpxFj~QK@RzP1N9a?YtpHz?y%Jk8iG`uG_XrGuYO@Nj8g+DQcvub20n)ha^4}z{
z5n1lsjf{50v;xfE*V@@vgAx{r6hM9#Y#?2fNEcAmz5v#Y9wnzehTg62Za!n;|5C<Y
zds)bQt6X{4r!aBURw?6ehutPWNDrziMUp2Xli@@?mhpy&Z{l{A8e|7T`(&G0lJ{+O
z+3yh%<OA18f(<<e0VdCJ%LU_D`Uif>GxlesX$>)aVTN;)h@&&d(Oat|vMhMj@$4Z+
zrA+J72L%3vWYjDgTE7zVcN4T$^m{cdv0Ij#NO#WNEfK&y7L%ed2Z4@nDS(nD9cNh<
zDcK6Dv!xc0NV|?bt=VWnzEdZ#XB$JHB4CPX`U;@YEZ$jNHDJuVz6z<Sx+$)R*DVG!
zTrr2`eulmKGVgogUmB*HTmqR%+QVX5d5}_zPeIMCgOwREf7h1JSWAR<*7VSVLgZlA
zLK@9PuhqDeW0_+6+Sh?_nH~mLD8BO$47(t_T(9KP11F7}*xXrkcfPSTjHvaMnV_iG
z(+Gd|ns4{5G!jPXZI0=a@ZuDT<V;Yd6gk0>H2PlOkAeiXYZ39-yLoAEY8&S@;x6sx
z*rRkheaVaPzkPW+?OwGzJSw!mg8$lnvf78+mgj{Y$utW_um7ZS$oUJ?N$c`IUJC3A
zFwkeaa!cq`%c30x9|knK+p;`dR5!HwJ!tIh_V^a?$#MEtKbYd5a?K&b@HWXAv5eku
zKB}$kHzzdA@SmEDoL;RImEbW@|1vL|PkhsYH6gCs{I5v?^fe9biQJ20b_d1s)Lq=-
z$ha11tz3z*p?x@_B2_Smd&RL#^GVeC#=ctznHd0s>E|E0S~av8?(MGmwt;EW{eM4E
z>(vZv29-Q~hj!Vukn^Sro(8*0lJPI$C42fl@j!CA0C!8rnq2#&Bm%sHFg+5M86~+?
z82t(YzfWy4K;|@P(O79Atf!xw0ijG5+fjM-EmH%D`Bca5mz(asH3FR>m>zKaG)>({
zA*1AG?T)s<egs+I!InvSGN@=Z(A*#cSqd0oig==;y8KQnC$%1+qvOQdf8|zcRH3$m
zz2_sLTah}-m>k@ok<p#6{KgnVa5ab&3e{m=^g_)@rT70#M2&~0Yn<An+~K91Rz0f=
zQLap%i~ueer?|HdhfBg7eYfzWc~f$_8<%C7-BMVx=-kerP)Hm|fekO}bJvT}f-9%-
zq};y;p({LEkoFiXB_m76>d9D2$HL7tEcIB1<grG*;HX+eCbA+`^ekRVv^G!up`9hg
zXpd@ejfcgq9?IHzXP#l1nz2eWC}UTC5MEq0O;Y5C%iTNzd#<Tcqx{O)et^KFS<)`4
zUUIClAnY@zu2r%^HH{dj&_m>Wq7}|HIersIR#o*IwzaABee4uvX#9H!wu?}rv@=`w
z?4EyKkG;Uh($wj6fwp#lpnG=IaF8--0gokAgQQ85)5)Da&}?w*<LAyT%_g6Ppij>b
zfu_QM3sW~c?T)oPzAB9Q1|$s~4rYWKHk~;iPzok!5I?zn+r)!~2u+S?j@AG@#`PPQ
zd4s8&pmo^ze6|e3A!D-U@EHI)%b3oKC#l3(M8)!<XqZj~X}r;o#@KBr9es!PqIUWs
zQN^MNJ?5g8v01PSb};*7t(sA|+Bdq7r8B1wLKs*~6M3R0S1$n35~p2u4A4;gqGd{6
zO3IXB*zMy@4Nk%pWu_t`)J(EuU`m7);&SQhyI%E!!v^p&m_bPk*zfaL7t*ak<MMij
zdCdQfkQupT+T=?a>*Xl6L7~H@11Z%;UmpSoJAV$88erD@TW(K6bJ>(yO>|#SxpzU6
zqNd2m9T@3IrY3vb{7Vu(=FbkU6Kr!Hg$YE6#EugyyqgOHXvi6Qs6%4aG~6x0DW97F
znSy*gkWYa>%0P~tW#FA^I&i8BmOrUlGYB_sje|{^duZ|l?n3_8yA-{6P9^Rk<tN@D
z3$ahnKMKVxXb)5-h*8OMhF4SBk{uK@KRlU^>V9beOcZ=U%uCzeqvHIKlF*M;EnM|u
z;9$5!>R@ZrFFRq2*3xGsnhww&XbefmoRN&ar%|4zMpE*L9e@05DA8PZFB!6ze@(hc
z>K+0%e9C{rdFST9A}eiPkIAx0{)-2CWA1{8k|Eghf#vRUAw_RAZ?tqT`u+D0dS7H_
zHguH-87e1M;(<cx*bG4!)p8|+8^RRGo73bw0Za{}c|B%W=}NXeirO40)#L+}@YF{^
z(qTyInm)#YxPu<auJca!k$DQY?8@*6EKcb;8Op=5`IBmx;YB8csU*NuXUJVVPT@9-
z!y)lCns}iMuU(54UTL<x$(sD(dyl{O^n`S5P7WnQEVL>@?+J4GAOXz4u`*?#V>B8X
zuMD&eT{>a^jkNBo;=CMRl;S`4lmO$AXgmGA)>_kZeSe%M{JhY4Z!MXzLvek~h^O&0
zs#L2a)izXwg85bIf!+nP@>S~h6C)EXqbN!?5i-Rm5an)X2sCP!;U69dBvp}0U>6Ej
zX%A<y4gD71>Kl<)52zWQKe6%q5^!8^KVMWZx!qVPp^%|;h3&}V>qh95^M9n)2GwL7
zaKPYRN-r3!wb{AdsuqTk=(Y&^CTaryiZDyIY5bH?txRIP&sT61A_tad<iOkDsZsk^
z2sLxSDX1#s8?ovAmYwR7^Vs~w%{*g=a==5@_h}pE>|)61>n!7k;|$fJFZN+&nM3*r
zB50C4LJC5z0;e|lXKIGa`4HLT<$oj7Zvzav5AA(GBwqWM3j40b@*h}-KpWi$BWxI;
z<AMLH7$6deWMHB!88I2`3KBSM`^Ecda-rl)$}5`b72R82j|QrUF#c=>s;WAkse&p!
zeK|X*=U=;6@L-m%oxU=|psCII0r)sL3BN~!DK<$WA#Mt%u5XP*o(i#;gB<dPQk$ry
zUM~*z%)EJx=YK_2ULCy++KyP)Hc<!AnLD|w^}X1}s$6#44;%K+^VF+{He&}~Zu9$k
zzGhqT-#b61ZxId-W!XwC*#bIx(`_oaq$6a6NQZI4QT$mR$e-?oA)l+!gTJwPZ9jgE
z<hnVIC(BbUj!RL%Wl9#(pqD9DK8Q=TnQbdM4Cn32eMV3mvgSyp`3wKQAUrqH2l5&C
z(@WUpv>**Fj}tkfI#ZNb>dw(d>+%R7WxMXc5JDC7J~aMVb(iEz^wE$Nsfj60Ik_&D
zKHAaYbT+nzM|o=ALn4zB_K+jlU%sO+4IEkjUf6@*x8JM#SbmM1^u|zhH|3U}A<w*&
zgt0e$a1*~s?VVg_0qG^J^_wg|=~rO{MN!tp9zJ$ye6}*N=GGOdC<dn(>3<4H&Z|B8
zyPv~su(5GK@p_ce3KyN|`O0~<kC%?sL!*@{6Q7a@RjRClY6p3j1LEDjG=9w5P6TF(
zA=3GaWKuC5dd=_RfdVMf-*Vn{X_oN28+LvI6(9X#u4L`#vyAAhIt2(U!*{S|JHBSu
z9?19teVT0On%Rs~VzZ=B7-(LhM-7zp0Rgeu5cu}E-?vq=uDj@I$EPBDU-aa4Rc8|d
zmdbP7&qf4~M4(j5FtI0U5WqrZ2mbh2*m-meo6Eh@&*1vT<4hK`>=Bbv@8n_h-s;`Z
zGGT>bm=DqHs3;RHXm_Pwg*;1e@En;bbZegzZhmvZGQVfyij+=L*9kphX;LPs`DWnd
zLsw`qF8ygk(_!al1L=Qqxl~!dm&RVSuLBC8o7U`NOvzQ<M>$6Fnx@1-Uzv-#EG;d>
zQxskn@*q{{<)0*{RR*JSc)!m}5!+WgSR2`EhK}ldQzsLm57+omHii<IiS~nt;G4QU
zj%FYA>m{XU_7Ha8Zl_h(oaWG!Sk;{2eL>3WfZZLDj?<OPW8o|!)+xx3^`j6HJ^GfD
z2-}6#NLyg@I>d%Ib6dQCNmv;1HG7q3`ktxmi-rc9CSg`nW>R6{TPB%IX!kIH;5Pu&
z(54(3XLLw60}hZFhq%F&Pnosh&lFm4C}xYjpuWd~Vo5y@sBX-nNvi_Fl-UT$sCwru
z#u!2#QoBr&g)mg@#~nl`mMhpLeYzJ;Vv<S6&SfN0&=}e<qsM>nemfXKkPnK_)Cl+l
zap~5AKfn343($;hFSCZtW4RI!(O1k5QhR~F1CvPuyG&qu=o!{W$4FKxh$FN-27i&L
zF)2TAJQh0fzmxw$Ri7d0QU-a_Ep7p$Qr}W$Ng{Q8AjQ+Yg-?3eX3(wRI7fBcWK3GB
zI(?OS_nqNN`a1s)mAi7#Mm}Uz7rX=|x_zAhApj(pVUO3HRAJIRsDgO^j`W{dR7v3T
z5QcnejvD#Z(UZDm#;iso82NTzz+r;4v0u7^r5yAvC|@@ndig6Ux9H?a5NGINj|sQm
zMv9pxy!41-f{`TlJx8ub)VHngKnkBM##?zdMyw1&db^CDClNG>aL~Bv<!_U;X6`-S
zY%Gu1n{_^%6VjFz#81PUKiBL=f|VhtR!2=eh~IY(ntk77d5=&^dgDQQe~{jgFOke*
zZN53sYIT3NIWa{WwT{s>ygN9w5-`rGzfI+a(Tw)x@7}q@;79+CuSeAEwm@OVI1;@f
z{AB%-nuh{;3)XgX2OK9%1RkNERM*W2hL}E7*Q#fLIK@4_cQgsTfRNU)9z^!3^ooFb
z=HCF15Ai=fWb2Nx4zcyc@*%=do*7Rrqo+BkLq?<fpgc$-qyCw*!u0q~8HR(5P8p9s
zsRofv1p_jU&bruoM#MELerS>kf=EZ?9rxE$^OGpxD1gkee{~Jh!R%9Sp`mJhsBpig
z2|Z7G_=#;}{{w9ThL`oJ_afR^1E|FMQ9Es9=<?|0+l<otO`r2v&fgKzYVa~|jP(`n
zhAHKxk3S6f+S|aj4~dPe$b={Q-L$3)XhHb!IEO@E;sxX1fRQ+mwL(+|?_JG|;f{j-
zj&+~61R`tlF$2M^Ys?b#-*Wuvn{b9?AJ0rfq`na=+EbUM035f^0Z>*ge#XqIC4ef{
ziFYK+KaN25<QC<CWo$<hiP6kLe7GxrN}gW3|H%n$*q~aM!!a70g!g*0d(5|<KNkYV
zhCij7z{xq$xTf0rBh$<C!o4g*gli~itGr=-RG$;}1Li+d2BlIr!DE``S;-EFfWW9-
zAGLV1zr_T!0Yd-Gu(F%Mo~_e0NI!0fYu}IXG=F`K;tb21kdMOrC3<{L`;H?EKfz)p
z9$$>#Y!U5j)d-?y@$Dtqc~=={p;v&wtorn~6?y(gZjAy(>sFg{pNT#{@VrftJ&Mf_
zO;Mscc6VwYpgKv5I#NZz#_?OP@eSsDCNmaUiMUI&wIgn5il%_@9oM}lkZCxfr08v>
zYNxV16F`zPE0+8hzrK*PjQ!yygui)TXY>Y6SFCUON!+CTQ3kF`E?;l|{L35R0agwg
zjzjFZfB#d1N8n4EA>91RLyb-8B7#j?0mX%({ZN!Dmd^#HC0UYV>10;7qZT=OI4oKG
z_LwCkA4fr1hcIyHgwVSuCp~v9JnrV1#hd(B@=rn(61L?M1==W0H$7x32G=?VY)Vr=
zV%?wB^7!tW@&MdV1D`GkBlA{GdfKQ6&~hI>nxeLVxmnum7a>~k+snE0CVp()(~-rd
z3;c1<pIiErl`-1zVhc}7T8JZ>GHD=_HOY)(UzI_#y^5m`T;~`;@lN%zL3{Ymg&JFO
z0tn8&JV`Y?_KvwB6AqCNuv~x&-Wgy^kirR%!;8Y&>MTyPLwG;c?MKGdZWriZ$96U>
zE6_#tIsXo(cMd9$%*QRjTtlWR<e@GMDr9Fo07qpik6Cb-1|Cs8iSjp%t}N$PAenv#
zWq=f#tqx3&=Gb?RHH#@@IAXl3R-oc9`Mz$p%x<%Wue%+L!5#uyd_r2RPEYk8s<Pj%
zte-q;5=jAZs|DU|=;;WisnD`z#m01vG5bmbEsIzm-Uw6=@$Zg-lxiO8TR}J#fYT;9
z;58*^0b61$I(pQe2~H^2)gL{+1x>Futnk!RDPVoTz4f$1uk7}7p@#wF^2K!jPBX&E
zgP~*p-$(BVpy?!nRvjW3k_fV0#+EB&b$dq(Cp5eX-7nV-<#H%ZKNR=M=tPlUaQr93
zY;aU0PQ|RhNgx4LzXIZ|=CeMak^r06eo_q(0wGn8I~#LXKm0<wno|)M0kL;AH%3Qp
z&;K0W<K&fy@$ZwRfW$w>=+XaqN^fM%L<ek)r9ikM$-%7X#owJJaae-({^*RTxB)#5
z%a%O2SEc{X15!aO4P%wJ-5+9g#J|UX5odeO^@CQO^}O@>oaUCH?Gnp6!#q^ZbLNuP
zM*DSJbja-2pyQu;*<XWj7$7)w2f#tAb#Sl?wSk`~Dd)@Bp35_)zyeKZ;lz5+hesLo
zte2m^EkgZ=0G}+wNt77j#F~&N`*JCODR85m6@JUG3()QG5_(Oq%f{?BgG>xwlX0nf
zkuXda`1}E@TlLz8b~)Utl?OVu8OLS~3@Z5qpX^}D!L=G(r=iZ+8iBU*h)?I50<V*}
zu3Ev>6M#R=XVI~WMjxPH-;fynonz!ymm$5PJ@;UDiYB(17&}@xyEK)deS@J%z+oro
z2l@~uC<A)Lplq}Rh)Tsu?Fr~){q}9-NJaum@@E9l?DYM9CzD<W!P^RWrZiVenO6TB
zSkk8Bd|olT%`Fc&W3yfGD+XOUqavq*Dzo4^Tq7OK?sph(TZLOQ^h}oHKe06g{BDd0
zf!e#fE&MOl2=DMWO9mmY%sMy?j``O7?tnrE2F>Fj2wH?NWsms6H{|u?zg?vLy_h#Y
zLWpwQsw@FoyPe9Jaf%FcpV;&_OIm-O5wgB~j4v;dJ&MY-2@FS6ZKvG$S@HwNB`pBJ
zS&90u$fqR@tX?^ut)-xtlAH#bW66`L^qv{Z>8Hd3me!W-$kN=-MYNTZnx>&b3mD(t
z8LTZF^`RoHQGzney`3_4@jTFw1N``Y%`d;yvR)F@*`=%aO<Odo_deKCy2%o@7j{O^
zm*&`W*Bks65-pM$zuLVk&%+U>yf2BLC;XiiS=u+p(t&Dg;JGJ0#Y1fJM+JGA?I_hv
z@5cVmG!K|^qWU+dqsALV-*)I2`AS<;KUN7#N?SxKbPv;V@1G6k1ny+-Nq>j_*E+Wv
z+)Y5~+sP92yhheCRyXEkNHBFIVvqytTfO<vQ#wEF?~{QDPF}<15#yyn6xExeboB^z
z!49^aJ!DJY*KNL@y>TU*>T$O&eqkbTL6;cAYu^|?!MqQa|Hy}ML5vin32R1RtQ~oS
zF<Z`y<{Wcr7v`r3F;B}FdS@7^yuy?#6XIwuMGa+B4SXB)h4ZBUQOnG5=i6FU->3B?
zujuo9LlZQHi&zk;H6BC~yAvVvn&O4e023iE<LZrde~fMbi{?7IW%%@ZtEe9b*E@vj
z9m<+m*2Pra1PCtam`d!okzW|sKR~D*-4%-vOri626aO9zh)rNvM5E~4Ml)MX#7lvW
zC|@&}Z`VRR2W^7wZvuX^!16?8qy19EUHL^%-B@|gLKyr7x}bP3QtxLS@yem>HVmlO
z+f`Id<S%@O!3*7^{AW_Kc-JFgqFlek3W5DcQ-IEpumTWzaY-~k`L8Qv8z4y~k}<6J
z9$gFz+0MTwFH20Ihoue|!lqVf6lY{v9id*v%hg2qF1^v-Ias(4s!m0yK#?j)Akgld
z6*^a)JMm{$RMrXr$`ETF8!*DLMt6{tTAIem>Je7`R7#4^HZ^!QK8kEA)y2E#9?EI|
zKwuBQmf#d`Y6^Yuxmj1&I<T4R?F&CvC+&2tGd3f6oV_ROdOJIJGv{vmDd|$<ea3pV
zb4*Ar0?s%s-{cQ2C&)hkBqj893ffwN25xjpbl=r{(c=SpdFi!c#<v{Yc`a!i1_&hC
zz#$p%$WR09F;er*wxb13j22Np<U3A;t*KUrg7?tsd7JPE2wTE^m1n$sBa}F<-OKYK
zQq;FjXF4UcDY#&fCyH$NXDgCftiDQr-Odk>^1c~<xT`FwFV{lu(b{YHoGy1?>MtD?
z8|!{=Vz{HVcGTw}fGuJ_v;on-=}#Q1R~BP82-)Ylj(ktT);8#t@J>yGB-aCzMH)^h
z-x+-5b`I0UhxkjP@^+fx!vuQj_CnX9FW(^b=vHnPSR3W*By3~AdCJa|SCA8W9$O&Z
zCrGzE3$F{0e=e+8s5aue@?OgJ=UF0%vPBib)_Nib=(oSUss%fz{VF&@s#9Yo@K*WZ
zv5D*QUUxulQF%aV(aRzfmx|Vo6;}cEdR1)5tR;K0h3PJxJDAd?Tt@1-6GXlzRTSJ$
zNEmG3wTswwG$^e4G(SLES^)2*&`<8yZ6-Ji2`ydtw&;-N#TJ(YLjRE))t*l87<yVk
zwpAB=(=$qId`J`SjsHOIMHqjF3lZ%(JyUCzqTk9&Q+!yjmH@flz9-XiBDk#?qh+#Y
z8C(~R{F3aSo~1v?&8&&s70i+SRxo1;nVmKHkN*!ha*g$2oYdiRXW`I?0ld5OoWZ##
zVgm@{ws|A>bx>P(2PX{$q3X6YvNXKWuYfLoi}$a;VloJcmR{VLv&XQMDJMgS){q@r
zGv+%hSDMS(3DFV$Mu4AVY2Hr;xDt|a{S|)QB0E;y0O(sgm}uC4OlO<gZ;<XxADb+X
zs@}cNP9VRaX?6G?GFGfH%?kXo+Spv^cMb2{;7m1TvmJ+CX%WYKg}$y=yFw^&yg6v>
zaLJ+28^><l=+pO{R<F*gWyb<1Azv*!E4jyS)+TljfbWBh<=OX!JtaU&lkf@O_+?QI
z>5fJek+3g#{?`AR8Sy`K4uK8wa|UZ~^j>27MN<XQii?SGM<fZcw7k3;z%~>ih0em$
zR?8HFZ|BSo1Hf!70b}^qf8u`)?UEQkR2m~76yj$%rL_Btni}j{na$uD#eqiI0TD4l
zMTiYM`i?5)SW?XUXiKPbE$c%}C%1tr8EVnls;$C%2Z<vbLUoiy6V5Ck1`;^d0jyh*
zt@XpO#`}BQh>EMIlUVm>3A536L-UBaVgt;xj8-F`zvfKY#_7NST44p(a(3cq7MM`n
zl+c0+eKD!2ziOvsj;`+rKt~=tD}#9$ACD*arHucXp9vo7%!}?ruWJnz9(8p7-7J|N
z1tqWy9o+V#(Yf?zQu^)i$0;QbXkJxi-fZI1j+WfK)lE6~Fb21f5ZC#Cm{{V`c-W-f
zIuB-$&$zQf$tbz3DXY2T+r=2|<9F;Bw^|s!KplS8u)9!!*?T(=eG0M`AumRt2f1Xf
zlBr)uAa_MvCGJomgpI6=fPw=iWh5egVTSeMQiBRxz#sP_>`6kN<Wv_FQ^su_Qe6K0
z8FfsPDfM-S`Plfayvq;sktK;HgsoN59HhMG5l#GutKiZ<6Q-KMAErT{5-dy-y^T%_
z<C%K>ePr*qn9D)>oFIrr-IBrStj^^RuNO7Q(bGj6c2wR~h~d$r#+B5ffs}V31lnbK
zA`McZ68tB)`ArOtfy&??F9{6zys*c2Cdq-2?FQ2+I%2b!;a|E+O>()QReStN1KK8c
zw5F%#d$WC}r~~`n3bz|Z;NFq{hZ~_U7W6*l4v$tT6q=9<Xwh8*4lzA2)<9VgQna7-
zV*$V4msx8ZrM*vZD=240jE6w;NX<8b_Gs_CTyR0VcQk6<8--lM7$B|1(EL6ogIK^F
zCMDm={p$s>&#(_HYCEw7?D(i^&)|Po>@)GZ12(WT#)n<No)79~r24(TK$|fkg{Evf
z@CTL9n~|&jnUr9*tZ@ogm`<T`0Q$l|Oi%wG7_gAklMgA?+VTshic*xx*#Omr2xkHr
zehr;N_UVW7+!P81NpQOWs8Wp{_P?iYj|%<)tN17fR8@E1^dXP839hSG4FPSCiP$f>
zQzQtw(0rW))HvXG@4|JgT_=hDD+Vs;M6P!FR}PYOwDh_ES{rYQ%`Cx~d(M9~$5vN$
z0TN@ipwsFKYt|QmH}|?rk(@rwE1t71qW$3lb=;V<3^v%48LGe_M^CpONdw!AZ3%tH
zwEcs$*ngyW;C&^lHRS2ihm4LQ6o{HLxLF}eM`U~$tgp{}$GX8wEdmz{sZ>~S?ff5I
zy=7Qi&l5I`7AO>Lu|l!p!GgQHJHahb+}(>7C{BO^#oYr0mlld7IK`!Six((vZ+`#h
zxt<U2r(D_0nceK!bI(1qXJ%=ZC0z48xcC#q_o^in+76uAkP8<hCUN)1^F6Q_5Jh^A
zJ{bqJ{V9`k5MvR4j%oM|hU^<BAFs5#20%bTlErMU*RlWW+8`v+O_W20H$$3Nq8zW4
ztb!D-@ii^1(rSubsuW#6VMIz2tP-6~;^?BUj{afSKV5gI*_qwUuzTC(eSRxyW$6DM
zL}!6jDFr-FR2&!lvF$@qZOR=98|$aQwG?Mucca*k3V)^!Tds3AhFZqfm4naz5@(nV
zPFQoQScTk%r+H2a1_(;94vKKRU9&aZ$K}naKvSIzoH%`J{QV8}p?AarJwkY8j+Om)
zA*3y+3UtI{co`9gk^#&S(}B$BN4TqLzlikuCPPu`Yyj6o89kh5)zw5<<=MY4(x|%7
zLl4g*fHHRrx#GZga!u>}Y$^_C;$lzdKB{XIZ8o2_wgfesMIV~dr~1Qn_i2?X<|dw;
z49^6Zr!fO1{L=yHZ#F&1Sb6_9)tz)D5)*F<CZPR=*rae<w|*6hq-i3sioz`456~$Z
zVS|@obWo1!h`Bo)QV0mptonCU;`E^R(a}sN5RO5~rTw#J{MXM+0|NK-E6{`oJ2*T!
zY$|U*gB?)ljDwhC9k8Gf)v^2EiqH#vjjEqpgg0SPn{f+m%!a06dA}k+i8rsPz=(lz
zYAg|FtW~v7pZqbl-HVi~iWQD-2Fia^=%N%Wj881v7c9RNk*4zuc`I4WGh~Efv1H?X
zTAe!FA2uT3DnswW%Q4e_B48pY`rXYm`l3d|DS0NfEHTN@J9U@0riQv!p{JhdSsn6C
zz8haD=YJE9DtoL%Y+5#g2bS6MmdP=FcSEL4>eX){O58p{SNHPY^x^9a2EupLdC*-J
z#BUE>c!MxPOU9%`9-@14F=3e7S5z~K?n4je`|kN}zcJ8CSKo261}IOr9AIB}r&RSN
z>~F<W;lC5r<<Q{r_s)W5cLboAbHACS9Z>31OGb_My^JcWL0t!ypi9#j+j9)?i^qOO
z&X0Fq`ZQ7lWqN)2<=cngrGm4}iIlTT=#$4QQmt~s@C~Yx<)@c3-0vI%<!zF9l##%x
z1Dgkw=)|tQaEN9NeWq>Rte<PvsL(AeSxPp=RqXx2W0SV`8z{-H(^+D@PUjQrFXtvz
z6=_wvC)<YrV!2NEGD9Y@`WmVJu^b;v2dtQ7LN9Sy?_&=LrrFPyA3l!&V>wPU*^Z<o
z3#hzH%xc7A2c(BGet2tDM^;7}v#MIcSu<Y9%uOsk#sO2`q>x1YE+BzYIe8leIDg9E
zfbp@c*r1geYNtbYgCy4zps|T0ophmI@#;6G<?Q{YK(BrK68mPeVX^ECZRpzrDnKD1
z-D6gBHsf(<Zs0rdaRxI)X~|kO9XkBVD7~Sf@niIX<1!6S?{`O%J#ws|JLiv|*9&w5
zzr}l}n~`b37E1p;1fqiNtyB&#G9sRwQV+k12SzJ~vL<t+LggWE6)6^sSxMfCcbhZK
zUfO}Wb^};iG42+|zlIgMz-k`vy8d}xF{qWr85aIiK3mYkwCe05WUU@`)C*L>`Nwt`
zFHkKfl7usI|1MJ|eq*`flV(&5?uFr|;~0^|il}}sl^OtRiNPVYj8GKvAxF5%{Z|QD
znDRHpHeL``y~|H3@XtsmNzOlCn$<$9*KH$jc62Y9xO$f#>P*M=DJ$tE&U(0gRjbr{
zTr0CPzZo!yfAO_f3>Z48s+hZ(2V=Pm)jJ6pO#tU9VT2Td<5VWB=@Z#>vHxb;9oGe!
zAj}I3C-M;ep>qi;fB<<AytPz!&%OMvABpD{9Ad<^T!?;a*nWNaKIN*quc5&9ugoW{
zFOxuSfx~g&fH&S*OxF`%Gn)M)0Y{{uQ3ueB%u8q)rQ!(PNgA?gcMP%vTcw*ZhlS4i
zz1$5bd^NuHa-L|`=PBY#+YRe_{e{(F^AQac4>AH|CwGDav#B?GXiPZ2f@x?G{-?t+
zNF#zZ0=->NHexFiV+`d)_YFEmNs3F;Nk{7~9}(IUZGnYQZN~Lsz+M*~nYXnH$$ct&
z;Ycwy;B%3&S5jD0x+^vI;|&K}_z|kC?5`tb(&YYUq(PF<fPexe(@?&bsi?yp(F^iL
zWL$9Q9VZLNpu`sX(IEKP7U3@TpMm}c8rLtf+OLvXWTj+d(c-v%6{Vn?=@D8gbmak&
zgJeL|_RuJosM1>L%=cov*4ChdzLoq^<V>zSfrPJES}^v^2_D!zCs%9!`mj$G#`;KB
za*f{XoPE^V4D(>cu1K_BQXGXY-K&r^1j*!VgrJypP|3}_Q_n63{ML(~MLyYd3Xmn>
z2?d<$tOo}??JuNs89Wv-eIJn_&o7!`5eoU}4)=%j$U?@QjzI#NIqo@%KwxLLFLPQ6
zdq5(}CPFEv?3e*-Q=nDHjssu&g$f%?b9{h-RbJV_DqBkd_c9UqS}`L(TzZM2KlOzI
za!Ahcz;jY4(QChu{3BmbUsH=s7zj=n<-b-{un7e$a3ew=kVz;-8acW+8uk=Z5A_-#
zJgwpd7o_$S2Kz@35yVWjtxkCj`Dijr7rC}C%0_ae_RH_A+Zis=hwuzW=`3*485o<T
z__a5jmg+!5_k~5K49=>;R)<a+h{{?k#s<y@COlVXA;}H*^k<9&W@Q3PUPtK;u_C#{
zg02C@`%B>=YDX=FizN#9hAs?)Fu+rlpWlH4pNECvGK&KKPA38f{0K$y<iuF(0>Kk?
z2sJ1woY_q+6mtT4#+WHkHqDQwobS0=NVXVP^PD1`k$HNuQlG5MLNx^lXmYN<4rP2>
z=u(sXN>Y^eX_gf}R7U$Bi6U*Ums%3VWo>r+-;L9SVb(K&8%`em1>s{lgrWLhbYZWM
zExAk~(V100%4mb5-&7zej$jrLX`~cAb~9Ys_E?*J`s$yl(hnyE%V_C9F@Z}T3~zCH
z_2@NVsw17I(`QM#SCVs7V8$V_!wP_*de<E#UF77x_dU4TEWfQsrR=;;ZU=z{tV_9T
zFe7R6$pL6xrs-a7@)v*!+@wK=qI5xH$|qe>LTkClD)n3*%|Wihqv~+;T{!D~!9hL?
z30UT3t&)ht{ZLm@KqU`Mbs6w(Uo398tu2S?_8a-^GgmYPt_8o!$Ytts`zW(g+>cYr
zw$b$Yar-mDdh37HvoA@2P`twJHz3>FF?!?Qa4-LmkO|fm-yRY$i#>}5n%chDy}+7J
zo(N9qSm;<F=53R4`xM_6rvqx_Z(RenyBcAcmYG!MtDMDx-l#QE`^Wf0rrWoMn)VA<
z`?U%RkSi&lKTHV#u-hDs+FBH`nH?|dHWCo3u8Y-98dtEaZ92YFUpTiPwuVk-@llM2
z{xlJqnXnwn%YC-~*GNfMz(3Z7LYM3>_ddEW*9T{GFHAN^_m?=kCZHy(uTf1NTN&CA
zwaTOpM`DY?!5VbTSKD#4n5LlhtHA1Ir)%?4o((*7BkoXyzlDe${6^;FE-HBZQ1AnV
zHpEhCPj$}^Xa7P#q*fFr2PZY5%!sDw{ajO|yp+~Mqobq0<rgvCNoW;(FWGo>XhDkD
zu^o{|*~Qh~P5_s!(e_TcRebxYmo+?UzrNcvnz6Y*@I}s$wQIB9=A9Zmt~u|NinCBn
zo9cZhF}T9nf+_!GkeJoP*+B=w%W49>X1JyQ=w7K4UTCbDo1uP+NX1^{eTUpx#C`!z
zYwBGzpTZAQrZ_!cG==McLj=D0+&K^b9?m`P42PcM&`qxdz>n<jl@?!SM6V67i=C_b
zS>FbCjClIM_ktwf3%VBHE7GMya~|zXpNR_BFxkOzW&l)}7VPzlkwugV0xSQ3JXU=?
zxCYd*a)>pE8mxHa{g@GL9L;d40uZ2oV(D6RHoSJ3c=Ed1=E{5g@F^<z(Bh`GH}r~(
z7{llL!{8GKQyP1#u(RIU6?H>gckHLU(*$*zD!F`HN%38_1H;XSF<6+v`eB%Go>BSF
z@{rWBKWk0toT`MsLk7+Nj)vh=f70|*tsmtoO$$>ACw@)wvru{IXRlX91qsC(yE2ls
zhi0*Gy0bmufOAm1RYkyxDkx$ey%SIR8PVi6%gd~|!Ck%D&x2m9b00DVSM==?<wGZS
znlA)w$b~qMVMTM?zfC@`O}B_%*iv*cM;nyn+h)-X)jtX?>Qgvf8po-nu!!o`+mEHF
zy9(B8oV+DLtgWnU&?DGvkb#t1e)9VgKgNn2jx(Fa_u#*OE6bDEQTC~b-{foAoeaMQ
za|@BVE9qr4aH}rqE~I0I4`1MbWlLw+8NkUmczkaw5*utDEbE{5?!m=wv@*R7EFtpQ
zT6Sv$c=3gY8~A%Hh;ly*6xyX+EeJ+RH%@77{f95*uZ6&QP=Y<Auc{-BqW4@`jLMew
zPFJ8O><O(iEmau)!1G%O8Sq|m&jb6G9?jAfJZ@x!$SvqM0}4o_UlQz`DoeeQv4NLg
zN005xAm0=%$CVRCt2nP%_j62U0DBN!eVzmpcrdpgqE#g7UVSq)EK>_jyw0rn!tIWt
zLXI)8PG0SB^7dyLukBZ;9$Tg)D)%1wEXvbS+j@oSNIp$XK&%Eusj6dDe>3v(Y13bw
z*8`<T3tMZ~dC=tJ&frt53TKiZtSc`T{6Oh$wT&Mw;(u{`(!FXjyA`y6nS?d_!PFgW
ztjI$(-hJ;!RrA*-R3`t3{jr;scBIC@T<!<Op<2cE4at>k-%;p#STA!CIR>Cp-A`FO
ziyD*CO_Z9`O^E8NJK`t>&e1Hm3KT8%L>!HGgKyZ~=^`thsP=L)g?x9)z?QdIo&E=!
z6`lxZ{8OL_Qp)+&DN?$=NiEY`A^2pt{Cn3JX_cGswu>P7-WM09ZluaP0R;Vy2kst^
zFMIhLiqbyo#G{#v<T|`D_q$uwDWr&m^??aMw4LCcgDT`rh6~C*xi_RdcnoAP@~wvU
zm&T8-qifBOba1IQ3mz%Qed*x0ma)8ZE93Ixf;|)bEI$7zM^*h<l=WIl@ow}pt+`?i
zA<@r!9blwlsRI$jmQ_BX5-ub<gFYLQqbsooKqwH^M-q-j1WloU*koXMqLDc7jnSy)
z3b{r0Od{)GYGMZqt<;C1RbBsB1})3#pn9ur^So{C9aV8#==>ukQnLv;2@CeMn&40T
zrm3w|4(C^#vLHp%7C?zg+YoBg`x0l3+w8Vll$4Quhz}pi2xbVXpp*;R0Yx|!2`=TY
zaJLSyVy3sgOZ<m7oIjPX@lDIZ<LCm-$Jepn=Ck2>o%ugVByVlv_m*ZpE>tCXXbipy
z7H=s=#8*$86@$Mtr=*T^@>T?onZPg!`2Z`Lj$a@ht2ikAw-}{(U2L~616*@<A(wtc
zH7)U^66{ti1w_<6XirOWiL)H5J9@QL2QRSBC_L7I+an9N9&EKx=a8MgV<@*2y{fqp
zB(B#nrw>EKQ;NH@b_Xk8CfmfW16<`YNPQqvlg+e7b%4hFmwvRn@bhO4{$R`3tm@l>
z)n+YzQGeWPscaH;IzPv;s^~rju7CE=t+R-xUMx0lkvJ0gV))KVz1n@<O;9&(R*6~l
zXuEo(p`{#_I~+Tpvc)lp7~?c)#A^`0Aw_g9&^8W4eIjdpeWXNkvoe4$^c(V6J(oOc
zIK9QGYEEmtfp6BsTja~I!BjC3EJf3dSHkz3ESasqv^xHz`#_Fba{RtJ6Q}q~-z&xf
z!Umd|li9DY{mgvMhO3HvmvjHTISVRnhW<9EU>#!95AdGz%-A0dMD*81vIEfRM0w1)
zY=lBBmy{KxLq79FY{2cj6iGrpYrBMOLLTW6rpK?1SHbB;s(@M1L5wn=1-lAsdCA|Z
zs1-Fd9ZdA0x`b{ddlyhb4*Ed7LHg#CIBE|bnSV1tYl-f9M0eccN?6iE;El@8AGD?#
z!{6;)^T3NV#tSS$%e0Mctk5cO(i5w3=g<hDamL<okWsscyS&>@!DnRcETfLUfjzc%
zAT_cw;PS;*H`rsaGOC?=UULfAIwp-fMgl&2c5R|VGzI2D>{G{}bEz?4RyNwJYc<6D
z#AD-k=q!d%28ZiXLyMZ;Ya)&lU`!*&)sw<LB~fnrxJLf1rE!d{<f??J6^Myp`$QEn
zUwY>tH|QHc>R=2bcQ6(RIQ{xyuugl!RS`TW>ExJo+IUw6S-Yk-@&Vm<kM(QT2vyTr
z17hFicRGWrBmDnU#P!k01cvudvPMVuq(?&Zyhtp;7E+g#*Ou6K@na7FB0YMub=HVQ
zVc-kuI5@Q@Vm7Qw$5~tcqj&vRlgKBS@1r#zE$zB$Q8gP+j;n-l%Cx0p!nHE${Zd^|
zV02ss4{8Emw6zg244=DKm0QOs^i91=PpWw}gE*ChViI(Ct6GW9m(4ooPOH76Uo$L0
zlw>KZ5_LEU0Gclqc~{e=R%H?5#Srf&cx?32BT~1bi;rC1m+nSsKl?FSq_#$WFyBsG
z+|0LPI6%d1>Q`;p;miPO*#KKtQ(jq1z=AD&BwIMYP3=e<z2U~_RM7QcGSz;dsGAKc
z;~+8O3Ycf@ndJY0e|eXNzOk74W9>U+TgE6UT3ubfku8y@@9k0k-)og;1As5Z#_L8H
zVnf&){-Wah(QJa=1$(XEi@hG8uEm-A-E0TAj|XA)Vkj2ePI{WB@Kjz-pA8(Z_+mi@
zHZN<dN?HOp#C>L`?;WT(d=Pl}Z6?su@~z6t>_vMyg7gwrj<7r<mAD?*`^$O%sxxu&
zanapB>+yxz6?`>$B^4#{<42TF;kcH*5+?a2YB=BCW~MpHpiIdWdXYtC<@tw{p@!wb
zyAu7DYP=*>SS|fKQLOj4+blt7g(5FIr{B7M;zeMfngf;5f3_pR)<3I9<l07(0L60*
zYpR`@7zC~LaR@H>iCUL8nI{6u8ZQr#lG81oI9C$hft+z}n_WeA(Ae%gJ{e>&OFmG5
zTGX&tKWcDdEtk-)TL2Muvl24#1kC(5W5^NYgmgL+;dVMd29<5P5@-p(Sez&9UXn)y
zqf&SiR}|)>dPB#cOD)MgswmnUkTYKPkUYJMF_qIV_o&S1o9Qtqzm=8Ktm}Vp-XA7A
zWUWOE>4{Bn|JQ}1w52<u9ZbVK*K!nM_TpCKb4W7Fe%YH*4x<;ze*LsMs4e(X6@!3W
z_51cb6zr8%pzG4GGCxPq<SK!}fy0LFQ&hwzc`7DCLjfn`^K@k4T7rKk;y-b0u&r2i
zM8z1#HY6lPZx(tz_GpP6Nr@87898BN%S>qV-VUN}!G#--qVm4htSblH>_F4lU|Ua}
zLW!KtRcLamqp{u6DuLY+a11K%pXGs&P^xyw(f5)xF!43pgHX65BrK5)Z@eVOs10!a
zf4Ue&oi=X$uQOm0^(JJAF4M%$4Fd3*?={c#1@tA4YHrssi}wKWhXqCEZlzwS3@TpZ
zG_hqentLH6SX3dqZ2NT)NVGR@ZqB6Nf%g9TV=pK=C)g__EPQJ~KY=hgPiwqu(B0nt
zl3rWrIm&T=jJ(CS%yg^=&5KYpaVC+p*c8{5j_E4f%3o*KlcT(@xPV#R4lqpyBZ5lf
zaowb45t{JUn!A(fy$to+e36SHm3;{ZZN}wZpS+~uoENtVItPq*lf|i_J?}|8Yv&uz
zJ&4QUJ|THRPUa!LwaPw6mn-Agbh9HakVWPi`o5XU`IGzPm;o=>oHbDE-6k&kpf%!q
z(|zcXn~X1ey$fsKQq^P~i9Mu4b6@_OaMf!G2g+2{W!SV6wJ}=^+kc)_ljK!S`zZ}Q
zv_ep5sZ*s^`LgZWtgDADN1?Ng-46(QvyZ1IBRX2+-mPlp#6e<<j+Xb?b_r;oo8(pp
ztNrKxXSvLNur3)hQ&2P$@6xNcbkzH@WZt%|1z2k{F|L>xYRtIPJap%pn}Q=s|Hbx~
zdRLU~g%jXn-!N$R_XvEKJ$00Y4(Bd9#lt!ujLGO+_&&V(Q|%w%LWRz(Ln|0`rHn{V
z2(3<?UVWRd62qT`*v$o9k#&Kcvb()f0}vY69q$Ow85;@;!H%FjATJH?AtjK!vdzJd
zbQEAh?O;S&Bq4L>cL>m>vZS>GPYHL22YjdAFbdCv@bpf_{D7FTAI<jc9Li&#&3sp=
zkhr%rRtVk=zW)#nPTOG&I)&EC66BSx<;#D8a439%?5=kwpS8d_R^%W6RR~Vi3`YXT
zVNytxtuauY;(F+_hM5)UHZi5=3<^G3N98@DhRE-9{fGSbRJWI&`a7B?zP<(iIt(cG
zj#qi0y~?U(;g^mPmunP%ey^&bmo_^Jyvl1eas#!4gbeQS6F9t9Ghh=SSv$1o?IZV~
z65U;oiN}!3ZMQ867ZL1Rft%MeFYbE!2$>UkDd|t#c(C?sK6eGm`v#%m>y!~~jS*0N
zrWIbgNR9l52&;KwGmY!S58VB^vH3?BSS7GpLr;yGoa^n7cjH=oA~SV6oG;iL^cW9L
zYLp&nsZDu6*GMa7xW#;E!f#2^`cPI_o#Og>HS)!Xj&|JWMi*&BGyWmR%9B91&U52~
zzJp=;nk`2M$vX>HlehT?r5Or+4Lh?Jq-C}4uoe%q8e-3t+EdM}wJJ%~{a37fhjkb|
zc)B8ZO>eu{%El9zbnRl5zLHgmJ8pQw4j!9x@4R-N{JWd~Gba0aYesdtciz)}<yGuY
z3xD1qv;+u$19VXb!%auf!e`DfgHiSs!v|ZpqLN;qbkN!nuk_y%1=BFl(J)^=CX?e@
zq2P_&7kqBQ_z_<2pn}s+v4Xj%33bX*U0m1Idy4bmM?y{3ogG4h6mXc4$jN|;_JT}N
zh3J7Xdz11<L^DR)5konW-gA6Jou<Ntj5cv;#rgE7qW}t|OWv1X6w^{ia-W@u5xJ|b
zLiPWHqytKXOto!cspU7IlEd<Jdeczl{OZIQPB`<2S{^3C&g$}WzYq0MCReDsw-O3L
z102n=Y;9xf|3Ba3RofVT)hh7+{XAw8(}zX|^go!i{0jQi(6y4D1&P?D`Lk6enU8c?
zgu4;WsmM?M|M?qVz`=NN;u~gm2|o=1+cr{1sQ<@DaULg6`2fhtIN5F_QG()rB?&2E
z=z?M;u1FD*TPfd!TI%4^?-ciA#}MI_|Ni^W4n|8Ew=Xmk9~%5&Ba-#$J%cX>)jIe}
zJtP*4a1%^mLWxp+^U5&FRe=wHXly_sVXQPMouBzd5LHZx>NhmWnHbM)57It+tEhIy
zT_VSi$)t9`=Kgfe$=j%0o!{l038ux^iUDunkHmoCK3Gy2o}J(ztYNEcVecID4=iXc
zo!s_0wS1NnzyPi2D6Bmkjqri4Bbuv;R%Q<XUjOO7Dh^cwCWaNE3JS;-l2H6H+>}b{
zFGPrGNNtb#DE1b5AMkwpl~h&xpip)VQ+!gRlT@?7SwP}Kr{DK;T>^V^FJ2so;w*@{
z8IbjwIdd)y<uyZ4El}F`BlNLdE55AN#&-tnQ4X~{|EZ;poyE9>%PW)}b+HVJ`9sBY
ze6P9|P3fn{5cA!`C0Nxxvu<lpMgR4Oy^f39-f!0Iyzh#loy;Ae?hp=SH@At};JT6X
z{;=0k%3%7Moh!8(fUf1-QDZbKNP}@8-EJo+y9z*4BldNRhaM$-+}@916@mKuRrAqD
zrfJT^cQa=<zF`E(?}B@W&Kymmt?1q*|Egx$gPpOCKe(AtZ-0y<728>V?oRwcB6{9v
zAopl`&;(puIUm%jQKN2ru$7+VDTymg?FwIl<8PlYy*G+%M*WVr75+SbX~K5joZcD&
zDa(%wg0lFs#Z^#WL+|`rt@(!qayatnCk)CPf^#fPzEwkiz<<>p^=E_k*rV;CE0t<G
zY{HhA)bB6#b4I^Uk$!kjUugBNLpbHD9*k=MU>~s2T?Fj#bG`@I_&dfL6*~(mH5jZ^
zUx%xy^4_E~1=YFG0jY+scvM(lIr`{+Ia$zyT<Z4<ycBL!PETEKG;>Q7h{7}|T5;o_
zi6bdXr?~%67F?*PQqxrM)_{a-5*nR_n$v*i>XrzX`kdJUf12sDz*8qRFZ&3|lxEz!
zkq37$Y;^ZR;f%sCKM1PhKbbh!K=}8<Uhw6qJ%7+t2E`yY$IW2RL5`~SR2`ef&%fGp
zUPe6%dOvHV4zRX<m<@%jrf%13zYTMF)dJKh*8Q$n*4()b(~F-zD7p^*>8<zYTS6vB
zH(s1P`0P)n_L3{M5%_xt2Hs|H5VX<H$&BK)-sagObSoI#!|85rmeUpZ@stNKsGaX(
z104=gJFIc3bG!~_-P)N;mO&>98zO8V-U|SQLkI<552jt^W!C-4XHaSRSQ$e)4*cXI
z(xtmAdxE(eK+sh8dBP(%EXiX$rv{*-&;xw?i({s$4FksU>d)GXwZTl7K~UOzq7rB&
zsmf)r;`b%^RvZxp5BZGi6tHScI-!<?bW->D0tfVI-nhk@1q7Z~>4EaSA-osuhVuUo
z0GA>y@5u**(>37mht}|>P|1v=$>(=t@RXw%W8nli6vP!8V7!D@lN=-OtdCb_kkdL-
zdvq68<E%L+9Frxa&ny&v5d)X|M3lxzbS%m>t8Bi(urN@zdurcW4=__a!slQ+j3P?j
zV@E$u&g4H{-*3+Jdm}r_)~r*n1kSq->Kn_|%$zy_8K~0^AnQA~o;aGU4fM(VY~@R|
z9^|53heycD+wquv6;9Evh-FzX$2j^e=;x?U|4tDTFwGZF2U>hU-?qSxNw_%uPOZbb
zs$2l~>!>;x>*F$JjsDXV_vS!)6B=v(9JjJyuKBSqsqL%S^HFS!jKi;!3t(z_DxJK^
zoVcm6YsK81E@dL|z4f7JnT=c)u$Gl^@K}L^H()9Mq$b0>xIz1Vsl0=!Y(VzPl2OxP
zyZl?)ne~-F4A>;=^ZqQT&Z4hYJ(GE$Xh8*3rnhEI0CCJ%C#BqC%@nIbb)Qe|0KNFY
zK4cFrXt&C&gDp)tUylQyx@KrzCQS060sGfMcK$AR*pETR-xm9w0egK`9=zNunaEq?
z4n;Lz!WwdfMC7B7^UrdUGuA?VDATL*IgTC1=R@8?mOyrE{5IxRmJ4$}Q-^uxWhg=o
z5tGnoSID<wkibG4NAp<i_*c;0B@KGZqey?it6<0(=t?y6O)@`c_yyYyV7vKfJX1Uj
zMVi6;C6&|vz6-L6vU6)lVdeL;o1hecgCUQ50ZXmI2#t*3EW<0}|Eodxlp@Iiob*%t
zrnnoLwbATMZNEhId=UWRAwc0VUqZXd&m8&P#OYzrSOOIZik@~dfZe}B(KBDd{R@{U
znyYoLx`3iLLuY{n;J$FfE1WunVR_gU`Z&CMI*8@`0!p@#8KQ8gli7;(^Z|e!$Gb#V
znm*b?(+za;2Nz6t>Vb{^7GM<~8%A7+ARA_o4dJ4w;oe`LU;c60%nVUs?+dyQ9AG!W
zVsCKn+03kq%ru0pkLklS`QPV;4dpF78Xsw1$)6OJIXAS-1t|K!2W9<mnh@|rvTmp*
z+!UOn!IZwgXts3eE|}shH0p3zFmB}ATKCoCbYo+~K1Zm}JhdV!;}>jbSW&}T`jWlj
zXn@xcbf4z3<(kWWcuGiIh??dGRs4Vqs@dLTy7+e4#;?>mvm)ivfrd+$;qAwg>^Q$j
zJmm$IBYmVrHZx(A;#4}Fs$cj>9n6H2hPR#bA0BT^MbsSHE!cx_mJA&k_%&qse6`;)
zYfzTBC1Dy2YxK=wlK{0}<TPp4W)SeDIbVhZ4OnFv!cNBYeWy>`(pFAAPAvT0(>#(w
zc;I!gxoi;+gPi;9tN4jS$1?yjE6Ef1hbd9lm=m)aQzmUU#Nyvuk6-K2J#X{BecNMc
z>OXKg?(Fr(AC*I99J_T7s_g$WL|>`WMm~u3Z3ayKKG^OCksWaO=YJ>Qb85B~XZ^V8
z0Y{VAWpN9j?fJp5cBJ^s7?fg`zkuiN4;|gFH0O29rGNJ9^sK&oE+lD&WO{qkzM`2`
z(7Q`aFq@3hN2r6iMt2j`8c<52iek^0U88301yI|D8Ity`($2m!SP>EP>@a;-0_|&s
z#8$l}%c(mGSARsMyRP<)geZeO;P8j9W_)yQ53QswZ&i?JrPBaV6Oa1mwr&I(LzSvW
z8SEzGd*>#yw|I4mvJLOUTzS%CD;xU*bQD-w#Te@CdxaN3Lq7DhgA%ZfqVoG11sNCb
z2c01I3tugo<?!bE5L@R-Bj@qYx7tL1Hb<w)Vju`IGLg$BS{5dxT;syh3a`zy14j!Q
zFgdZBshzUBtTS@wzCqm(Dc5I0duOY;nL1vJ;ZJ$#SmUM4$Ig4ziZ1nmSK35A+?+s2
zRS$ARoCXVnJpLJ0woKoVy{n0+amjC}^^_Tz@b3ls{U$J2e9ML#<aX!0YZg=;eB)Ra
zTmZKw1F&mzs<EN}(J;Xgj39cAN(Cic{!ckGR{NtlA>z&AWM@RP99WT{LZVN`xgM}E
zN<~%qYgD<Wha%l@<$#)OHMt@5L3VZc6&9T&mVLyI)cro!5YBcK<M9)PPfD*&UzL8(
z$*aCdsu^yNYIMbr1gazd5t<xkgV-xh)9}uxKYBBfT|HI6vY%@PDzRY$>dSl@Vp&9f
z9*EoDpI<+Fo^`^J2?|-VMdKat14)$lRPhoh7pDJvp+7FfQy=^``K<Nqj@}}fnXQR~
zmI~6{Py>Jcd}j1{#S9eb7|dL1;=*`pxO{+&Lv6F)wyczTtBu%snyS7d_MEFc)-d<2
z=oKr=DqXDPC;8_TH)9npzG%SW@{bab<3jHy*5q56gjlM*)Z}#1-97Q|b}6gXF+PD}
z5jbT${khDiIEa4^d4eMma$d08{97g9i?y5)xcDe~g|`G6c|<|(Pf}C~U;buGIG8<9
zIksqz^*?Hdp<qpjivLWtt=PPLnMia66XW^Go(}HFivUHXM-WUDAHA)DY3DRG*t4?j
zkIFowy8DA^#_z7+_zcEitV#Ct3>ExDkbo@hQ~Td7Ig@O;5}=KR<c69ry6itU^jO2I
z=~}lMHCEI@6sfeN-<c7V>>&MlebsQ(fZ25V{t-6`1A(;yD)*oP_U+6Ny~7nYTo$MX
z;8pW!)|*ziudJm$1?=LS)*H=)tgz_~>aClv(uYn-VcK^+TbXBYiONhQrTBlK_!mC^
z?anomUq7E?W9&QoeWHN`V*ai9l88|N`0!POGj(k@^wJIn3??ug*_hwuKCK-DDSkD0
z0eg<*bTwOsvzvI`08DU^I9s=gGy}D>@Eae=d19oJ?~Zyy4fjouzaSXCK{M2^^F(p6
zZxR*(<R%U!TbTghYbJ4h%+}gjlqVDlhJ?t~XfFksT{G0Oe76Bx3mE!$3Q>K;`E-g;
zc=Z|szW_`#>1c3ezVO&?mB5ra=k`g>j_fcqhkhEDN&+EKJXd+tQ-4py>Ay2m3Z;>_
zVHfLKtgX#fd0<5wj)AYXZBN&QgjIEE$pXX@^|-dAJ-wIA%%8p*oFyGpvP_M6?LYKR
zY(1mX|HtMLG7i?_h@-v}tQx%U>Oa|VnTSfo#yIisb@4)E%-O4KLWR-ZvguMvEZ{6h
zO%GVi6{{K`BMv>w$5DnWY<RfHZ4Co?p0>}L4trrM2jpaOEgN)ggMjAaPMTVvo?k1x
zll3aU5@A|92_zYw??4w)HSkq<mWvs0u04Bmmho~kHN3LAea4`6Z;`1vT6Zu9{x7=E
zQsZv$B_xx>fHHFlng7=eY|lynTcWG|;wQ_^{3MS`9vQL$04<F(F2@Sf$jQ-MDplMi
zA2^y^z9VN;`Tx^!dh0JU-hJhrL=ESy+9MI_UlM=D90%I8&;c4>u7%d5D&(=`k1Z;)
z@1l$A7=M0TZTym*1HQ7j9;n6Z-9flllkEyI?^%06sxYzQHochLA+<*)2gL<1SHuzI
zGQ77pKdV01xP%;`iq`@Yr#@<~)_m=Fjk#96CHIO(Up#isNlw1!Woam_`6_sIm|FOR
za{SR=toem81?x2HX2}d_&BKK4$A`<QAPGkiwtxM#W95gTMutp-@V|9vl)nb{thXPb
z)b8fPFV2tY+#z*pJu97>QzFZv2rcL7GXcj+ZI2&y`o~|n<ny}kv_H^pO_Fcvf2Ybu
zHI`Y`B2=yI6o(c2C9zDuET%N(L%fIWA8BL~bdY8I79WHOhzz?t2@mnx_d<ayEOaer
zYxLBkWp}NFo(woykiLf2<@EQT$TQ_zwW@!M5>FVADkhG^eM|YD1;RJB&2RJ%6{lmL
zHcl|WW2_l7t}=v_qgjW=8p8I*^huSW<MO1(-=kkhcYZQDdQD{2TBkr@2UhZxt;>@K
zFV&LuiPbqr%qb4g!`NbUke53c__}CQ^qmZg@EC%?0NSD66*0ubY#if(j6;SL561?Z
z2?v|%0#TQ20$jvuu;dmlb#DV!@ACW4Nc_-(A`Mb=<{9X}&Y4WB>p42zE&xsf!G^mr
z8_ql1XDF}gTP`iGCY%M4B^SRxFOq+IJ#5Dxy@-t-oy};+oRR?mtG-Ps)X|d<h%*0I
z-gA=c=gcCYn#t+vR4gn_;+~eAaD$-8_5YhJ;Z@U2;{j2PB8TMTrk2wa=B%&z8>R8C
z+I9rQSYa(|LuaK-@3ZN8e;a8KXO|c{A23>PXRaA;ei0(<8|xumFQ`qmsT(?$1rEC;
z)meXikf~qFZ7_9aJKrarqO$e{*r1=wb0I`l!RrAw#XlCQ1W3SUYP;Cgts?2k0|ey1
z(zsYkl04ziCiB>zxwlcRz@gJc%Z7`pWX%blcl@J6)iEA!^9=Af=OUN3#I8X*v{hMq
z_pAD51z$x54;pBMhp?K!S6}Y0WI?M=R2#gqg~EeJPi=pJ8>U$iul9VaF@3^a^B-@(
z<UIy=AaEO^pNjefXnE|x|E8C~a@)8dOu?=3u~B}F%!sB>8}Ze{w`&Y30p-A2Rb|Y0
z121;VhCM5{`C#6YCUo9QLlcam>+3I^&nG*_I3sGRV4RT^UCy$2v7LCnMG)}4;nl_{
zdgo%SlWf-!VV83dI5~YvB<J;1J4XuwzX<^|yz&=wo9r7~iJx&re7ocae-EB*2Q8BS
zAjCDnc^{Q{<rw63jv-)U{y~8ASGf2ZCb>3)6jB$+XX)yVZF0pP?7%R59}eY1xlm}z
z9pL1!>1oWo6FS}v4m@)}yhl0lQD~7hgxv1)|7<~g1!XM(nmcQIroj$C?xNQ};ng#k
zS~cesO?VE%GvMHIQNwru1|OpbFp{j?!aDjxb>*&zlpL3mk5aG0=z(H*eYw(jD;Q;$
zDvs-Q2L`w3v}z1^VF9oY<X_<|2<n&2Dyv1~L;kTI1kB*dzjk#Y_VRw_-_ujwIPf)b
z$0pHD(qm!It|jRnX$UPOKt_M#CDBwh;aiQ4?d6w2g8yVJu%4%|8o&aqigXVs{B$p+
zbD9CO7u=eWeS}e^-_e_ns>dT#KH!k9$I#_iYUdckoIoFamn@E<YY(6-5POgqj|~8w
z7KMC4`Vehw=2MoX(cY^7l!>n35rU~Ncd`}ifP}<-20d@j1%lB@6#h5myeL{B^NAjR
zP-D<P7}9|-tU(XRvU$k!13Ix)2x&DqR+JVSYgK-MM%VOKMA?rEcpP<v^&mSR-{Y_&
zfFEhw{KJnr-ct)ochNy`wr}9T7nZ5X(I^}7-b>Rj5vIy%Un7pJLrh6iix?}b+4R&9
zNtn|9?F_lUcRp3``FTfspg|x+YVFu)L_j5u6>JiX?n>ZKpr0-W;;P~3%%4FaR{N~<
zeLP1EA`t^utTCs&s=wBU4Z|7oX{ciU_1+7dn@Q7sOmsgAAzmusT?0rDU}-Wm`;5|S
z>=q9EQS##$QZ;BFXbR){KT3TKuNo)G?@<Z_IS@{OZc0T6B$}>*8v?HV5}C~%-R}?g
zk~n@DuWT7VOhvig4~j4Y`7&S7!pO19=OLfy2;^RFmVt5OUZD#h;upf-J5sbBZB+hm
zkyobO(@BrmVo2`_0&Mo;y#+FL*%5{MEdca_@;9ow;YKp#hZt{WTZhBz(~qM)Nem#;
z=7#@XN(pzt@alIw+u8MyEe1Boq*km;OmWT<2=%);=-5&t$}FD{L7kG6EQcuffd#ZS
z|3z{z#zgtuh{pv<VG)Fy{E`a<F|Jghp~Cm#sF8p1p%RRx&<ZFK{-cF3RV#ytrw6wk
zi3{ks`_t6q{6vW#N%vFs)9pEJqM$w#N}wts-ZH@gY&+l!LBMB$kFMN4i_izKL_r^(
zoMM6F9K!IV{Jd@9GVn1ysoU*j%I^JcTUFHxgzQ*(dz=e2G)3h;5795dR*wyh<v8qP
zkcB_{Y)LE4uByv3*bGmaFZp_60eCAXjE1IHiq?;Y!4>(~o{P|Z!rRYS!d`|xf|p$c
z>WR&Ns1TnLL^bcPVD2%!*%!J|Tjax7)Mj#(<=#xYuhSYIU|zW=S4_B*Bzl^j?$Zej
zEpiGt9^qGXk{eNhJlGJ0682^AB%?3F7`)R0cEgIq<%fL{DO*!FYtx7~+H>1JjQ3lv
zT&-jy_ThKDT6U@}P@t5L((54=GpBui2r&)z5U!<BpV8EcBm@0IAefBlAE$@Z<nB|}
zwHSyIYNdKK_X2?Jr+W+A!27k_ao}m`VLLdGD=Vae^E9UWq`Wxw>?>D5i`f12K`O^@
z%o!d-1FIJvBt_9GcN5tb>8DDzBbwV1GRC(!2><O(u+L}b?pst~x)jUjQ=(4w(|ur4
z5|+?1;LzNkstrKM%Oub0*}!+!kr%JPQ7}0i)^>!u<jDsg3m$O=<@4<m7TrV7UT~-)
zbh5K=wfw8;28t#yPa0AOp*>2oP@z3!W+*Hf(f-`mH+3ec5R}FEGVJ$UF?LdR1*Vzi
z<52z5r3em&)gm=3tC1r+jgBJYha%}z6{1-K4r}31y|*OWpUIvuxVq0fpI5=GW^AoP
zgCu8*I5{a+{%ts70e)>6V(#!bVhR5}n!=-lIs9+Gik<1Y=mQ`bX^dbWWsRlBmtbc0
z`&h3zglovnMaG&kjz{HT%db*7zD$LhcZGmFl8-ea;2Ezh;;XkAIX&0*Ef+S>^oI=U
zG%*=a0`*k&4zM+pz1h3nb0X(maP2dzAxjV+Qjz#m%O8s_^?%JP+CQ^IKN}p^ov5y?
zCBIsG^rt60zueX7x&VgVqbI0WS4F5;xV_vtx;lFNr>)uPsZQJEOnmlD)v7!V`^{F;
zm0t9Ng^;>y*QD(4UBxiwfa@zYk@u{oUG%APu(<P+nAeqCS=ZD~f3I}b;Ne*Zm$LJQ
z4K%OCLl;ghc5lPXObl2_z!#Ic<))yzcSi3sK;B<!*>tMjuH**`hp-`jx_9~+%!=~R
z9Wu{aH63$apN^i35{u<XwC)~t0{Uh?@N{q>G|k0|M#y&#GR5b90~263-hl#Pf@{k7
zhV0wOLSLZJ>=|pZ%-u<@iNVxWSZ_~n%k##a(m-xPtD&Kawhc2;u};{Z73f<j3MP>U
z&w!s~q{Eh*fqgQp=ZI;i42unZW^s*QJOZH1N%Ao6`uYYIYQxM$-PNGlsuut}Jx8u!
z(3ADwkNbaXl13+!Z%fwdSlY7R5n$F)3V{<7ra>fK{+mRIZ8=c0zA#u)OnBUWXfD>c
zlrpU7dTIk~zL|kqal~!btygp;MS@-#s_7K=5BkZXv%jWw{u5N8X1=?2ED?xJjIuWF
z{3<XGg@F%s<_D8k<}ogHP3R&zsf(y?Yq7IU%CDo_&1`K_TFwZCk8v+aR>wv7S7?X`
zNm~rDw8kQiS%~SY%Y1Re_kYASiMW?OU?`&SP71ckU7AXMLW$d6L0JELWWv<e%~x2Q
zD41a8x+l|1_NPFgr-%*DK(UwGFN=RBB)$i@L*0u-4}8lP=ej;L0hQ6hd&%}oP)h(q
zahFO-teGE84m}2W_{YG&lh4lK+&@LHh~IZ{>9^*XV^K?A-$`+iV{|$=N9S%;i_c``
zpf!7ouV<8;@_(vSfLl~-2A_{2zBB~-o54zI^CKX{;o8`T)L^r4{3~*e2|_8BJv-TS
zCciQG#tV51&2iSlX<Z2$0OrM175+aL!fe5pmb!8C+B`Np%W4gT?{Zc~3){vt@d}Fq
z_^QkbG0vy2zhKvMom_2?A}AZ~NVuFlxJ;r_(zT1!&{Mdrxmw~ukT(itR~KWyyih-T
zrW28DWMqEKgJR=gzzT}cTz=)n^|Nd*>%-C7P?QGoIf6(En=C)0vU|X*)iG?s4M+^M
zey1#X6xWVc`*7KOTL6o)Y_WmE4?jY+lViM*d(%UgJa7?~39*`X?d&M@NbRB4FlA7h
z;LDekiY#4?2J<2TplyAEKv2Rd&xECp+TX<}{FoJrlIR~l-N|Jr2COu|Zec@}V@sXR
z5Vn46TN&07Rug>M(hwrHMm;0Rl0>IdZG!?HjR&+Ji9q`Y+adC2AN&dSg6`VeYRMHF
z+vhC#PhZ>&61_PRSUkLHMBU|;dKTUF!+A_{FE1CDPD0fJrP&&R@4aC6Rymw0?n<;c
zxP)jcgsR^@0(FyTF~{aKRupi52Dmv-q6qA@SC9V_sYD+mLBX*P^tqNjr&@EzAfKjz
zSPphx=gK)-IJ1iefJ**b@sC#np1eV5D5(dN=y<gG?Mn)=*hIsB<;UUAFuoPPx?*Pr
zd{i{I4eS)!7#q3FOO|SytdE#r)j7ZSS~9=*r=l*0uJ<r#&&7(0^AGF1HtC9AabK(*
zXoPb98D}Jn?XAl!G0c4}SBn1h_~z$r=&=8-SaE(u`hnlo1UOonJg&GH&wKNGNZWC7
zO{nbibHz*Rpju|V!T4kax+D+Jar#(X;I{y*Faw?Gttf_}VIc-%_?N2f2mhU$$h#pj
z@Q=w!WUA)x8krnKO8+pqJUzbkVBCJOf3L(Z#$i<lIUF`)_fXY<zdc?{2w43~@VA#`
z!3!huL7s|gCC8kmcQqw-Q<cA{KZtr2=Dbg?dUd~mWz{4Qw1lr$qz1us3*<JYE>Msz
z1Wn*m!c^%wS=<MvB2PsUj=(Dh(Lk*mSFjXHbTBAID*WSM2KOK4nFMt<icfh)Ywi$-
z=B#7s){@$e*Ja~)IxYVc$H%rmT$<uc3-UJYj4q8~lQ-fpkyhIT+j6ZmIw~%;S_|Wg
zM@z`mqog)cgM+uWr>IP3(@8A$c<}FYnm$*XTtV$cf6rzJ5Dr-}ad96OTX&i`M#e;_
zFtD&t((=)3!L^<O8YjH-G`NDcw!Wzbod}+$5gXjmPg#q4mrwqIIzuO2sls}hu;v#;
zRB~WVtzHx_B+)Cg)w7s?e#1HMXZ+wQo%iyKXBE+|d~R;M=-1N{+x8y?y3pVR`^}(=
zH+Z-vx0svD&&IHcc^cO3+wh|e<Cs`I#m~jgn^2*{ws)V%9yCRq&|l`6m~Q=5VF~CX
z;Su|oS3^#k1(jkvt~l!V!N=0PR?PRGnVNQCZbSDtC~ByFPM@E8bvY8Dz+q9Pb*is<
z)~-*fdeG>}lKLy3@t+Rqs&Y34hAkNwH%ri4GdgKpGU;29RFVF7OagGQ`>SBU_(6*W
z?w9kz(J=lG=fh|bTe)!LtJDJVOT8~NBMV}=e80-6$I&nme=4Omb*h<{Nb5bMiURUv
z@6q3{%v)ZwVDPq&_ggcuuC#DQ65dQoVq<XCWlLx*k&)3#w{7B_s=PlD4OgwsS9M#Q
zK(B3nbCl=|ctIk=2CiJC-7`k#)rA@DOTs0WeiHOH5i#!3=|D$MnunLjoLN)mj)4MV
zVD^YQnz7M8f8$uyRA@}2QyyW6#WU{jofZ^`SLsFvBoy!1$DxK!H#H1Z#vPn~9+J-8
z$D?gqyy-V<(7aU=513f%!oD3~v*PHOJ)E|#UuENt5N&FoR7piRykEqmM@&hBrFG+L
z`MQD4m+6y}f^YB2{t2U$vZ}Y`t-k1|J@hum*!)<IS7BDuo*xJ6ujV(_7ku?smI5+F
zI;OKelLqpc1iFacZqV7#`<iz%nCfgG>G*XT>G#HHr?L;Z9=&EK@mF&n{$^h|#ReU9
z{)6#^&|AZePuw{r#7gSSZuhF$?-GZRXDK-7WuX@wT<Xd9RyU=&3!lcF2eTK9al|Rb
zo(Og*e(iM*GvLj0x)m4car!%7WopjX^plx^?de58L!k8G!fJ`9lIVh@nTJK`dV*`r
z#TO+>&qv4IJ&NCIJ00)Ib|pgF-2O7srFCuH?u}mT34u$W6xc6Ynd`yC$Ml$Kg%{c|
zCl~79sIW>*%P9tNZ)d?z4&N0f@CLB+_h9i?Z^Col{xvK>4HTSL=y<1|qVdYbW*J^H
zZwC!UL)O+$avRf&xjNdKxSx=3l$x}9NZ}D;v-|zy8#P_2lvuU+mNrq)?OJYb+E*VT
z)+vQ$LI{;Jp^ooJoXfBn3$D|h+@S1Xm1>+jWyAgCPCLu#*BLS7v1*<ha@ai8qu=Y4
z-#d<FkhIU{x-q7AbbSw@uDBj#c*6+OyAcA*+;Q0sKgr=S*LB4vD>Et=X$pM6ZYCre
zdz%b<`6`e<<i~?}oj$aMmD$MlN@EV5*8C_=Kke|vztR;tFq*cy_9`A|+cY?F<+ttp
X4p7ZYqJcaF9YtAQL#|%N^5g#pu#RB6

diff --git a/modules/web-console/frontend/public/images/query-table.png b/modules/web-console/frontend/public/images/query-table.png
index 53becdaece4fdaed338b050f55f9547e67af4c9b..4d63a682f57c6331d3e493db79c234cd6855a26a 100644
GIT binary patch
literal 28065
zcmYhi1yEc~7cGjr6J&6A9o#jzyZhi0oZv1A&Y;1a-~&N}2OHd--~@sMces52dv)tp
zon5E8Pj%Jq)vH(U-KRTFLroqNjRXw_1_o16K~@U}1|9?h14oMj|6bF)DO37h2-Hx~
z`FM48#lyoB8X7u3KYv0-cXM;|^71l1K3<DOaR2b|_I58QC@3c<Cm|u>;^LB>lLHF_
zclloX_GV&Y>f`OLrl$7xy2sDUx4g8py0%(dTf6w)=<R%JYHE071Y`qxyPG^YIeMD@
ztgEXV9Ua};+RDqX{dV1#1_t-_^%vyjtMLj{R8#~62D}`#zU`H6ZEd*<@l@*bloS?r
zc62_!y-D%##K*_;@W}iw)co`I_V@HOc;*Hu7k*044}m~<_$@D3>3aU$KCQ%Sm+n1Y
zzwrotx|U%v`|;P9mj`Ip@^-fh)T`Wjemmju)J&M{wPAnGGr2n4;#Euj)A#9=UiaP%
zxcpW8Bad?7&g;cc_-Uh#VB})L+Ee4v>qX(_eZM8K#b}@;P{yYG(O-erW$5AW=!Lpo
zIyd~m_1)m@+jN)=D?B<DJQEyq6d7gR1q5`vUhmg`GB-PYHT9f<9jWE`<$AQ(t8Nil
zs`gYqbAP65*qyiY@{RAKr(1Ua+uM?CV9(voOa1zlJqhgf^}22Um8Bft)X^OxvgG9L
z+uG%YW!QHG6vXtNjoX_O$?P3pW7%j0o})scQ?sVi&B+#J<ff^e{M^P)7Ou(0ba^AM
z)~~k1%ZK}IwW~iO>%2bZ)-M%LoTcjUEew|Z%*_Q?_FlH0Pgb}g5L=(QPY$H$e|6E3
z`*C@8b9+9$|Ih#XdDQZGQEiQ3)g%F#BX?zym%+^b!G)Rs{HK6fh0rGGOuE7J7BSWQ
zc>Cn=#y=VPhVsZ}ny@s<dQpn+n|oLJncl@wD~$xC-^K=8YxgplilE1RySuw*XJ=1O
zPw(V7I5@btxY*d(xShjohk<e2Qk0d_@iVh(`}DfLGS&=RV(}q9yy<Y#+(Izq0xvZ0
z>d9d(7QPMJeFt{&djNM1bU^ReCt-x7g+xAiLfJt{qg01|PH%OV*_6icCs;JnUPsmt
z%wSa9Q58vp`yJAYLy|&yG*GG*M66X-X1Ww&|68PC0|BmZ@uloserK=g{A?^jV*Mdt
zXr37d;o<y&$8uV86bFHi^n;o<qZCdl$In8waH^57OAdf=D*KhIv;4%DxfkY-IL#-a
zsW%wnPri)8KB6Mao{|$d2<(4@(auhmfEEqkC{Pfj#w5$(4R32=gi*sYia?+8yD6#c
z4O=9(D!#Re$pVTyuc+ju5m(VTsY_OoZ2;jJAR-Uhs&#I&3CN%l1@yN*@As+ebI=Nr
zRMlPUA}r@*=Pkt9pXzGTD7p~D!$*{Y2Wnzzqb;3X%cONk=mv66mBE!g?ong_^hdZh
z5c|m2$AFxGl_b!a(eVgr3x?F#HD3x$an|g3S$^uCCoaHe5z<B8_}MfgFBS(+eAZ?~
zNRLeH5DY>wYGFxL?PZ_L`Y9AQc!u~afQm}`NXNSUt3!pw_g+TJzmHFt10N`ivUGpS
z-fOr|a>L^8kC>Y<un<a_1E)zWrC2v+)$iu7qTpdlRxT`>z&t3L@DYUzE-3Iy=#Q;j
z_z$P#TGMP$(F2)gTJ%jRqZl)@%%g`QgO(EfH%pi!V3{_(rizV`;enZ<br=bo$m877
zl2p&t@0XqUckhtysz>!&b~1#{{RY>Fma@z}(P_Yq^z@A>@Lvxo#)?;c89P}Xp<Z<b
zrB@%ajCHh(2py5Ul!zc#`S1QA!54s;{Z<W+*bN^NvJop=b*28Sr15d<OlJ})UVIb1
zrk*VV+kOlWP9NYiyomsPnO{5rhVX6A;Kg$#OZ?SB#>!MM+NBNDEu538YTHJ$@o5=9
zJwmJJGrqMB2za`|KU`Z2d0oaG!)bO_!2qxPzWWz1|5Fw4^8NSE59PbS`$6So4#guK
zuF^jC^DAws*pIy>C#g9_7?vtzVPmG<KgF3%`90$gzTnzK@n{Ww5&q6S5cIjN{MW>d
zS%#hBi2}DhjIKAF^yFjFH!?mZc>jx=lWT7Kf+$5_Cj=Bt&_ut8(ZMa92i?zPM-y&f
z-)ax&yAV$%wS9J`Dgu|a!y~&#CNN_h<GC$xp@AJf4|?-RX2M6AdacN<HkC6fF-_yR
z4(10ttpuBQuK*ijGBZE}KQRjGmh`u?M67NbX$Q}$bSvM=T%vM!=q-{E|GXR{P=-}M
zG?<x3+a_eJq~d}`Cb?I0V3(!@t=?IUYSxeO7iMm&@H0oJvdfqiMZYHdSZ_n6EhD*P
zT?`S27}};c@|+y{*bZifl(Smf3?!2#0a>xkfWL`SBRYreTixjNEW|l?A$GapW^slJ
zuqWOEx}0i(56N|BdkWk7lHwqjZL@QFa0u~>0EMOR_8uPm>M!#f`MKQBDs~as^iqWE
zk$W}=*>N9wVHi76;1x0V>~mg&LL4$XX6G8OQ$f_?Rx8W4+3beZ?c<7LIOD;Q$V(?D
z!&zDU<7BkPd`7HtmHaMdp9j<WD&>S~7K$jNdw=8zH*${M+l)Fik0+M;zD=k}x7v=K
z;C|DOU!BYjY(30!R$8<-JV_AEq|O?BLka#>`oo;C^>%Elh25E+v)j(wztO_YtZzZh
z-U;t(vY`YU?O?L4G2z9&u@TY=wtex^H?}64us@m!hr+%;oP{gZmRFLf7j@TcSL_6u
z`II58sn)We^mDwTCyYH=q*>ryPByU5jI0XhKL+Zy?Wh0B463U}vbhWjxg2^#JV-~K
zm;7=f_8N$6_}PITyR|FJfLYWCPy5c>OlxSIrd^XiOQMNUK*>=q27eH;GMxAeORcuk
z_T^tRL1g4C28K0(WevCCX|*?0X*P6VWs_xUui(4)P^&RRjIC&f&BPPPZgV{t5bkEM
za?b6Ovcj@&JOfSGr-~yvNmdw~sI{$ftmj~_OUCug*#3;7mp6&~TY^F`amdfX`BVu%
zYxlmc*+7xRf`Pgyd}a8{uQXC6ngJP>g^5|C!+wLu@8~V{Zc^s=Wm5b~c1kpw+EL_e
zPPwC-ubq@A-D%iJZOf*t({hoSQz&L!js;mSSo52L8Ug6rL4L->Qe&Z2{1O}FzI(Iv
z_iv!du&uXD5M|~s++#BPcXqamDJczb;t8UJXTZV*n<7i%JZr6MKWmYNXKd@M<2Ikb
z!%0m8-?bd)f5t9@(~KeYpZ!<)x&(?IYexKU&4P;_gSkD*inSz0g4x<qR-twx&XS1H
zYr##IA=)nv!ICI|>AtGH%nSorWP=QB0S32(287i&2reH}Kn8?MZNA|FCYsiOj9Pcu
zFS|{SYy*&%sqCHg+94p@U+y&7H#ft^;P+-3hDaGN%Oddk{E2KZW8Y=HjPp)Rk1&Ml
zS@H5D5m9sYZa<I}7q=x))ghv#Fbx(a<Bt@z==N($!crlta{tQJMlfKQ&La6BYw}&k
zaUKVk9+!x3biX`P!s~p(-`2Ta;Fkl9uC6>@OV`!r&(Ch%c>-?iGeK`GuTBsNjs(?g
zExn+oSuM9x6{FA7zu#VfDQDN#0V+t3-sBYb6sxFych(_qX~Xw`H5sBbXUK|kN%cS5
zzG!x(gDtm)c4XwQ&ha=_cLPM*a_yp4B`(%;i_B>eVadA|p}}WWSkBkUvx<~TR?6vW
zJIW*6SjBxkcR1G8#hxleDbX(CZN0LiD;6T7Yfy(Ii%!>1ZRsoWBfZq$PX8dI36!AP
z${h4Rk~wPPd94yTGm9^+^PipT^x{)zep_9t(%CK@<3Nd6`*&3+t*~lwmhKR=Og&f#
zKH4cBT9g*Sxm1i`_Zuw}q0X&fy+XsrI(lV*;kebh(m?%+*o0k9g7{Ln@k@N0;Go<+
zq=9fR+9VKXHef_D8|@9Q@&xLY2&@|PSif*8$9UDnb&}`I($JfK%6v+fgTj25H>UqB
zh6JS$|Gm2U$R7deG@cfxyA@GCbJ_6ML7=1%U0;2S8Dd^rPAn*K#dHzLo4jgZvwLzy
zm>zcN>jineOdE?VSvpLL_&yi;=8k!`B;bbPGdm%~fy#s}lr3$<)DCSwVR$KAiKG<;
zjp5W61v#bRFv(aKn-Gl{OPfh&RWgI7FXqK7uJTpX&PB*xkMhS)XmwL|*F=^$n8)J*
z^VQl1SO~%ddvnO>!x>hdd=)CE%19l#+J$rasiH)#`$(;b$d+7t+5%YtAG6Wl5X5{E
zka$d`=LH$<JRsU(GdaIkFl61oxZRk>6z;Xq91!5<CdisUUpWL6@BoAm2bv7Ho<bqi
zMC+%*s;8_VGfUUvt`s0Yiiudh@=AykbjbBQaL!rsyRAC3nUB>4b<opBI>8v<h`K?6
zb<umNS}WQYcJm7<!CZXfvjzn;IFJdy*g<l|G*AY}6i_<D>Dos`l9%XzSSQUZWjGSX
zJ2GFp{0HJ3CUGo%2q(C4HjGFTCV`TRLC%j=1|_Yd0cBRndT*U36VZNk-kP0ms3_(%
zLeb2R8kBA4@Zj>3SpEX(`1AD^NckIXtDF5q1f9#m+Ezhzdju3D>{&FC3{RJtC~_c2
zTXk*1=LxkScJ$Gu`<U<!Q&@@?e*74Qtyy2#<uRQK5z)jLKm&=z$lXi6D$!&<kkh)a
z=$7C@N|r4vsIU&QV2X^UdV>Fs2gw~Jrf=cbWvXL1;yf8czOI!{`2e(v_~j+rZ6SAH
zQDqQT@lY7#><d0;(in`!?ZGY36IyDl&ZOf2Rp0>m!3xpk0-eKw4t^z!nGmelt$qGa
zoOM<|hB9cvArncHyx4YHKJ;3wOalx|CDOKFRMS%W>p>fxSi<acP%aRadG|JK5*zFc
zm0pUm$-?qF@<3j-6yu}Fxk++|j*Mv@IH0xKYL-_MA)!Mr1yH?FHMOA@TMqMcQWEof
zN6g&y4O_h%gY-zFvH#7M)eq!n;h(ct^r1id*Llw*_LGXg-cwm%mBe@E_uITxj|JEE
zg@10i_KqlX)nMRbd^7#rJcH;T!o)%9iHrS=KU${i(NMz|0nzmn=&7t9mu8*|#s$6Q
z5FJ9UqBP}1eIE-%SLsho_)|PN0(L$&k24R9luxnz@cMzFA`Nlp7u6!KLj2t%16Oh4
zu%9_Y=)*ypS`B}rnpY04LN8=(<<_hlJG)aH@H##3$~-D|=G?tq`R3eL7MZGOI6M1g
z?tQF~UZ}Di0kr`niWlu9%q2uX<^i6i?_9(heb~(X7=&O=&4LHBAz4*s8TA?0$8Fxn
za8Cd_Ruw9O{P0gJ971&4g68SZ<#k1b%sh=r0a5L!#fyX<Fd%1~?rJpXd^v{@`Dl_|
z!EYuf#)6+>ZEr)vr=>$4`?aKlM6oabE(YsKkIORBz7k;<c<2t234w7f#)PGq{|-KD
zrX3_1ELAAf^R;MmOCcLp^>WVV(Jn4S<VwH(>!?hIZF4mROR29L!PYy@Iwq)gIS&a6
zw{)QcD}2w)*?Fz!OKXjNA1R%VJXd1c4}mUj&{;cz%x)uot|j7B_kz%md-3ueK<00T
zD6K6Go+~b~&c_yhos!VX%ISTqd1(!iUuG@C?HCpMvds9mmM<u3)Bx}Vn-?|6Plf{P
zv!z%&eBOS_hXN<pfrwJi0*N%>@9qf;F#&_Z|3I7ZRPXjuWXT2kb97pCWL|7ZM(zs0
zlhQilP@vmVYUqgra>D9tZ_|PGTmlJCQl%73))B<a@X^!T%k!p)OQmRZDYM^Bd9AMl
z_qEq0bFYu8++rj=QZ~K4lEZ*nMn&2TeP&22Zi72v_rljL46dDAKj9nq;YYJtSz4Y?
zXq<hz2)+*g^X|>Lf1vqSHMHb5MVXz9WYwxZjDJZwMcn8B^;Q4*j!|qz9=9W+ZG00w
zPv8`akUiI0|0w{hddIMrF&@tix-<7rkJtQD>^#@pyycnQkOb&uzRKTN#z?9>a-`F+
zHUG=R$z}bkX}+1;dJpM5k2_U=lsFE}Y2AhszfaLFNs}KX>3y0oY40>0qXXs62o4y$
z`v2jy#bSm7N{-CRH}CQEoVyOInItk{BgFT4&|7TlJ1wuF9%t?2XtfbImBF#`n$lWI
zNx4+I+%+cUmM6i#Szpv6?(nZWUS9D1I0l?Z2#ge;VSCQa^fo^YF=5%6&3OX!B;vu@
z`V#h%cFIbw2x~_y<?ck_zI|t?MYjB<Hj;kTn>r<r7K7J>WKlyY_5$f0@Tm7e^bpvc
ziTc#guGv`Vy#D6nSIRPLpp_agJI09n#ThzZurJ0WlgBJ`Gr;jaz^D-k0ogbU@_eCx
zXn#KTF?oXi?wH~B;ZhVKeKq*>L^#b200XQv|C5DCwMr}%toCIpmgxc+!1f`8M;;ku
zkR}I(68;W2?=l_#>k4tquM8#&xK)`fQnACnWy&yt43OA-xfcr`oB}xp5?-n}|Mj>_
zS-$1?oChfBkXV^EfP7P4sXae`vD@!670Tm&#b*lmiKwb*7w^%L`ewkR)zX+ZUb&~<
zxIkm62q%DLGT2OJRT^)rd1n*LwrPS00;E5Scco&xHGOeX0Uz~S!#YxoR_<ZXQnY=7
z5!p2xVtH5>zbjbG8}$^-Da3F++QwxQ-zh#l05`Vx&$fOvHyK676|wOcx6n(hw?#U%
zi64suI2(-$#QXa&&-iZmWZURp)*}WYnXa?43GT-XwfJHoI|+f<2-n4>vRw<P37<Cy
z)~^x0^e$jwNF&>5C(8MT2$>G_`7DM|zMi}yRE(Qe{dVgyL%dW|I~*m(1k;Ch*`Za`
z@TwZR4Mtj-?}BmF44u&)L(uun1Ali^b`8bPn8q!YwDS!;oEHtb--%WSi_RiFI(U{A
zNY|&*yi_$WkE!G<T?OQKi&UcXRt(ECbhAO}%wAA@Eo>B(kw7czWpl?=HpSfU>W@Sl
zPnzP#;xZSo)oKyy&bg~zmK}5-x~SKx1YxTF5-D%tJI%cHj)m0(&CO25`<@IrPnMXG
zt2NT0j)b(CZNr21(Yj`J9g=gO-2BC8sUJ&%n}@feL%s9WmdO(-`^!Jfw88Kd{i%0t
znK2-~10Iy4{=qn-d<@6ji8_3F$j=lTJ_Hq{V=~kmiEB&-inhHOnIbzA&i%TYpa)b_
z(l2o+3`}Ue*xw#h2NUzi<$d?k$M(7U89-6lpMTxa!86EQJ<AL}s0jU;b@u}lf6&vA
z3gNKdyJ;yVGF!MO^>i5TFq6}n!w}<2g6dT`9Nan>N%8bOvjqsU7sNIsy>U}wWAfjn
zyX$DX9CYxewV#XGS+jld2vEq0%iV6Jl&@F8ox0iSxbb8RXh`j8W(jg^oEH<bYxEl3
zkoZ6pg0*l~J(y7zH?AbUQ}XXPhD4nSW@d-G(Dq{axs#*NS&1?Pag5||-R7757pFyC
z%<q&M814!R;12$3i<{lvNmK_=5K-p@=a}Pf3@&}ip?l*e@?A1xeMAA#o*wR|e{!!K
zq$JNBavx25@(o0JI(><=*8jFrHszR&wsH&=#d6M3di_cF2P7FYdL^e3EJhNU6(;u(
zUhUl6F#mB(n5da4NDO7udF*_{;N-z`cF+HTy1s`tLLBm~hw!!Q<<-gZ%7jXy=Ho}9
z_8g&OBg-B_j!?*3lW%vus;MQJIWzIK?gxY2`5p3VS`_Oqf~_S~hM^q7G;zr;wbt3q
zBlZE~<VSc$81y;xo&42l#48Gm(I8^B27t5!DT-6w2cy6P)z>1i!#)3VdWRa8#_UZ_
zRf<W^$<d&j?FaXBv8co`aMjX5cA$-(HZl_r2wIA<fC)zZsPuwhCihF*qaXd4re#fS
zU~&|!2-vhg7vXy_MF&;M4Tz^%{a(O?+3q8`J&kL?`_=azU0Clj4?^bqETt!0Mpv4&
z7}nF7y_MO66CelGB$m<yWYgd4&8(p+Q2*bn%+O8$gE{X>cDa8PkUL3y(2see_Hdf2
zt&Rx@-nb-x+v|%5R{}VFG=UEu6fS2dm>`m277o1hOo?V;T%WydRIx7`cWGVIA0?H`
zR?NP%uVB*7KQC?jD&Os#0QwubwgpFJN!qWZD{TijwY7Ehbg4Et$b{*VUZ8*a>LrW!
zr4FyGI1HkbK8uU&N<7pDPlBkfSzbrRF){{@{uFkC&<!*$4h*jsEcnHKhKG4L4~Vv8
z1x2i5B@~bq<f_JQu|CJ%S<>eLw_mgPaH-)oKY#pffFU}L3kP$n)GPr1CZ7#{anklV
zN3iZpKNP*WK3=Iew$4_|w)@PpYiUAl1y%a|Ip}i`qP%}DZWBj3&;tB$dKqilP4@w}
zQ8d93jJoh?RjXh3GZM%<t5Cv$b%#ow$oYJFE?1P{(w8X#U(@7cS_?k4%@+=x<wfR|
zyvC;9a<iK_M`2LWZk$AYWZaanOfaRVBRHk#0x<UMhfxRFzLqt~Qj`66;fD*?w7DQr
z7entC1-vQG+X}kR%<eK?0)b{L$5?u+@G}G`Yr#$5oZdHq>T*{1EbyX^$qsyp9R+>z
zXLIFNz-(|f4p>@{5%E7GjrV-9f5WVYW36c-Lp!7qK46E86v{9qZjGA<D|R`0ni6J2
zqs^=##<)pe(sSvVN%`nNMcJYg%<j@={tkpvMmk}^P&U$v>rG&8U+ObyyDf*52nvKe
z^zPRFg$P@i>Wv}`#DmKgzu&7CDmt3pCTsJ@hQzZ^CMp@SE0<vU5VJmfB5U0CD7SHa
z5e8M^dHy0c@HdSH6vsR<`+EcaLU=I<$XXe9#GhGzNwsm|>NC@pjNt$C!rfgSyim^K
zHOlVqCwCRm|CtSB4hYzM!q2(&21NWVqPC`6`7*7PZIp>?uMD698A*=m97=*>(}rs^
zxc!x0G5q<Uk`)-OfL#M8({k+tE-fU1e2r+oIHLaXUocVKi40%MRiIuYt4km*(4XK>
z-K$#BK&@zFLBhcA<)HXK-edMz@E>)2vIeRwV<NJ;rOV>qY}1arY?PsRLUib`Rywda
z?AEg9fDASy+nBJpd*K|m93ZoZV2ICx?b_R@c0fD^_sQO!6{C4sP0&ylw}#wCHqFEZ
zae9sN&nNfFUtE363v@2G9v?1w!Byao2PnGyG<X@7zqHG^H(!?B%%`mY=fOH>9dX+o
zzId6<@r~xQ%uW9K=f;2^uz~H4sD(2x7Hq-JAQRNNUEHzyuMs#F$o(%ofS9@z#g&*B
zc=#9pq*p&|?ankn&(Q$GgAWNP=92D?@3Vy9BG2+?=%&OFC4>0OyaCdRg;6K#Mm`YA
z>l_bqWGrA36sYvMEQu~G`v6k1%OJ@M)W>o>Hg(vnPa0{IW0YV!28WT`nJ6xYLx#pI
zqvJ0!R!3t3tQX_%)6~RaP{*TZ{;7#~qasIoU*ZeU|G~=!pF#|dzi6FHG%JhLZ#}0L
zH;eI{ZE&`fV7On_EuJ6ACmq9h<w8yi8fKNLr2oOI6KP7Ee7h68Mh#Nq?7e24Z{st=
zbo{=6ssA@Q)WSl|`@$?}j&S02dV~X2lRQ#VV6=4HirVBQh8JqGjP!PWaLB2&(&umv
zd&@IfSA4QR$n3;3XWU;ksnq*v(bo3ZuMc*EH4V~Xs);1-?<T2v;(fL}z$`=l+G2O9
zH}zw$tE7S^-~w@q=PP4U<gEwvc1yW%$>XHb+-{3!oE>{pV$~O5j1^`qW;YnbqN1%y
zc6ZY#k0wuaF(Zwv<%1u>^_X@3#{Nj6nNd;4<YhE>%0vobko|-8E;+-KD_wMWQSQX(
z2&aw+K&N%9B2;EJ&|<CwQB;B@`(PQ|7$+U~v@4CJfC9Nq5urZa&lQlOPajOt^4i&L
zu-o3?L(Hw%Fk4egR~(DzRKjF-QfJInzVGDo7Qbxu0YF|fM;8I^cOyo|UuZ>bNBjLg
z@@PWZdO#0}Oy@l0U$UbR0e?dah9p7+2AvzICuipbOU3hp6`gHjzepM%Q2gYVMZ|Hc
z=)ML^Pqm-ApV7o?pte8G*cq;vD6oFZu)lkU?1w%4?#NGmAX=p5Ymi7@^w5c<>~gQ6
z3$7$fd=Wo07P|~Kh-58x4N(!i8wo$f2#(ib+Wt2)b|ctqs5oc9_I<-hvpj7*Q9xZR
z|E4S0VvK?ksmZyVC-Y(I@L-lGgZ}4dUdH8?abhFPLeO>A6#Hl;C4)sDn`#2yju1Ju
zQd#lxlu4;!oRw+q7@l-s`*&Kh>lhG*?TU@tAj**1;Id10mGz22D`?0iWZ;|p*zExh
zN2}35!S#Lz2J7Gy)rPI1<^cNsYeRe=8|eqW2_A{98D95>&f&Hj)w3TpvTCCn1hhX*
zB~;w)r*LuT=tF9U1@}c@c5gop-fO0dWL&68@YP7LS#ynH@N1UD^2^*kLHOt+w6Tyy
zq{)^*npkvAjt*^`mUpy^N5L={OB0=;45$RBosJNZ)Wauu_XZL1z#m&rkhqDxuh#n1
z5uQe;HauS@hCez2B2B;hXkq=YHvnFjVsD#YEUP(7cfm}&S(^Pfi<HQ(+=`n#nIcS#
z<8AffK9{VakzkBM=ixa&XPUe~-*&GbceH`_YWc^6ij3w=$S2}?9K?OdLnANYtDFS1
z5r*Y%AVxb|h9-8gbwtz5jZj}g)i-<Zzi+>N&Pd0I*^*|ajSTDoTdw!XsWE1HBnIM}
z=;J^nWwEmD%?Hv5*L3R)(|!xyf_)zZTR?;%lKLj3=VY-F6^081QS&~JwRDj~q%Gu7
zAzOO-lUqzglNWnTb51<8?p9{eMyDLR*9pktc>|ZOX;?`lD92L$i9N<TiCm--7A91R
z>fkpY4GQI~f19%uC?inA5K~7FCcO&d{n0_Hd^eVFaAvN6dtxb2C{*%)RtS^?6#_L3
zmszdBDB)CJAYPneF7{NU-YwDJ0AIr@!${*}vjBd0ZZrB^`8MJ0;a7j<q^VJ$RO79w
z<z^-?A@ubfluoVr!nY<^*gcPI)V1^@7>T#;V~@fr?qYKq0SqxQ=0Tf^Lq?DjU`;kX
zPB1>X^KW04l#Ni(E{;%i%INmCJ`rpLO8lrl(!aIymF9@^Ih`FnK5N4~nqi%6GK%1v
z<h}+>F1cTuZn(D5A=I-Fkg3g4)_)%m5)rKXeYTNN%E<!@$}d}?6s_az%C{2=>o{*z
zMTqOb_oC<6`-?5}r3}K^E;N={p<$*lM)A9ltZt|0DzWL?J@T;APYz@g``&O6qdMvS
zGeAOc-Nm)&&+<T*v8}BkGo?9rcGLWJwC>@x*}c@=WUWM@)pvHZn^kRpVf1h=TglZW
z-fHX?w$X{p+c~WbDW<KD3Z`b&4I>7I9$e6&F@wSv%cLuf<v$&+P>N($s*vJ8`c>|W
zmkZ+#WyfLmN{s1qQ{`t*Ek-Td?3D-cCXuV`&j*IS-gNn50sNZZAn?i!t7eP*xNvz9
zt6)=oExSQyQ*6}iCPGVFvN{$G7tSlh46`n!y9|vX5>pb8^RcqbVAcF|e6GLSptlR9
zB5egJ&Rp#Q&(y+90%%mE-Em$&d7_|Xk+|J(KyHMVFy~Kn3`NNw29pG5!1EZCM@dIZ
zO}G21mCPENZ<_NQd3j0R_U|suORAc*;ijrOFL(Iv79BrfVMN_dV~*`fR?X=Q!1VV>
zfl4~gTrHcR;%fel7DO54$%Xp{u2L{OMKo?Fd%9gnbm}X_l-b(90~tTCB-S^z*8dFi
zlD+H(WtJ8R>V8<MNjWC%_V0O=nKyPZe|s=H*A<co@E!J^FgRlvB*I}y<h<sIS7M7e
z9sCXjjjr_@_?}-U92n$YLdgB@78Lz69UH{FKE_7`?(=Et)$=A1S69wNfa48bmNSD{
zJO*+C_@N`-Ud^gpLb`Wt5kguT?-^r-aDC@tgod-r8VDm0(h}UNLY1zeXI4@18}qX|
zGWdi?`4go+lPdCGk!2mm`3V{LA2p{95ed>giG+q;Su)Z~bgd)AA8rE&NYK11%gH)Y
zfU4167y0%QX#ZYyN-@&ZQ^iqM$_!d<0tQ@3fLAlR{B8r$+nIKyHi425G4l)teW<3-
z3=CY;s+9o#RnwAeHqTC8A`gEHJFKRT6hC^{_}_oT=*!gt34x}}k)9y8;yaxHk5m@V
zRb%3DM|g44{e5N;aYYgk))Hl!trVcEbtyR+cpyRx;?r_@?=IUsIdAFlV!Eu)67Gp1
z*%$@8(cjy{h}^;!c=Yl4*FRhz^vY{=DqNO;xf?$a>c^P7DO+k;s|wXBFgY1>`#QXR
zJFXPGkXEtmo(>~S$&XH8VZz_@D7qycT(rd-6+uW+SX~f6CC(Etn#{3WfUM0#^QfHw
zTp=1USmYQoTy6Td0{%-<;F(a@zB{2HqJ61YL;UE4!J@>JW;s+cjiyZjSIGKjGJ7a&
zoUujxv!`K4&o2XiLr7T2A$m_k0H-?w$bH}sev;SQZ)eHWQrr@IZ1H7X7kREex+bg#
zcBjp5Jg9WneE!E)l@?zMLgV}_XHWZ3XW`RX)q~xbDe-c^v_vH50pqmPr6V5{ZA;Jy
zscdv@_T)H!L&R>2erqwzPPf!332eByD#QfVFRoF1p3~IJ+UR?=2Q!oU%<VmSP_!sm
zNI4L4)U@39rA73h-;82ncU!0IO~ii4;;;N63#j5=YzRBYhQ!)@<NtzOBt_cEz7g_^
z8GgaA>APn0fv^QNQjsHg+pp|?Ufhc@nec3()_SE|4uA)z9n^-U4AqJbp;PkOT1Rf*
zjRdmI{r%@(BMV;jk?9wS`kwFA(1QyZM5x!QSH#+z!E=L?m!7bu1I!MRp;t+OnA51X
z*_@V@Tz?$gkMEP>-|6IF1e&xlx<)<7cj_FO@$<J?z-Z~a)g%yuvzP7U=-oPQC_`4o
z?~-a431XbK{qPy`)H`mziOmVRF}VKwu_=ZuFhU(LrWi0xWDDGi{k4+|Ty%l!{mS&W
z1ymWd+48v2Qd?itesg9Uscs87iiZJ>x}De1Px!;H%19b_g2(?c4!?cZjAk(O68Xv7
z;bL^}jPEv-I@71@2Gm+O1zEW@w5>ANOB$vHbC#$c7EIj#lFbf8eC*ESm2a&F)aGaP
z%+M!(Yrfx3D?w*B=bi{!KLs$xJo(PKo!{JQvTMpYdWbjp+p*?0-PTc@uRQ*WsEi=o
za1HY33<qyhTP&)O9Oo$8reXV-6zG@LBu|L%u3z)8R$ys=F%bT^;w`W^;hRx5o;_i!
z?2SXEQo%$^&N7UtyS4q{Y1`_Wt2%rPso_hVVxq5uIR;&Cg{B;^^s5k|RtQv!ux)Mr
z7q(`il;4<D&URdWlItTsS7im8c{K0=n3`jP+hk9bn`X!H2x#U;C(cbFw1{Ok2i>!f
znC`a4k}k(skfyuS7TpTN>tS!kgN10Q{Bs~PKeEhS+`__!>a5TAMSIemt*))F>t>Ke
zFab>BQWN4E?t{;K%q0f$@M>>KlGrPKsSpb5AOfi4IcRc3u$w&rXdl0|`KzMSqelS$
zm|zpg9$d6Z=v8z?9-`LJ-`OB~Tw+ulKTq;GIOOBQW^6`7&!ZG=Qc8Visk`IE4-7LP
zWMUzQqU7I(fr3U%n@*uK@o~QHEwQpUi-w$V85lL8B>3(__*XoGk4qKI--dxOl_VfE
z&7mjl1?yiwev6pNX467&^mD2;<x)^SvDE|ee9p2!u@J%dmdGR9fehcvu{L6!+!CWo
zmJZm2_diUI=Caufw5DuDD#<UXKZXGbrktMwn8rsKw4$3@@{Ta=wZgAm*K`vg=y>XG
zIwj6jFhffm1e$sQoV2Gb0f3d=xf~`LJ`Ov~kTuaRpDS8Eu=Idd=xeXBGwwMtzvx%=
z#MMaB1w2q`v=9Tx4IoU<R3bIN?+;bjop(+S%6~EsF?urbiR4Dw=Qb&MigL)CQp<Y2
z%JWa+))X6BPG_3MowQuCY#77*+@l0I0{h3Sh0$dyZlB5~wlN4@5^L`m=luW!X9qpP
z94h_e8@H>;RPu^qwlO}(Z<8f*<nxkz*OEH~S|G)^cA%z8sju{NgC)O_a%R}^l!3#B
zAzqM7%>bYz1ua9~u!4<1(0FB+T=uDJT8BWIs+wUlK%@@96pZ3rEs|+Uh4Y2EnzMY|
zK-RF<6X2k~G4@9rQ5K-g6cH}*^X4_Zh`q8}Df`({du~`F6XJT_9CiyNX_lta{{|eA
zEBM`#u=H%!;xvdZ?y^J;{^y5QOgVY7n+P|)Dc$n>G?J<ixE@y&d9o|Je-3)eut>p*
zX5~VozJ^)h@ZHHShnLm3mQkdp$Sr$el9Leb6fTFzza8%#a^4uF!><gn*$YEf26Q&U
zQUzB*sGDFLg;p0h5;m5B`GPOM$6RF}!7*Vf&4B)W>Z$37i4l~LL$#;c^ZZ^pxcs*4
zGD|!bo==+ZNL-g_-c0d9!M0Lmh{YRQxnQkmj9u~mL*$l<(xl)|QKz$ikm($;clxbA
zG|9Q)GY>2$2JUK4z$l_(a>0^367M1+6h%Truni2Mjzh%8Oq|t0uc9uX!>NMiJaOMM
zlkNqR6Yj)_J#*_|r*q&E4b+Q*R&!{*-<r1}`cRobbjba6lkB{v@a`-Y1!_NC)6xy4
zpw0eRK!#j$;3iRH>U!|&Q_y!znoUiwK3Oo?D|oK61_oA4Z)^KvW}BtwrGbp#$l{Ob
zQ0(K)M}D8)+)x<!1BIBF?G<9lc<wCJ*InG^(;>(BP1Yl-`tV!7+EH(&fn|BXs&qD7
zV+dDy+7WJtpZfK5twspT+wH(wDl!o)jY^|m<(*jkK;Mt~I?-#t!<L`yJgFZx2Rb>d
zQNDwLWa(j&NHRFXcL(t}FR?sy_)`a-!y>IaTn<!t?)a0_tT-Y#+@F8KHegD0wPFk$
z#hUp4OItM~dii8c_uHH~bxLiOl_xCi&u}|tck_4tY<vdwk>H~k5hqFw<gX=^+Z6&^
zvB>SWT_^pdB)rWeFI+rUFeI=Xf0V-pyeC2f`)ertH9lzLBFj`po!g8G?t^}q=Y{C2
z9-7bW2z1i;$f#wXIGGmtA<iqd6~XR_&g}T&y^Ks9SfEEGTK1oNxw#<#b-p(LUDy+y
z+C^Nuo^+|&7@?FU=N+@aE2iA0FHr(G<QjQyoBmATJZ0;#e!lUIFB^XIPie1yxrMLv
zz!-YQB=w2?#Hx|ZOPkG#5z*fwRMU&0f%WXGu_J+zGKh&4ODNH5&dY?Ys@P=jR;nQ0
zn%^>@fApIuC|$3mFqE@xo`UxNTxv<aAWJ1p*`Q?gSCc`fdjB=-9L66?n#W((@6qnU
z{V8%eF3PCC`3`<)@oAYd`;qm=t>i)Tcetq-*WP^7eo29SP{{E8uKE1+uG^?s#;PWq
z^Ohs>IeTtIR;LEa1!`Ts7P1;)&y2hyojKZ-uk*VjZ?n{n?lm0wS`q6a*#p?rVfDxZ
zLa$WEg_d+7w^1KPs=!qwmq&a=XE&R`H+8yNLW&7cxh5_@n0=9_OA^gupp4QxYT-m2
zJo$k%I+*_tlL&6_hYH4Jl*joGt;j1(25CuaIvhVAQRQJt!%(aScH#Kfo94=y9G4Z0
z(<E}n@Q49}F40>#fo^#l$_x(%31@*tsL54kC)OVq@;FOG%vEA6IJ6DmU2hT&fH^Q?
zQyn8SFQAgAF*#rNS>guFg*k6I^(DXrs2Qn*aWEnOgMfd>WN+Avwz|B!7`Yg;>ovmp
z4E|5pll4YS7%B|-J{H6$&Wx00n0DBrKdv$QTJ6|1`FhBX2ggU~2LH^C!tc~m{%=YO
z|DKk@r^CKm7_<%!VvO`(diwvRrT<G!0q^N4;9J+#Uk6%|0YCL!`k8C!W;*h)%&}4W
zj1L{Dv6bV`NOD3r?kyb0!mZG_Pw<Xyu(7>pTah)6G=xQnn)K*ur_T1_{3g7WG%3a9
zD_^NxHPntd=kId+F!BLwBY%7K@?r`{$8#l2#*^-ZlH_x-3qn4d8QZDrWxQELdA*!=
zc6pY~Z2Hbs$Y*1%WD-p}{tZkCnR#?hdFigyuqu~nxtpSCHPA4BT-v;L&@BIZg-$>~
zfR9h`g|p4C9}H`|Ph47!uqK;3x9+>$%rxiSZM73$&Y+eHxg%IP?6}1!D|J4fXPRRc
z@ho3Yt>q0ggihshfsVg*Or|FT+p;1cBy`odb~;CLRPyt&S6?6g_F(6XJQXxGe00Og
z7PI;KyWcAHX9<y$y(8c=5xgOM_(*rpgRB5#s%%dbZ&;Nw6R0d~TBfzC^%|m@tp>1I
z?C$AdHW_-bQEOxh_-eW=_9ozU^+>iOD!KBb{F`9C#cjL1(N^2@Nrh;R2pqrfk5iED
ztVAX{&zc2o*i|F$p$7c<%a5(vmr8JYMfL5SgpJeNK=7(ZAoN5mO}YF{7wQGQ6w6yz
zdByZ<|H01x4G0r^X8eZF?0NBHl@ImDH<J71eod%RJVN^FPw+k=PXx&YQ-|eq7T9pM
zclnx1_`{l7VYP7KIK_o!6xg^$UKlZw2KGU(E;!2V^=GNJoInJnvF(atVSjtPRVO!V
z%V~qfbZ-Q#DJWD13rfso_1D~*EJr@^<6pk{gQnU-Iq8)r+{kGVFkc^%vCfe6t1=}%
za0}^_EgWlhJT$az%)CYWUHYE5d>)gep&HQf{B9((TtpkAc&z}>e(dO_y=&Ua79JEb
z+RUxE%&tT-iOG^tL}#QeMRhj`pNl@V$Z+Et>!Ws<t}S_74K_zM07w^x;1f;tDF_RC
zBK=JrR(1sYv`7H`qbQL=dg!OG>fWA?GyZJvFd=cJw?D{|)0V{pP=j~lM03pAEqMbt
zy3M{N8E)TG9ra*PjK@eBfq#^T_Tq%t(RjP}9px)CJ{TE&uv9P_Ys)FGSFu(9HxvD2
zJc{v=J+Azl%^s#eP;wS?@cIb(c)W<_o_Fs97Y{bIp5a&)pinut%uXvHU`oq>ZYC)P
zX-%SUU4g6jS<|H0jGs>SEUEoTWqtK-y2pD^imB6Hk;=ydP;|pCh2hC#@8O%@Yr+Bz
zt(x3Yeh?`xj=wDXm_4RmSsL&E<e5?JG%fcL&_L8k5d+MFbcA)f{q8npS6FeW>zoIx
zKJ@UtY)>_pYO3quH_Zz3r9S*HEHB0gBkP*oJ-b{=bzT2GFdnM{@Q(j9>)Jm`zl!{?
z;e}&*8lETq%eKt09~V!HNWVeG*s0(#RyGEx`|l20nX2Pd7f;{u888oAPRZ{}3lPB4
zxU)LwMH>Sfj_#Ts`9=z1lL|<L9kzSUK6)0$cYcv%*P=h+anRNy+~H&5slX}D!hBgY
z3I%R(cQ-7MV(1TjDXH-rj+M07$~A`LHq~He0+q!6;eu#uGKg3DH{oG4*HIO>3GL?v
zlJz=5vq&soj)GKpn|JQ_;ql9P+F*WQfV^z!LBD)+8uZw;HClwQuPBS+u~6;sK(smF
zm37G#-4Aw-IGwM&p~?!`{oF`=%zq<8l%Ty=wvO55%)g30?vQEhO_1+TkVG=sTD!Wv
zalfkq{K5wO(mmX6qTTXCP0rs?*0;9Em=4U=>g-eHTgt-W8!avV%(}}IZ{>gDb=2Su
zFq5ZfD9$7KVjbuM)g@VLo#Y)0u=n~bP{ni%#YBQbWQZHLsc}FaP6=9?O#qhiP~VZg
z>2RI|s~T1aIM#R(6|0zKf;6MEkH{$y)!Q;0v~4T=$jx-L<Y3o6gwiieU{YufZY}$X
zzS9@OwAJLMwOXWC8c6Of!L<>sTW;;Q<;r7m5V&B{3iV#fT;>jEd2#n+*wznHmnc(7
zIoMZ{5KOu@S*AehZR1OAp!)`FNI~{SNfP5v;ZPc8n#ffJ2*6FfF57h-$Ghk9NASb}
zg^T9ued2e4pcTNbCbaGFYqwtS`5qH(?0(6(2nYMPV>%etMypSY*00%*1xZ)A0waQC
zOG2MRb)#v(y%0jd@~!KP(B!0J$+AuBg^_*u9!=YYNF5+1$3Yjlth#3L?<U7sNdk($
ztM%BY!iaEjr0>C5E|;60zn}bxv8xe<fDY|Qo-8JqAgF0z5>XI4j)!c<SL1|@)U{)>
zjf2Aw&&ohef{8DNN5->INl8N3u=#Zb_=Q!R)0nm(Q)*hPkr{!g^tL4N&k!(6`6_LM
zmbR2qPw3^((eS#;G}=)NZ~C2fpYd9br0Q?R&u$;!?nac8eBCpN2HqD2^Eqz9atyMP
z_-tZ54LKIWRL11}3Ka~ZLY0b>vw#1%Ow()i;LY(s(ROup%D?zFA@HCLBk+Ruc*|G!
z`0EYqSVqaB&-|BU_oNYYA?T^kMdTw(JsGKiS83y^Cc$^`XmLXHlc{u+cp;wQjL73s
z(!q9!2>~Ok{kI4=T`^B)%yty3cpA=eWY}qeutk2h2bwvxQmN|06<?gXpHi-yzB14I
zCbYPzHtx*zF<5-C&z7om=*z{yQN1jpku3gnLCRE}?JH4Sj+6x%gwNErdFH-PSwxMj
zFdr*yfo3r;fe?VKrr89A7=uY4v(Jc`K`jRuAb3RPk_G3^5$|jx5Z&@yT-^KprRL?t
zC|%X={mDVB&yQf;&gYn{n`;)y$FK5rRu9abmMkjSYKBWbJc2`-Tyc0H3nEyqV`5Kj
zy*D%}R$S0z4AJ7V#-<m6;z~2=sPOywZ{D(soU5SXcQ#r(Y`#o0mlR}n?R0dy^39u5
z-#EeeV)TUmSZ2vol=#z8)3I9%e1Q>FmNi_-y3Th^nPt{?dP@evf~qDe{0GoJFJ-Er
z=`}wQbXR4l8E(Lo??j((cTOZPC6OQ*QIArJrS4f{WG#pd)IEv<c0*R$Wk`{Vm@rD5
znu`@pweJ`TrMxksEnpe?ATWeU)L{t*p$M>?cX6IR_a3-OZ_*$k@dlliGP`!S%PwnY
z@tYeWz3p{KIIFhd!p}DV>j1m9AzzJ?KC(`<@xc^PdON7Glad8}?PcppN%_^7u*VWK
z;E1u-VD<Fl=sqLroO^a^&Luq|EFhk(%o$xm*4Fr*Y<U#<{|Uoj-b1la(@X#d&m3NS
zxF#1G%*h-#E({C`JonkZ`nqW1UN~cxtqO+pM}z9`UpASv%i<w<-&sJLg-QU>cd%yp
zccpBBN-eo;77)1VyB7brQD5S52i{3IBKq2Be+x`$TohhuC>|<mDVxDR&8s|FrTZer
zHR_i(e|Ve8l(Q?EIbcyor^(0ExN{h)%P;m3g3t-Uj|4i(vV<_sGbsaHdIgI?@??`G
z6tPSe_s>X)v5PP`YW{|T@cZfV)j>@7=0p0$k>btP4goOe$AQaHY;KynqvkL;{`&sv
zIucvbg(`*4h&Hm!(HAJ|;Pf<@I+crpqcCco4zS*GGiAnn#%V<*UnnrTOZND!rHWuS
zFh!CKf`nHLCyDEDkD5+RL?I8G;VuxmK8qze+5VX9)EEDTAdEJLd<N@TpFZ8m=xioZ
z_lQ}&4jjw>{MkzsIsj|04Re@e;v*!PC0kVbgfW|iP=@_HAIB8OB@*{bfo>1%p3Lwz
z0*ZHw_i%!@r#VK@aD{&>rd`dziTUJLg@gpvNAhW%o|cpG;U)SM_fah4Ru?W&6w!gS
z>8IodAr9sb%xSOg*kzFnKoo|YHDx&2r9z*1f&>GSa@rQ}JntxoG|Ea0@~Us5$IRQh
z*~frqYcji|J>L~={EG|T%PWr8m07Q!1l^@{yg4)EU<;x!;54ryAC?c>Zn^#=Oy3Ce
z^S%iZ^bjgH6aq&cC8RET43*UL0#C<D-uzysoz-68G~CZPq7FC!o%n%=4XqYKl*bb(
z+vA0qirWq991ep*WB?Huslr8ESB(4j#cJDv%75Y`>mFY4`MAXQFDecRL$F%HLZVck
zMaj&*hlP1jjIn;A@@UiAU6}t8sr(m5p+B|hrwl!tHF2Nvcr=FoiOlqr#?y(kVKK{j
z3r;iCSH=_HgmBbm@$Jjlh}A~|6Xi9sPFS)_nfYZ$GRzN&vTJ2+pGmsClrXhe$OuAj
zwf7|lNax{3;_Q*^hi<N=H|(*8JrDoE>up2h-(aBq7*dhk-No&A@<RtdcEA5_gtE_3
z1sp_)*nZBnu*dvdgvwDuQV~0CcSoEo8St^nhB9g!8@-qcBvIae^7ot1bY*P(NF(NR
zfkk=eiH~(=lnm+|J~_XM5i<2<e#0IczUdI4qvmVhMu=aeGER4Ul%Jz(@mSVP&--O0
z+%}rHEp}On4Uk@Gcex(hp{)4?cNTTRA(Kg$S$6ai$DDyCCm-*wzin6O-6v7IpDgye
zoLLu?l7?9)5VhFKa&Y74)3)Q*8jrCLpsAmqhS`$7ei(*hR`oFa!ax&0!A3P|VVV5V
z@;ZQ;v&Y*|mR(1h2oyf%Uu9IaI?6$N3scKR*sL{D-N1DMTqvYr6B^^q+`!At+M&j2
z(}<FZ0-Fx;sd@pLUhhm88ZiBnDFSaydsTWeCQ@p)!W{>lam$xM36R<)*13PPAbKot
zQKap<WW8D4?Ypnshd{G%2-gb8(CY#zx)Bgp9X!R-Dms#H*=C&~|3$j$On~F6<Vxe>
zwpADM8-aQcrjYQOHS?kIs<O$**|*JzGX^8tR#4p|o3dgC6s#kAy$|E==VMs0hp0Sx
ziA8V%yw|*wiOw~RW)~Vynmz%#p8fV+(a=Qa)+sxfGtDTbVHGas*9|ojYd_K0*}H-A
z{`6WIa%4%C;&C<T#ISuQ0h)m09PbMS@f$8UN~i{J>ko!}#5VD_l(u8vcSNyQcsSR&
z$E>S+E7MFE61dk;=r6xZ3=r};<H%Wg(Dis`a}BYHeizy``dWL<;Z%qo^C*+aJ^k4$
zn;+|zaFPk(J7eYyEOSQgKKXac%!jBCG+zN2p6s8FBAL*lp0?ioeLFK^s9Vw*afOWW
zs#xwuocNSjwyJZiDXkRJm2a1geMb_gKAesxUE+{J8}=mhT!pJd8VW%GSz>8<Pb~q5
zZ0YJA$*b2dWp2mOea9~g@pj~{s}F$^Lxpn0zcq<#f87;qa5K~bS*Z8-E#RAvS3t5Z
z5~qnN7;0V~>@x&sE1+5(1CNZjDG&i(c~-#~Kg#a)0Qqjul}U?meNchZ?6SNE(x-^O
zbeT5;A}~}%wbikYqEwKv%5dpX-k&}z%raqfa5=;RU;%R$!Tk4wI%lZgr_PA2d#6Ck
zGf`g5)>o_U3vTanVR#GwAC$DV1qYDyUKSb?uN(Vv{qC?2pbtp&f^Vy!CPT^EFxC5+
zmvDxptj00>FvsFCEnXdq?XR;_99O^Ii7A`OUKoRsZT8C|S{N_6FM(tCfT1Ja@Xb0?
zH)}>UHu#>hWXQV`x-B{iiYvt@Db|ZqCd!%@l<2JaA^AUN!{|QU=>T0zTan17Of8dw
z>#T+(yZ~=W?q$F7{{LB&$eskTGkd4!FkR*k##sMAbG;>`iPhe-etONxml=*ayom(I
zEcDu)n6iyrmqy6h$BlUNB_-r}a#>$Ig>$635|?lrQ|a6t#Zhby>%s(VX~b-~OI3o)
z8&n!@Gy*ZDHGkP-Pc}E}4BDlq9*s4$O5wMz0L*}LKAo0N)83~I*kRE>3!)eUlJ+_F
z1h3cEOYcB<gF!YgNv17?3T$Y_7eR(KO1rxvL(AX6zuWa+SBqDa_T&a!hco5!|8cIu
z($7sBs>YcT+&-(%`}xBLu~7-clV<+m*z&6r&`_jlAO+a1+fB!Cb?ID*k)tnr2AY?D
zD!z$TrNz@fg;nlcS-wQp<4k%VF6{qMh^g;0V?2E^xZK1pya>}TL0M8j=Ox_U&;KQs
zTQ-2%M_`NfbhL?0u3?<s4ATJrk6gSO#zOF@Y;5R-dUrD^RX73jG*`=C=7Tb+x~O$_
zg5jje`BN8VVkO~?#9eUBsO5d*{Dl*{z+H70<ycSCoppV>vk}cIzt-}#X1dBjK`~Y>
z^n*sq&{$)G^y1z2fbBYRktPt6{KJ0M|0(M$pyGzQbq6W#Rw!1q1Et7dEmGXw87R&`
zad#<NTndA`4G!(#?lw>yibE-`1&Z6Feg9kczI(HhoSl=blf84~%gL6URk9R(p5msu
z0I>bi{<4A=yUlKA9qKIj;y(gDe!J`e=2bg^Ou314l-{*lmayJ=uAY2MgFLSN9oa(;
zZbS|i=#T&a9OSNtbJ>trfIYIc=@pZKMwvRZ=sM1%8*we^qOuWwRV^5(459fy=n7L9
z6Zv1Sz7H0SIOEUs{=Pd&BLx_onP_Q6<a;iD3?0b<P?x=A(i}{Esfy7kN?SGtR`<`l
z)SM6$8hbZUhyB|EC2Tyc&;IayKNYMj@0$<Po+*3ikTPMpZL26N8d0~<s(eadU1fR*
zHi-6&x&=#3+23T!SnCZW8B}h~QUe8RJtg{TTKI}VzOC(1Of#FtUoD`&6z>eQI*7=}
zey!(9dLrk(OT_PNv!1TYeYe2Mv5aXpw*0f(3$N6a=G(`IKq1@kkD>KFn^DX#$>M51
zru?(PdEP```>)9pXeweEC&Yjes|md3V|`*m7UPt$)gm^CRXTjYTFzNA%+mK$%kmuu
z@HL$H>zYHEfd3ogS+(}%{lM+l6ipa&5LSr*VhJt&S+y-<yIbOW8d(QHozeI`g99DQ
z6RI2#qwBumzK&9+2Nd^m&otNb!oAu8-j6#gS3_lNrfR_q0MhH#1|Qw7_*kX0e9Z4@
zff|qm7(H{Oih}!#-n3NV-6_p{C@42YMNA{<PQ7>_1X|(R>IQ4iseLB4o5<R^6xzZ1
z$W>UT6&SRSubQkrW`9u@p)cTfyk7#1$ziLRF?&k{u|Nh?PUlm*u3k<cMG68Q_#o)j
zv3pn0x?M)<Z@;&ubyVzOx=<bTvuwEAVE(l3Rirgu*hX5`QficARCqWIz&ZZ|<|AfB
zA;AN+OoC+>e;`rpZK(A_2}yA{9zb`+J6cq+Be!DJMa81iA|9@b9Dh;_R3JIq3fS_Q
z7^N9w&<N}1nPiXwk+w>L`DuAz?;~VQ1~ru}ho<>{eZar8X14J~fS!Hz8*C*4UYv*e
zubO{bTk?7lV}pexHII+ksa!bmi6zm8=lngyvM6=QFA{Vhdju2T@fk6&1sm>dVs>~V
zIDd}bbSn3E=V;ml(!>1LXs&!&nX=n`$#^L@j%R(5WIwr`Q<0p2KJ|~ZD1|;SH(?v*
z#zv1<4%ioZ1Vbr&%CPR~v{tJheZ%4Vr(YxI@zOJQgOD{heWpHqP-MVn2{MZHY=2@u
zr+<2V=(!3`+Mcv;+gb04f8Kd+Sz5&jOO$ZHi7REODD$b!4yj*av%t^P7(u$V(}OV7
z&J!+C@CK%n`(rprMBasIpxAbZJOhPT9?1yFhL!6z(pS`)Y|suCYZ3)^hmO(&#sLG*
zMWyQxzeHvk<l<M3;Q=-1P*)1YyIP+vXHjw}b*pJA$P6FwUHMZE$Ot)fK_V)@kBX@h
zAoD_+k6N*Jix;tQ?Fu_Rz}9Hv{+cIa_v5Q|&J&BTx5QBtQJ!m0Pjsk_^AGXHW7M3y
zt!Q|N52*CSQR}}Kt?8jvyO=|Iw=7d^Ym~fMQN(r;fc`yDiILm@2VrU&Ju$C))wZ|c
zfq*UPL$9*n`VzA}+qR|YKRo0qX~MLebhwNkIDEoctP!YY&x7H6q7CSCaMVvZ@Q<q{
zHnCf8ki8%qlibm^g1;}S)n-SFxsxn|VSLa{9+YWQRku6yG}>CrhHJy*$qi*O;g+f)
zgH4oz;t5ZM`AbZoPz)WSthaCC7Ft&ao3X4S$3-BcLK}^lAx@eP`hsRZ<I02~sf;qo
zxT<8I@u>_8MsOvbb9SU0+FXI7;cPi*keNVxd>UW;ySigYUk`O|1s8p>7C4!|XD}Jg
z`DF9%{ywWzh!S+pgcvc8xjpf74m<`fYDTac=8Y(d>W%qQP(bU7`e@ad-IzW-VnEKK
z-kpUaA1Ne)mX{9wQ{apx&<6$ZH^qQ68|b^e*z?-9fktbgl_bc!Vsn1azPaNxFaN-`
zlZGQMGB&HD`ae3+>@OkIJN*qv2n%|jQ25>azvm$VYZsrt8UitlYs?yOxl2X}18PnL
zyq=A<t8^CTYh<Y~LsCyz9Ah8C;oSK3Ur@0`TnD&&DnHFfGPy%@9fh7V70K!*cn{-r
z>`X2Tc{+b0y~20-=^K2AW@TEoK6H7Wt(P0l@Z7O=U*3D>VA}(L@o{+&_|6F9eW}he
zP{!%clJe|EU{S;3#I8R_umv~35oKXyd^$4dL9(Ox_sosEMoPzpPg->P>UI^ha?}>R
zO~{a2e!=Bz{~do!$XLS67h%p~xQm;oo(=2bQzP_t*8kgAoC2}$yA`vd=w_S}dGExA
zdE%$=(7&iiRPSU6g_{>W=X{YfwYB%v^TH1q09|VO5o~|Uj>pvMm=smfZD-UfH_x`!
z-BDmJIea}g=0BJaUaL}AB?)bm$wTf_p6TKywBZYh7F2QhN#8Brh1uaLok#YT2-BFg
zDgM<O$u}xdWywW(v_R8X*nw>C4@uWAl4Xg!La`;R#Xc9o#1lxJ$>Dnbv674}#P`|M
zGrW=%K`KJG-vsMDN{RdO0fXJ41cwL5?Z(b_FEEX6#5ZLlk{2Hi@m0!gpN!_xT^`6m
zNyOLb;mXm26zbI6V2kq1WL{AZdOh1R*rMRIU*ZfBN9e>`vP_$}Nm?QRCw(!6>r{gR
zrm}iTTNG~H7vACb10S{gR^Ja?si`TXX=Ta2tAYBdPe0+KaQ?VDpYC5Md_qJ%-2Y%?
zGU(~5y0T<Y2HJlxQX#Eqq)bmsVP51QYd^9;o+R{#wrDOpvfc)=S`gL?5@bTu7r*mP
zGqJP{cQOkny)(_i6wNNH7jVAEf75pOv5+%OE_RzNT5i2c6rKb8TRIesuz>u-XXW63
zNB<46|5uJ}wXSbx8$Cj2Pv2ojjg)LZ7{t@tQO<gd{M!2<nSZT^u=^c-zRV_&D0%v-
z0^9Nw=MV046LgI>kFG|0wR4%RJjQ$!#mx+UrIGM@qg#2&T|`@z2zUy^Hi0NO*SSbp
zsr6*mobR*n=SDOL;WheD9KaR+0&+$0-M#_4BUkZO|K;|orduIlsur+g*+!NGiG>4e
z^X0;UdrOd%WipNPg_Tp+g&`G6`yf4&i9A8izq<k@6a;-|cMXW6o<sA9mEuCz9~yiN
zKEYvxbB4OuU(bQVy1PS?86*8)2MBa>*>pOO^4xwtSzjrNrs%L--({4|KQr4U1@<J<
z|EWN==5wV8XcVjBA>iR;dfi#{+7um__PI2{<U>R5jRtgv=S~<xXlHcW%nI38nQfh6
zHB5pxA`58$EnX>nUv-vIROZ=ugtq}gF~wC*x3xT#HDE<YvF;MTCIW8Lnc(fdCRj$n
z8cA7ys6n@mfl0UwsFkcF`U#*aBrpk!U)KzWYv=D7sns*#H>OqU_Q#blIzF1nUjuZb
zUOv;?dxhwC>reuh>#8rY*o1h(IYSh$z3t6Klg!Kfqu^Y$<QnMo>bjsL4c4?s<;<Mq
zzt3T-@`lQ@^{nriYfoCMl9KqgxMokqNdj^}Vh5Ww&&<$Mg|sn4Xohc~pM<2X{Sf17
zz84{lIC#f|-$sP(>UWqCjvF&GmW%U&4kZ@jEWA?3XWfZhpE+r2=P5#$leR^vXz~k#
z-1+9$z8Y^woRUINX2#M0K}51B0A+y2x~YF=qaZ3B^Iy&t!L=v&EP}rzlTt)JwcxZW
z9&AfnvqX*(_UJ=nuE{mdJI|05U>BGj*(ikWKF-gzeq>j8l`~WK*8sU%JUQ_GOmE0d
zUou<9=pc=sRRa}X0OagJls$%tpv>?PbjkI4X2{kdTjv++mztePAE*pmlCDyL5-i@L
zSr+K++Lb1#FV2?)sg!@kPAF<H1s(#fus$Mp774%X(Sg$8HFxE$b5YCW=K9f|h_Fm0
z04bOeoK?3FL^t_3>ww(HPRHX(B*%>rS(jNEwRV|#?wkqDqfaK*;{c{*fLgn17PE~3
z!j+EQVxZCTW?~~YvQ*9=`NDLzvB}V14$R)#UbuzMQZBVybXO5ftX)*$WN>+vJCgY;
z1}j$)wZK_AO(L30e*1~f`$9IPwUd_~Cq@YOEp-p<cZ-e^V41p~C(j^@0emw(Q(I2C
zW3tb<g!<>)dEV=|kqo>D`ZlS`S9NQENLb1|i|EOuA76=CV&rDPg<zy#0)4i)(-HSl
z8n}=*e!je1M>mPQ9pkXJyb4&syv*<6IdjXEYtYCG6(e<boQT4~Px0_hq}<b+>-MN5
zkLa@z?1f9?PpZj{kJ^`bki<PnidefrWo+oPF5CwrRWw%vG4_B?MahmSr*QlqZntBV
zH%?RO4<CL|<3N5j8|5jk-aJD%Lx6&fRW163Y<^D@*Y^dbOGcpcH;CDsie2%R!$2Kq
z(q+WtCou}g*7J(HuE&&TECGe@Rs=P6-L(+;_d;hMkd`f!=cRA}zYP!z$iMp@S*9|C
z;~z*NqNBXI;9IM<#*rf~8Z<=oT%#G)k9#Z#tYI?q$Yo#-MIE_w=Bi$FKJ0Cq1Xxh$
zj6!MiA))kGwf@2UyNq!@PWJ)%tNf5%D`{{Q;7vOW2e>29GT^D31piQLZ(sZ2$n~|_
ztVsL3Jkw}xSXBZIJ7lh=9*^&pYb~ve3&E*s$o6%hn%AR7gNqgp{ig~qj2)g`q*uT3
zoB%v!J0tFUSci<l&}iqbxVVnvl2E*0e`uDEZ!YqVp2wJPG}@2sSJ#}6?Kn>AR4^5e
zgSV9^WST!<IG_kDEhIwWV}IMS9m%Z_Q&KM_K7OKvLwr&}26_)yw|AAVo5p;lHBcya
znKW*DCxE3rU--S{+^pcUD~#W;$0xm7Z@Kt(qb?kWxNL;-L1nr(Cw-^~)u9Q@H7o~y
zcJFnA;qSNExH^&uwf5iN(izaRG7(;ee_YiOC(X2r*qJ2Y;*LQg9_19_)EOi#yNa?T
zp+P470uW*eSwhzD1GyY0=eOWs^_3;u#fKZ^ELcPD&u+~U9EmNE*-B6Xq`%HR;}oWH
zL>bK^_=c{L8GdI)O{S?+T!9_xhz7Xb&boW~JbxnRvA<A|pGf_4P{Uw2--PSUu+=}I
zxamd}V9xo!#U=HnH`vOb@B^kLma+7~GUD`frPdxbv7uCM=X1@=o(iXp4<C@rQeKG;
zJdJ4FHCzXsUauu@sS31`^88{?WTf6ew9m<h@fGG(BJ~gz*5V|(|K_Jp)6@eVU9JC%
zwf{Tk34Jx)Up&F&0j>L?Zk^proHftS97r|?@p8ex`t*~e%Eid@=w8N)y;v8ch&7Mt
zB2Z1veZDikL4gCdwLDee@r|2qDPK-tl~7RK{r!mh{%uabh{yYVCCCB_;PDd#kc5;A
z3P1o@QAGbIt%>$8zobG3@GxVD=4gugzb|o5i#V7K{30|%1JLA4_-<>$T<L#U^i*I_
zS%lU=BE%sHNpou*>sUh{OqZxs;cYO_<u_QUSe&@|cAl20K+$;$hZwxKq!cP$C>JdM
zg1hNC3wqL^ydXe1`pZUL_2A%EX?#+)#q|xlS1dzA$B?!Rq<l6s$9`QlxlAe>O$y8b
zA9!0~UK3jkpUm@<6$G$ONio6YW^%|m;FZdD2HyJh_+Y>EG-md673-J-S7U!IK1J)9
zau9Cx?E{@p>~bVt03ws4K`y%);mdXg(n`@YrYShB%G9zCI~8jyWg{xK^@G@N&25yr
zBWb$-DBIIa&O*3CpMSTyNMM^u^I$Ev)NZY2OSrWyHvlAs&dl<|Z@}{(KbcNDJsDo`
z*RyY1Pu?K<;ipM4iF#Rn5ED1E=dI{#vEO{t-mrR{o^=do|LQpm`<%8oUKlY$!TkLO
zWOeYPrFgivl4HMk(T?gyL0-7Y8N204vaf@Z?slpX&>s<lW_Q1sjqYP1zshFL_^-8y
zTb%fxXJil#=^VYwnYV2D&4JW9;OTtzCzx8|?AD)jC%fbKBQT&L^nUlZLEi1Yq6kJV
z{I{dPK_sKYxeC(%tLX#Z+<PR>w;}>QGY<eyVL^e13xRQez!m$bizS$1lpFZs0}S`y
z=Nob$DAUqdaBOHup7QAnP8V=(LkORMvL_VVwie}I&?dmGt0L-CG)&RKhu!xHbVB?)
zqDnadi=a->R~$AJHk9E2E4Tj<mk{vPMMT(+%7kj*vQ%h2zLu~JCrq{UkxEaqJOj9B
zK%rA5K@}ZOMOap1x1g!YR_TpNMWvaq$4*~wT^{>|fC!D3i3H4h$|bzf%fv5E=TI`n
zY20a;7htU^*l}#qnr=WVl73!M*FSp^`%xPsFYE9P!<?o$Mj&|nN|T2>WE^&){x=!^
zAxR;wTE9HAe6r`(_L?9FPE5a#XMg`oD{R~L>ut!$Q%==$+X4MF%Pw}Oe1afQzbidN
ze>p#m4izHAVaO7ai-RD5eoy93xVwe`Z>z;s`Px<#9yY{r=gnBpeOp)dj|DX#f#;f*
zKCBX|+9k5<g^EqyroHmz1=W=gHJa{LP~Zx<_@Ma(yP<;<1OtIcaJ_(LUwuYHea;D<
zKTzkGAEKz&WtV4F(h8M;vReX2!hDuF5=745QZ)@ImW9&J043K;rHWReT3Bx^@}tNm
zjN`{90vh0*dQw?J`3LLxIjazV{`R|~q9i&Xc51B=5B=NET*-osUW$gg4f)*B<<n<E
zJp|kQY%R4Hmi<S>&6rcl?=9WG{ER&1?O^m$6jDpPC`>ir8G6*Vu3?kFHhlmU)np@(
zx2`>l4CgQ3=NumuQNCYQacF>pP$3MUW>vcUt6COXtWlsQFw^M0>)FoSYNSsQMsOr<
zvrq^;(o~PbqSaFg^GhnWjAyLh?!*_3y;C}!5yO@9RB}j9jY>Cnw*t6R(E`{G74RM?
z3Ak=oo!0YyBcZybx&YB}YWN*MgNbWdC%d82#P(5VuowNyy_*BDxODf6*}IXjH`HqV
znPhzY+t(yHKgCR-iR*gDx%+vF3=A#R_Hk<5X68EjV2U764;@p5gj_-6oP*~#S+l0@
z@4r37b*|cg5hZG1(?8l(V&Lw=<!)XGo9&X@M^R7U)F5zW{%hi=?*sRJgL?WD-P8r9
z%p{;|RqVwLF{MAW>e|V9BF#ExG>8#84#-3uepB31I_Tjp1AiYn!*FIeydX-^eIz5K
z`)=;uvD+s~#am{T3$G&)BE`Mp5gpN_S`Rx=K8*dgCHMLl)jTn!2W8T;k;^wR6^7va
zEdLa`gN+V3NoRSLw-*~n=)#jNw9o46hheJ8F$gUDE9bUbt}X>?qe}M|4jLWnX@aW#
zmSZPLF^CmPm)E@JhphwQzctfG`*<mxL@Zu$d;q894X!A^kJ+44-i^t(lhWB5*4{}g
zA7~rC_o_>*u70TS%ocxLF{*5MU3jUS<Z*g$1MVWVpBy9PfXt59f+e&cTMw)S-*ZS@
zg(b<dLBj8@E5=od?pf*>h3m{u=c)Rr*IiyW%v{ZMY3#P@4Nn(*^s(iY{-xzMShL3t
zzhLy1gq3F__O#*kH^=BfpyRUjkNrQ*3j9~rSngXC$p?JZ$YYnoK4CKe1?}|aPo7CT
zJiS{L<BHAymL3`9pdEHB0HUKpvz}Vq%@3W9x3B=K;t#D~0AgSl?$L)s0gRtYP8<)G
z4v`WV4#H9x2)&n2J(Uh9fg$$P-#-&T>Hc$gUp|bJVuk%Qs*yj#NkMv2?c?>lB#@jy
z{zA&%85*_J*>-HyFE||GjI4l1AGF@#jV-;ip984Sqj&<y5}bc{s`_8FP(mehX!%s#
zVLhzUs`2#qnL5dL35a2whyM55k`;=18l8+m*ZrjowAZ}w>_N3CSRVW>y!zEAafA9)
zf#TGoEUjRvJ7)phq@s&SZ^JjGQAwpPORi%cxXJKrgxr*BT9fw1tf1vj%M7#{q=sDh
ziOZ8Gqcr3{>jmUE+GuL9{_W`Zpg4X2h#^`(bGeyfobNO`onOrFi#(B}8Po7n<ht6^
zk0s-1omZV$Fig5uSmSzCV193EiCmECJT0#3Akj-~>_VTZI|-ko8K;?@=9ZbxAoM}%
z?d*`ImD{H6%wy{Clm{JspD26>v9ZPu7D&&p@C6>4*=Nx-8xNGK%v81v2J|addE9XX
zsRE#g7D@-N5jQ%Sx32@DxI$egyccB<XwVZfou4-!eQM72$=Et%(k@=9MKkJCKiPMd
zER5H#yon(EUWpdUB5}31cR0$!XW^^sfQCr^z#*Vvp{Z?$?ZL&&2OCi|Jtl9wxcEkq
zVgKvTfur)2Q{G~xzRov^Ff<Qdla4v2_evDw(bQ*Q!%Zj>y7srS^8xVqQGAgUi#ZVq
z@YhUPqocC!%D1J;%4SIrqw*1&W;+B~c6@Oud;Jmg=i1=xK-{B9@<g7;f)wukpwCp+
zHn~s4zIFePmH`^x#vzyLj-AHwT?}cY&quJG^k*O&(fR2hT5LzU2EPJZQ3?ky=_@Q)
zlQDz}66I(aS0A)#BBeweF+<|~`JLm&24tvJe>OfxZOL3Za`_`;tbT>-E%b{t7M($W
zYJbJckR@Ggl!dQ09;)=4<3j0EJgR(h;-V<JH}y~o?ir`wZLnxB$zYO4hZa@POH1gX
zXxi+-)};_n2a1f1=|BvHEN{)seV2lz>Me=zR;>l<MReu)o^Z8;I+=5q<UG}jrb9{y
z3c`401KRuLLFhP~`}OF?V(WgW@r2zS$1#p^!MhW%+@<8)n_2!(ev$U2-{e~L2&GDx
z$@&hOSZxb9PmJJqUh*T%?{tw^2491zA~VZ-ag(P5^05{c^Kfk2<tfN)tK{ZhQSG9{
zQTpJ!y?S>9=~_{wNd8?0a^ZuqU9E|ivdNA-VE7teB)yVhU>t1KRL3Sjtg*{+Wqf$J
zF~c_MF_f|Mr1Q4K53p8An>h3Kr1Qd^3f!LI6dsIujyo=}PNJnFV3GFC2OkqMF`U55
zcIh4W%0azea#t^I*lra_sS4xos50<#5!;%p=2wvrLi9f)+K|R2{&RbI<hz<`8nbe4
zW-b2S>BZ|YcmQ~vcD1VWSzT$%&@f68Ek{{C^W^8=m@&LwOpL@YON0aqkIguAsFbK`
zB6}1JkT`JKx3grWSk6#=APP4t_E_+MR79X4)fp)9cB`=yS>l*uaFC=tjsFoRRt>@V
z$Nx2N%ia0F*8c8w!Wrpo?A3xs|F%JO2zD%v+|^B9+rnrkj~DZxc|}*{3~_P^ZFidV
zqif=QrOuvY+H3ZP`(j&y;gZ>KSARgcVak`bM6g6JjSDVRsT3KMKE@@<N=x%u(AU3y
zOHw<CkCbf14j$is)g^&)$it7-%e7(QaL_J$Xh@03XHJZ;$zux%|9)+bTlEEmp~#5D
zBQ9!sfz7rVE5bh68b2?RXA<Wd;H1e?ApC~CEgaHCy{wvUqmx3%X-SA(WY_5E<6%so
z$6c%G%cx((vk6w&Fpp8w4W_ToyNv6FqlYi3b^s}-AsrQO8(t>Y<=L{MD_nRj>V6K6
z=OTLM^!65u`&_r5YAJNl#DPh#dKefS1&xFV9x+*53Rpx}LU$)Q<FS`w;g$Hb@8vh>
zA1EPu#`tSf0P#8}3VooB^K~J?hw9<4j+$I-4N;KMT1-9;A%_?^C*zwTb|56<2PdoG
z`A?7GJ#dvkd*8Fvf&P^vH!yBZ-pT+xuzv?fHReQ6e5SQ3o$JMM9|Z9-#dC%p!BXpN
z(65sV#GV26JlTJ$C7E~0z!V7+ScYL79M$ZO)rw9zqO)lrDVKiTWf614f<^RqXpsoO
zmr88Mf=vSeQDSeKAlJdB^18dAU?7*!q%0Uv@{l2a5w7X>884FOtl$@Sx+VB9ZdL3!
zXOf(jVDLtCey!D3*lJ;Ug=F@_u}6cOO85<-Afp$l?{R@>+C;1G3rZ}WGTL0Oo09$#
zB_-p2W)1Bh<aXnB6_DsVC&2Ji&1bKzEZqB0j}Duk_o%Q0N9{aQ6M4Ui9eg9!eTjP>
z>~#xE1IeOW@lL%<=56vl^_u5>$8lTD?PWPeu_H?%xv`^$rR1V88^cj@!`1HO2mP()
zx+Qv_diLc6(@W>3Y-4=dTYm;$Z^p}J+uT{<uuGM84~_&$Ek5=qU{k$V;_L}ZaENJL
z!zc8X)eN!9-_R#Mk!GUZACAbx935`m?0vp%iYnP}8<3qs%0A>p(F?DElW*qoCc%+l
zs5C1Gw&87triSgl_Kw>;>F{KFx{wMHpF#54FjX+xYil=ig<Gin5&?1g*l>H$i(w4M
zFE6ll<sW5`Ci`{1)lTcCw9&QKoq-@n=&-KML>hhiV}D)Z-`m5#J_>ue<Si07t6j8u
z)tzmyeC#~#uY5dJWis-2+lI);z-LaLl_nhjl@96)s-BJ56QpHJUes3?K|AJ>|Nbf?
zL9;`Xr{k7zso>bZvo|s#VJG{!R1z(x;L|O7BaCzIo2Vi*m_9{)i!i;YWs8jO>8xgA
zzLfwsFf1G$`;$Zjh%}XJX3s_NIXsV*hNfjA^W5SX^cCJJl=QB|1y=VSvnHw+DB#oT
z-(-cd4=roXJtk!d@`;gS^l(xUD!T@~uC0Z;KO1l)K_|p^BH-uDrUcKwa3;8N5mL!I
z@@x@VeGH>gQ;sxBv-$bc@)nX|;r=F5on;_Wg}I$<<-_IJ?e2k|C&>PEdBedWtZ!3|
zUi&lR?m8coRH_J>b!#S4Yp^W_?F%HY8!~0znA9wxSB$7<&P)Z=)a<L=?^!+Kse-4<
zi&<;dvOLODoujZ<SflX&6Exw!JXDa33@jQI_5V!?jT>={8}RI&!Y%-@7Dfl$qr1td
zOW{dLAxmCFBMJ}M05)mWf4(b!{Z~qG{%)FNN&5=gLa%U_`SryhVOiW3b{{a2f4{EB
zU^V(FFXi}4xoVo7&19bb<y?SfqZ*deqQFanMxq;(YNy1lI%M8g5hTs@@j09#yWj0=
z$2KyD`#uy1ZlNep+E{3ZitylwdBmxV9cT!N2(M(RCr5OAXA+BttY*&@d);hlkm0Uu
zCA=Jj4`9<MjK#f&Osv_lV+OIcm_B#h)JXnLdgo7IP63www~L3EQ-~#><D%lvOC_JE
zb(g6h{E#&T&Z)>!Ha<EbVfHx~Kb@xzZsWR^H!uYoKIhP&h%lNtICdHSyx*YZ<??6}
zxT!B44{2@dGCNgq4}@41RAy7c4_pY$=e5_Ih}f>N#EO11(jj!w-8k}yj`iIg;y!<q
zKq+rCPE^=@V|(^?Y0TUBCMBABJ=eij44qv^uxqk9l}}-P*2OcB34qmq_V%m(C8%V{
zuo%={6@xiIj;)BhM5a(5of~^XAQS4iCzM(Wt`P#kmNDF;J**RMmSdd)5S)x(1g8el
z`2|xDXQF5C2e*_pGg`Gv)C(t_1x0)=VE8q)(VUfz$=7brt+_)eXsbxguI)_rsn@e8
zOfAzmT*?-IHidRC_k2~yu8)2>Y?x#6Q7hct)#f6fT(AmyYb@wxpZ1B_s@8;oQ<|H>
zks9+hZBedYL9wq+Bf_oDQTi8+ujasbl6%S)FUJpZ|CFX!t}oRr@2dO;c80H-FAmyA
z`0d9a`qJZ!B8QvWBlKT+QUH(o_4gBus~aaP50h;r0EagUOxfdE{K-Xjy7L{Ka8l2q
zYJ<1yY=XT3(^yhMH6uUz?3PkmQHSrH-!Ke<-m?hN1lf`WHvFIv^tS~I%#Y`3*Fdal
zwAdb-AcH)7lruAEYDbf;52JA84xgW!GRbld^RxZzO=gUPaP?nY5!gStXzDU-U1{}`
z??Uq`%9HlV2as#H{@lho`K@p<`z+#Fw<PfoRQg?&WI~22sFFpVSWwD1U*a<D?zW6Z
z{Z8&smxi!O;?tiR7UtmB;OH5-zncJ^97S`n8s2Q7xy@y~2daaWyjlsDtW9{OI9HWe
zT4^Hb!bHt{KVMtC;Px1*Zh6;=jor7{%-QBX6(`jGt@NqVj|lckpS>Xhr|qsj6B)*U
zM;Dp>V!mQa8`#Gc`g78Zf>A^Cl4v{m8&8To{a9?Z#-9S5GAdVW)mKEu_ZA8Gx#$5&
zr(O$oq(UXVhUq`pg%F&DA#G8&>aEJB3SaGZOGiFDH(Pgv2C(xH9QA_O7%*-`q=Q}#
z4}yZ&7=|%|*lrW)20`%Xmpg|7sA3M01^j)4e`Ynj_gTcoDO(@Sj$`Ek08w(R1(D@W
zaTo!~k$}+Lsx$ct@UL<8-|hd4zcSCBD-h6!!Rmwi*9`l=nPLZ+9Q%G)c-!-KlpDmt
z4`hs9lDD@Z>&Fx%Q|ja)OscWAp4K#|IQQtQK#+(UOsMkCVnyhFVb-<f9j$HF)(fm7
z(E;x3<*o-zdR*ZjdL5ajr}m0xkj393_xdDWdJ%bw2(8{SuN`|_C~wGlFvC@0a*1g3
zD;mq_yOLXzt3JdOP^-y8%*Sns72uR`TcTJRvr_tnt5>07L&`r|c_Rr<{0`{SC8+gE
z2iq=y2*umIwo(6aN27`zIg(az*m>|{*f83KzteugT=sngXE;(GCdmwpOgdt!NG7Fi
z=L~+p`qlpIZja53fv{+#IGH&H)Ln7%J>!kc8$tR0xA>c|Y0?y-+xy<nW4+F@Ul|}-
zM`G^KlG_oz*~BB{?On&p@eQHtbGfH~reN{&-f6WSr499kHO0MOx-#$^-KHZ)@=!6M
zt?*uh1#YUO^B`gclDf?$SOmqp5qvq)X;`XS`j;~5+Kfq?xnpuFj{4Y!+DyEj8Z6~g
zH9@RP9wd-=TB;-$KF<V0KL3Qb%XZ_RdlkwRL^mOn7zndzC0Hv6F^<rbMp}Cc3dD~g
z>P~}V6ktJ+F|`T7bH%K2RqI%BUbPGN@K%}`m$F&)xdc(Xol-dQt{B*}6E|YC=jE|{
z9a*249+ezC+WmuEaWq*=8KYNZMu<RL^BDPA06C|waDavf(Kr_tSA>uqM|gENCtsWo
zo??aZ_VDN2^t6KY4jRW}`O#fB{J39go{WLv4u^B{1qbTunpwA*v1&!pl&8yIZ+|f7
zr<_asT%RR5j)%kz?`P}%G343Af~#rM${-6C?yjzHo@p<RJTz-cOMH=Mtx<Gr8ha7#
zzPbQAgmr7`uW615%O>&LahvZpI$3FQFbdR={T8{@bI`~!fxDdoCF<Vp6v;VZL6#UN
znN$c}BK(r;`|jU-%Q(N}^&bkM%SmYga<D#m<o?W|{N_dxlH?OT{V0KYSCptayjQT^
z_6EJe{_N%B#!rdq3q{o8oo`9f;JzXo4x{!D-W(8}&{<y%zN03qt7{!dA&cP%DG!T_
zbxAD2u)r{YslI%mbx=ckF+{|>F^>d+A=Ubk=j*u?1mD!;VM2fAGxg`q>_Rg|R|_)2
zADXx*74#OuJS~$dZH}T~D~@D_g=xQT8<d+7J?knMt%W}C3_jh`VYGstsUC`@XijcL
z+c>id#e^8>xocYyl8$S07PXO8?8&z#^<qJh(+J>j7FzB&6S#rey#Av&0i;J%J-@l4
zM$wSnpYYskzaU^;qQBAQpehBT-rq!3ez`qOM?~78%g0$HL6`?qhq)^kRD>L*hcJsO
zSMXg5Jqd)TR!@o+K!PaYF(8EO-O87V?6D~n;RuH6zJldb6IHN}d^2N@0crNRxyHm%
z*!Xu=;LC?te7LUpeBQ8EmF%cu&4%wJ<M5s(W1gd*IDFJTwkaGgmIQKYYHYmXmDp_N
zC^*xn?K|w#G&bd@8p(vX!i=PP)uf5|JQzDJIU&>6zu9cW5FeY#Q$t^sfsThQaCV_}
zYrd?tZ8@NyBI%jqsf`@_PhH@ApqnQzu8jxb`n-xn%kYhrH1-g$j?jjlo5F{)&LW#_
z0`C197g7ar8;CzOL+Lnica5$+JXeDeF{#lS^kFy5SSPOfy8bz?K5r<K74%DtVtkkM
z(Tk76CIcnz%0<z$!NEb))@ea{TWsYC(z~0Q;37pU+jDb){s{@_x#<$UqWHO3f2uTi
zcuq*>irUNyIw3{+;M8t$*Bp8p-z}eMn;iMN(etNw&5`dA$2`?9ZrIL{jH-0p!?9`X
zT5i+WrEeb$M!VW8J@Y3xBmCXQttD7JQz2V@%rVT&`;!aOD;dC49Krq->u=MroUX5#
z>o{D$UxY1}DrJbvSGRhyGG7)J(n>18xoz@T@?o@tf#gX1xUa=ysBhYBL)$reIyV>J
zggF>4^RuDxF{s*c+F$+nlj@(kQbcRFtUG3@J{2dPK~%xm&S67RPc_;pNjtI>DREta
z-9H+VRd7JAP;ju&*%{!hQ?D81TMFH!Au%YS`{CIa1g*1BP3}Dy3VSOp;2Lm(TE;MP
zOH22Ofe^#qEMc`H`1+<IeS?x85L!}rcY%T)&c&{xmJk=w!NvR&1@aTmNdT+BJ)l|r
z42>D<s?{ZM(Bys*4}*Qu3^e2c2*DG;M|s<lmcCJQLvHb@29=byd$^+GzVnYC9iBlJ
zqW$JA$Lg^QLwUjyH|AP4zbPiu5V|FpSo9i~cZgQ*_7L6g;q<Nc(k4>#h#=8D-qxf0
zQGA`rOPtQHm;--Y$=zHs1oYq1catxc>>Q+t+DZD0&O4-%6W0YcCr74=LpIUxB@RjP
z7MKYpqCVcdKYDq=M4K`!_sGT7j6f@`9;w%{V5J57Kn|gcbC2=mK)-fPXE_CoW~ZVT
zw5sYFKEL0q&U>cOF!#60-u3D-vUBot@o8>p4FfeH9G_Gk9gN~8Wpd%aF5#>XG|wZh
zU5`0l7YvD8PB<KRh=6p`XzbWubE}mgmKZm;nrKigMb?Mn6sMYGY34802d;;d+hyH#
zlTNM04>x|WG%>7E75MQPWSgOu*nUK}3VTuq^07(5ww^Hj>g-u(Pt58ql4BTYz`Q$k
z1)TJf3wi0y(z^hk(<g8Df6B)HXr}%<U4rwZv(!X0Zb1L_O^Q!ru|5v)w8%**OIC>+
G2mK%JT5Wp(

literal 29189
zcmZ6y1ymis6F-U-*Fv%4?s9P}?yklCQrz8&I~2LNyW7Q!J6v3gdvPi5@c91z@0=&+
zB-!0$Gdr1?%zh@DjZ{&RMnfh<hJu1Zla-NBgMxwuLP5b$BEo*uG;W9&eOy?pC}>E&
zy}bbdfVsK3o12@Huaqw@FSob14-XIj{{1^YKYx6Dy12Nwyu5sUeH|YkzrVl#`Sa)X
z_4VfFW)&KaxVU&rOA8MV4>uQ=wYBx`?p{$*QFwT`jEsz#nc4BlNql@f2n4FGt`-s!
zDlacrR8-{W=bxII($UfB=;$ysGz5deA|fJ-i;KOzy-Q0=nwpyK?(X*X_GxKp&`{8W
zgM(gPUbk;=IoUa?s;Vw7E_r!*zP`RABO`Bb4;&oaK|w)p=N+@Nvj7gGx4Vgp)4j~h
ztiwkA>w_sy4!M|^nBJ?m{nxj*=cC&p`?swe4#n{A9DujC%eS|!Mnk6SoyLpBY=D3>
zz-B<B?C)X>-zgFA^V^#Q2f$Sb;4pL*JbU{*7qEPC{&Krkv-Pj;;4K+Fm=PLr=;+_|
z(#w?)?efDL7990vo+3G6xk1a>4-UZe?VEhm;*bj<ef=R{gCjx?aCCdi9ya+2No04S
z;gm|}sYHj}s9|LB=-!0=l*fHh+ebUP?J-p58v<OF6#SpIoU@V8#V^G29;M<1o9BL$
z|KwC<*~P#MJFvW&$yE>xt)`2n>gF&NUU`cQM-^B?pkK>ob6)bPM6$7<@kQ>2RsOJ<
zG~hQa^TX*|_VDpl*M$K-+?hFeWnjruTXd>EJd(RQKD~j2qYt2!XFyllOcMywn}?+j
zx*i@J$^W6@80AL`^SgJosBePWbf-<DD1`&|@A_C{ymO9R*_l^D_urnvhLgLJ;CT=@
zH_4{ntuZfnCBe(BAaU2HF6HUzd3$U7?CflPeSLp_e`RIm>gwwG`T70*UGe*w1r(Hn
zmaK%RhNqz^uh0HNJ+?SrOpvPM@qjV>a2zSnnAiJ?Gay^Q+ln%}59-`$i$dNSu?lsm
z_yh)sx9X>pH-?Mfr!wByNEMGyIc^ynq!LuE`U8m$Q$!Y;e!v(S)=!rH8@&Np7t&aw
zo%T!a-I_qs=*svIP?hTTIUsVwM<&NVE1jc1A`?@hJ%x<&tTybeL(Uu#M4liZ&g_&d
znultI=J)wK0URV=*#v?~C!S|p#3i_rw7nQ1l7|XmL0W!o<~@nD-0Y^ZPze*!sr19m
zxTB7B&O;3Zn_$(%@UT#SR4*SccbjYYwgm#{qKhI8#h2faK%zRP!7t{yH<dP8`KR6R
zki065cIm}S`%>9tYYhTY(x0s}M(KA7-6TNq%c0(2Q8jp=3NGJlq^kiBx_*Aaml(dc
zh%e8K*nf(d&g1K!j2%Iu1RyJQ{&IZpNuw)J<i2!^jK(+FnT>Xd32p|I4oQs!lxI=a
zUE;St$pWCUTI}@bw~Uu71#zA@yx@{$vPyl<)Y1s8^`$Dos65Z%lD%i|yG}*U-7Avn
z5Qg?IKh>#cU4jcyA$d&aDzwXgYXGc~*h5@irh2y!bHxl9wxGSe)!$qf_uzprC&8Jm
zYKd5ev4TDA=NkRzX8ZcABaqiKXYy@2kSO}ksJ@cMf`_KtH~7IvyYMQbZ;iih_G&C^
z)V;_xbcP;F#>e4h2tcCCpq%yU&jf!H&8i+CbboeQ#Q@8$l0`C^iNP9aUg^Hnjl`LF
z+UXTQ>yx#foUtZ_<A0wRCr10kd2+e`v>q&1iqUXaTJwwp&-G5_iQ5M$x+mS+dbT%5
z#~e^qxp6$=PS%Vt;16T4K4dS#VP7N%Ol9soXF8HFpI`)X)F{&@a^n>vG*ewMDKVI#
zRX{7LP6o3}I!~<GHD&1c?KkUICxRfVO}>JqReDVKxpXl@S+m3?Oq2-;>>k{lY>*H8
zF#VL+Cso{71HSrjW*8`eV0ztcNWmAnq#Y2rwZBu|QVvaT{My?Q-J0cW3Bou^m`~l>
zEUnYhS{ReL6Qj3fbgH`V0hs-JY8L;Y-bwsXy7r)iqan!S377<ElLL6ar~xP|(lqJp
za0nZY`ST1t&aBsC44Z$e=)%EJUt~rHjpnrE7#wSUP%$-!EkT8_y|TqzWmjbQhcM28
zz}+ywD9G%=kC>IYMlS~8g$(3&^{#@qv-3I*scvoT42}KwH`14ePRP#Wee<7Dcx3{R
zNut9VWp>OI&EqE*_Nr9s)Z$<Lcyj4!T>Ntwy(rBu^Yb8aLEXb~4&9tInVF^gQc`P~
z&zaQCxHt%%F&-CBlT(a7{)Z&xO5EE2Nob&uGTWTrquA_t8AGXG-<D)~bW^{r?(6af
z8+x-g!I@%K$&!47TrzXcir%pBsBG8oe~Aq;4!__3-Q(0=QoUbJHMNrv*zp6`bIPz8
z$4ZYLoOW&Gx=wJfsW!d)mn2TDGFGi?!Fqy&<GeAp)BjBpNy&1HO4_Q%Ek3Di*KcXj
zk`w?BuRwHpdWxX|4iC5QE3rijd?g<0%dmuu-rwCb^cxsH%a!NP89E-==x!vZbSjj!
zudUMWC@s@()JIwlI@EUmCht5=ujXIIt3ZYPQ|KxwH`6dkb;Eu=kp8xAB|78sN%ehG
zTfxAu9!$1wTJ1BBy!fK%ql^Dxcwd!uJl$>W74anLPjXw6eBI2FKUf4R*<JUGEh#ZM
z>GJ#;kd*E4vr2Ax0ti~+GwuvA1Ow<8JuyN<JOD06`uJ|Q$tRnmRbYWF2dV%7UHhFS
zo<5Y+r8_ZQFrb4T9fH|$i}T@l+Dk+TV1S4p0{E{aSAqZkpD%Cn3jliPpz4%|>7e>V
za3oR#ME%Jm`7^V~?z;o!?18&TO!`hgR?2hzI_BP1P#}mvx(0kKh>8@QIKlr8a|#Rj
zoL?je@)Zhi7xFj4Oz9{=6DdssMZUJ|AS{TQ&UkTg5&lO%&3(`@CGfs=N#H$|dsZ+`
z86Dkz>DGVsxsL?41wq<lnMj}FQrp()Bu&Pwl3O;`vdPREMv9ZmE=eRb8DCW;X{7`j
zhpLeq#Qj5&*UrPoF7^Db+-L9S@B8cV7IPY_WBb%a17XheQI6an->wyeYkj*EM6TqM
zDC<{8pJ^Oo!55offSd0CI)Ow=NZkKjx(GepU`G#Ljh2S0P(m1fym8v51Jr5syM@2K
zwMq=s4wZM1{Tcp&^aXE-oN?)S>fvD>@9M{?tOTw<xn;OH9BK2VWMtYvlUYl+4{S)5
zMaFUf?=Qu)a{jZ*&)a^osIHq7!R|UK`)BfSxqu)+@b8QIVZ`*9+EWPf_Kby=3DZ}K
z4PV|0eDW@1?tnmiRRqsJx``8-F^EdP+v;UKy?R)^|M}Y|rF~+|154m27AR5IG?5uF
zr9i068W{*vX)p^P66K$=GYb}*lWy)juia^u9o+jUd9m~~03^nxNfFp6S{MIJ@~UP<
z-z!E-f-4hJ;;hEzpB0w->Ox16qKkcsRGF^*vQ9YE;mdM1yIx<A=-Pe0UL7z%NTvh*
zPS=>%{hz7>Fz(Ca&Yc#kx42H2I(Xerb=W8u@@;RmS#r^OE)S}-Y-BMHqJbxhkXT;Q
zIO+1*ffeD^Na_2k#U?gai-YZWNA^}zLA${p4?f&*5E!JB`Z0X#<P4qA`hPg-sNvFF
zl%fwUJhggXgzA!?e?-tpMR0U%WXR)+n`5Nb+oMJ1*z&btb!q+r+omh2a^045++=x3
z8EBnoT@iA|Xi4tQj@Z!YA<`hg!T95jk*@3GtyT}aM-5jM3-M>GhJAv61%W*S)+uh$
zN#P;?TCn75o5N#enp~B53Qt?7hKf$+FHfw(@m67H04WWh{5gTc2+Y6rojD_b$@y_s
z9KgOmoZOL1kH<A4WI<x-Tfg6#zmm~=1ssU(zy|)4!0yeA!8#t4zxnAGf-m8CW)6q^
zndoO!EE|H;+0H~}?ilNgb{G%sE#6KMEmYXpOlXqVaTGLnKeqA{J#Z<v*GIiFkEz4S
z?&q7)c9|($MoNLis{ye#qG4icK7Qbt>YZpiD#rAXM~Fr^fP3UxaBjZzw|o5}hUe{*
zg4Tq*4>2%!kV`+17uGv{ga<9v6-HyCVI(bm3Xx5)bMI_552Q$(5>x-P!~^Dn<h_@E
z*Z+i3hSdM?X$R*Z!_9G{S$*wNW5o$-97dQgMU-$DAR`1B`6GAu`3vMm8y*6Jt!Q9}
zR7l=?z$dBF0mp@{>99SbN1S({m8DGm|6UF7xIwgp>eFr7H9=p5Dif7-Tq^TU+mlF{
zg<*je30tPlAy4i6Y%;JJ3E%-Gb`5a?C=AYm7`lunjVcBezwZiKx5Zqg<4po}0S&uP
z+-RWn)yNt70JL|i#E{<8mCm!B@$GyCkA8`O>-ERo$na7&XYAEz{ezn|;;~q|zM;Ob
zVmv|PIUydWyQ={4F?_#IJj1t*N+Un`>3~0eA_Gk|4TgA%s?<^eAiZC?n>5x2S`^+)
zX0VB#M&dR`h2unH9gooY_DuaAymV&Ch=ORfXIlYeFvPs1CT=?JNrxa{7VWeBS;e3{
zD#%Vp$_<jC_X*hI{_Z?Q5LY3UcQkt1yO|8lhC{~DSf&;;GWZ2~gb>1sKjD^6atmj7
zEH=B**;-(bJ6d676-(bg39>;z-XL!OOWrBaA*W*j_*wwk4pD&tZZbIxh@^XmZ~lZZ
z;H4DW>K$D&P14k$@=h9_8_4Rn?Vo3T`>v$l#?#BpRQtYf;ZS7eeZH`GW%=_s5_gOt
z27qq6424H`GDZF~#!-k~@B8RXQbV!}IEYaif%lJ*T{CAqk^BrZ9piy*DB?~D&yQ!l
z5Sx5!;Qd#N7#?Km(catW>R;#j&O~%6qV~d_!f#?K$ADh#RIF@(3zN;x1tCywA@9M`
zA!32CtBACAslKVMI|Ja*g}+i=F%~uZ5<s59RcPxv-315Xq!fKW@+2SyK}#)fXiXa#
zq~;$-bLT76j>!d8=x}K&tSf{$l>hkww$@#-p(`}8E}=7Fy{(!e!M@Sv-kNwm_*=^e
zCuo8>KFEX6^!t=;0F1JZmn?M2Fsyi|q_0i#m-m2z?%K@fO5@i-MzFvfkGF+a$jPYc
zUpYG_v$Ro=(`)(70(pn5Pa9NYyH*i7(fN3CXy+OYvB%%pxHo1mAMWpIev!=16JOXh
z9e;^3&$wUOS;Qnif#?U*wYyn%b%vIxQn*IeLsnuE4$^-$%Y)#WR1v#PS*q1(Y>Njo
zVpN?i{%CyI0T6VyES9Ih@%g6R9q*cbf?+Vc^kr1a6WWS&M@ov{b4XiA)38!5edf5!
z#L5>IaudJ@0Syq4m8#Q>SM7Y>t1;t->ryG-GHB}$4*-)>r-*j&Y`1n|1%!@hZEG92
zKkW_HGgc+>IJ;Lro9Kqg{Q`0$vgNO1P@Bzrh|FQL(ey_IemVC*KP_G7MYs6~6ndK3
zW|V2&nhgeoYu+yscnYctEMnG~%mgi&f?quJt+i9Pw``NNOYX@Brl`f4`c=j6gCVGv
zHsQ}~-MT$53=l+imHm~+;Q!6xm~55`0;JhDB${&AaWXm%cj1ED-=sBDp+le}_2fc+
zAWdahG5g~-sRd*SX=PO%N`sZe4dUA?2$**K-RUPF2-8(Iv|Tk8PcPLfn+}lq0*tOD
zvp*DE<)xJ-6iSg5d>5V+N&h1ySy~l}9;9!7$`OON289w#RH_y>pD33nyl_kKo6dpV
z?DUa*q2;YjL!N;3o1eZI<UwQt-zW(Dc^yk3+cz6Ll>H*>Q$>?fty{_eSj{vu^AFZQ
zy=wSSOkhBPK)zBA1OO>B02Ql_L4I4Psa%x*JbWLAIV&U7To$?Jfl*jt$Yb9ovVC1&
zrH@sH7&BJpU&k=!s%?7lRp~i&xL9J>!)^1zz|f2#Lnh0!#t!+ns-JNy>CBYjv>&7y
z7}UdlD|GgUmk%$^@O;`lizPENpOG#`B`C~PMd0S2e6IgdHKqV^3-v_K6(jg+UgCUI
znWdbM3Mf(4qDc#gKeV3MURoW)pJFSP57Qoc|N1Lym{i{Seis@bMMGZvd>IliRJJy_
zD5H~o&`%dZD4ZN84J6ep#Pdy3lk)uG2~kMTmQ#fR{`-~;-q*U-zRR|tIvh6lQD|VO
zqaBGRFCa82i{S0vF6UIPr6Z}v8~cC|it8j67;9XR7%h&H;+YxKvM|77(KS90+S!D1
z#NL<-ctdb+n3HYO)mkzH_;_W;9RM-D-~H>Tx_;?gl<q%JnmLy~exa5gZp>93$|f=V
znf&CrGk*0{S-qWbe2(xv7mQxqTrq)<-H?Zj{E5%ZHQAa&N@nVUn9Sk&>=s{7cd5km
z=`6hG%1hI;z?sTY{FFX^#P!pxjOUZs`WFBx+sRc%tOdn<R?ttEO2PL)EAl6u38{bf
zoIUUX;VKSoG!s&^2SzTWBF1wICnx5Vu}$dmLrZ=2oBnIgdFnq+omO{BvtQ2nJfum3
z0<Z%qynRuaU)(&V3cvj>Ki_M40FA)5u0gKBK_LqDWRa1Wze$zTtWj17pCg+^lv!Q@
zGoLD)8wJ380})s5lK&VIIuqL)>?8wp-&3)kUhGcg@1X%h9`^!L4*unBv)ka`0ou0>
zqsHtvl`q6*p=M|3zSnc#QC54U63BI=F;G>2Ao8GmNEF*hT~eMfjHp08BbmoBW#gKu
zXjIc-w>H7fbR8?LZ6>eM*XL~BuA{<+I3K5uy+h@u8K)4z0jZW2X6JyI%WT!5YXp^V
zL|RxQKZlau8T^@Fe%?)PwM0@P@75+OC`S(j2D8d^?n(HtXJ9ju#~?=WNT4@1H^r{Q
zwI`g~nq$iaMNw0VSffkuQrFCrh0?jU*AwIQ1jOhwVkLgjTYoRVnBYB0J_~_z#x@on
zB)J%0&DQt#cIb+seO+NzIF(H-nx#!10Wpz@e%_@i+j3Sd-1)^|YNhS^4@Ld&HejQ(
zd%bUuz3&_Sb4XDic}ijbZ`w`YO@}VBn(%0VblkUti_}}uD-=aWjt4UOM=MX|*9iYL
zRH8H@K9KC2FzM{#I17XZxv%wp)`uGHpF=a|TyD8Djn=25Ty51RlKj1X9ZFfYgdicl
zRa#)8{+Nw!l-sQy$ZgU(9I1(OKDt3U9dO{9ZRRM#-+VFWMnm%A_H@1^(~$`_?KJ#3
z&{5BGu11p*Vry&jmFFW0ku1Tn1qDi`CrAwnhXB0P$BJ{m_&tvR*Fnq}sHjEDKcv#d
zbqdiDsP5mCqtsRV@htf1IB1a_c%=;$3>^8ik~x0=noi<K7Q1pTo-P>&s%qUO2<+Cb
z&BJ8sF4J}+9%f)!saxb!TI!M^MSa5{1ICi8aC1$x!idn1k3ggD2k(4m_&XR7p)Fdn
z@3zq^9aQz51?AeE_fmXk^jIHXOddlBNiNzV8bHu(BA9A+mHmk{4d>kG1xWyEKH7|A
z%KP&-CBD)e?zkT4Lz!J-Wg(%T+$-s0ry%-Tvm|7v(6V|3c6Gnb<oW}EkQ#C$o%kc;
z-#}k4#mwH%0H%e;vb>mI_}>?=@HIISafa%4bA0Ax<hTk~%B(oz2Q3qCq*-EN#a+kb
zb~B08EHpCtH*{x%c5Oxq(wb<2ZSpQt7i&Mrr<nTD6wdVBzd4<(4ocvM9bFwu=?^h0
zP7F%GO;*)xwqM?S3;v}xiL`MZ%-6g6YESFw^HZt*Tpw5bOL{7yQ0%;&6Qt%x)TSn~
zl>jg=h)MxnZNMn~`;C<X7m)xpX(Qd&`M!WO33j05LuH6@uf(^QTQc@E(-2ZwXW0^Z
z3W_y7%`19{zw8@T$n2ktCKrR|p0L!gl9a^!Pezt3$o@rkw5gBu;jt778>I~atRd0H
zetctMUuBE1`=6<$Cw~^EwKkH3=bV70X`%iAZ*{AzOKl58E*M<PnpmzM8AXNFO0a=3
zrA(sfP%$dI*<6(9I<Yq1DviKs0RDVAC91ZT?g`+xif4B-HoU<JKm_8y9HGoSApX@U
zo*j6$2cAiVhVRfHz=Hh1OOT&`tvy$n6Q-O^5Kk}HCzOS`(N8TeFa;T%eA62&OPPJM
zhg7*O*MMYkffkEt^9Q1ARDpbDk^y_xnlXt5skHQ?sv&>bGqS*5@l=N?Obk-$_Ug?b
z98fwjh}>|yvH%+Ami@5sV=~3EIFH=vruGuFKa2XF$q_)L(gRym2I<^1VwQZOqXrZ6
zqC3#G2=}xoVx=uP$0<+SVw7j|UED16QguM8gJmj%5xkP8UnsRADK(gKbh1CyU_QOS
zCQ*h1fS0G&Nwk?=oq|$9QTa(mw6sPl%|3&?_Jyd~QHnv3Dgs7Yd=KG_J^y&KU$L?q
zVLsLPlE0{Ux|#{B_#XAX4aCuZDT<*O`cqMtG=|wezjyaX)FKnFJJ?h+#t6&}%(=bU
z%&{WNvX3)Da2bAu;=K@s5k;p?$@Cx%*}gg+>HOX;9NC|rAg=Q|f4N)A^lO92-flEI
z%#g72PBNfZZ?Ba2wVS4w#%X%>ezZ${pLmy+Tk38c7%$mzSUaof#tPw7TP~<N+sl-R
z;f}=^r0Kbqiew2@t}Wi#+uN1F83@OLy*Y{f>#Hq+kHwe@9c#(+Jr4m$GOs_c#s2M%
z%ATF9!rq5_r*G8AB4pjpp6=-~UVX}emyz$$1>aj{Pi3Fq+q-?nGUwZ|b#7;pOS@6*
zZ3!yK#-Ugt=Xc#&$E7FL#*Bl~D+WlGSUek`nABZv`P|e(J6B4QC{o~<SX9x6v4{OH
z9IA|&lb%_!m&WCXgrsU6WLI>}UMNI*y8ZVFyfTtPIuD?Y)KqGY5RddRN<lv&T{8ZO
z)aS?jO7oYY?Rz-<X|e-|e7}q|BBKRoC*ki3eMK1Q+wiKT>OxEbB|C9)b9ZZ6v`-jp
zRfV7AdguwZMC?yRYd_=W1@<Brx|(jiTV3!aQchTn7&FUWK10pS%&4A|OomTABGn+8
z>l{v^^=4zX%Fqb&X4~RT)ie3z14}eRfov{L_&e{~?vc;<tQ(h3XRoynhuHnh&t#An
z<yP09ZscQGZRO<-7k*!IGNO35`k2Lw@aHykXAYFxnGB#8Or@W-9hv7I^QX9)hMX_?
zN%b}^qB!zL`+aX7!@yMD^=cACFbj2tM>;Qjx_8{9cGNw7!k6hfG}$)kSHl>*ch3WK
z2RbyxZ1Bi6Z00(9B^(^F<GlHZh&WS#!(r_cS*BXzaWgf_vZk9tuNdpSls0Ji=qqdg
zPd9zdRk%67hzZev2K}7FgRFI-KJYlhIp~Yx$C;Cx;Zj(O1kIWHfePHO;2+Dm=|D26
zw1er3e(Kqa{x4xnnVn~A**RU`{HEHtzQ&Z!?ASv^!~sN^S^__1BJgpy0_}YZwMK2D
z!1zch%_Pw@<!gl#b_8&v^y$N<kyRv}J}XKch4b}SpPAZ;5W%RI%<2lKfkZdr+9{&+
zDrK!U>DJYbV-G@UVzs>J7|Hy*wwbu%jaNtGihOIsQ4CSkxI`g90#~-7cuQ|PDkjPi
zup^>$%%8ys_LmE<Wd!C*&ES+ktFwGjIic6WU0!8{v7s@a<?K{y%E<XtAFv^Iij(hH
zW9rgB@*C-1KoHZ{Xx_>3&UTJCY!Z~oF_7USY|6F@E3avYx%(eWN1q-0x3%3lE~$8h
znW;c)O-T~Z{UX6Sg$`EyPh*H+TT_-DL<$$|t(`Z+W+2M1n`*mdW*p~--l!WX$6Hu3
zu2G)-3a*#3!Q3xE*;OZR;`DdJedE=pwEF9+F<U;Mt9=WXFUvmg{MkY2PO9&No&MuC
z$HT>H70>h9{7zDC3=2UPw&vEj&?v*6<QIeS7mKR&v|$YPCKXmp_VUkyhf$p*>DUQZ
zp1<!4Z>-f;>GyVgsfGRWnAl5RC*SkOH-#B~MF8P+Xks@}rzXQ3;^f7j(W&TLX-#xk
zm);t>_JoWU7<T+UHFY#(0_gm0vz|Ss4HKUzO$bC&$gy{M>|>Pm|1}pa8ji3Hz`XA-
z508dXjv=ljvG#TT<w!n4_Qe_bU^=PjT}?Ix%Y1nu{6irCa17c1im{Kj&qzJ5n%?(N
zK>@#~D&EeA8bC@+&8xEdSw<o$SRk2cyhu2Q6)D&t8SjY(g=~Xh4s&oD@xk(2om$$8
zRLUoEx`dw1B-Xe<QB3>g$ALz8Ch;o1Djc1m{~9QF;O_TW&GEVq-O^{|=-g8Lft=Ie
zaolKJGlj}Dts8^Xvy}Q}lMPF|<&E&tg-!Fz-ecJLO6Ho;QyWy$rKDaIW_-mlL^-N|
zh>}5m1SBdmHpB<%z<x2I6=dpZaA>jqV0LPGldG5iwHu6DQuf8__+sZkB~W{K{qBs&
z;-}@XXDn?4Ja-RI$8^w4V@zJdEK6_0tmJ87Ou>~QF6B2KX6f2x_CmxOU>VVkneN&I
zA>9(g21?Pac)V6@ly03UpmdOrfUP=N6dzhgDjrzcMk*y8mfNoVk9ilk2{4R^Ud}Yc
z^h=j7k9GKp#<`H2TD}M&YZM`o{XblXzOG?lJ?HCIOVqKRG&aGD{4`DvNjd89m_>Q(
zI9*B93R`GMK@&rfYG9@OPhNt!S*gshMKQ?c+<s#f#fpZAcfdZDrhSr1_y{gZNr+_`
z1@+!ZY#`9kGLR|(2uJ-H?h0SA@^+nZQGscG?km)NH6CD*s#RQ!$wX`XFl1#sOu_wr
z>FA7=W!`1D)30#kyKqhcD1E|0Y%jnw|I>Wxhl&HL_S5TOwyI};#F}Q;uZ4|HrB-t^
zkGLRjc=1TV;+@7)j={@#`v^PsetVVgqTdZZR26I()V>zYVP8dICi`!sh$L2buy8LY
z(~6|38kNV~DIKUm-&k`fWZ&yN5U8O-M4%+2&lZ>9^akO%*+Xarkmx+|;6*o1{;8^1
zbjV?($<+N-{5!fy%H<{kd`Q~d$NPoPAP%(oy7UMMC`^WCfK3jBSW|7KG@!2GivpLv
z$3is1ASwh5>i<jPz%qsZrK$#x^IALe??m0$AUIU#xe~qI%~iF&Suq`r0AhB}7&}wh
z6mxndPH>^1F|`@DJJ`LRHiamVbfF4~oW@gAIAo~~KM!C-)~msUgSrv)Rw{I`;bobY
zP*Bf(x6_HRhw5^W<5ImgwdV3`?-9M`bVIQHtOixre{v4x#N~2`zJ23o@gI}#Pqjuh
zTaz;9ChPQH*9?sAv)32@cpxYW9~_0=Zi5U2&2#UJTyi=W-;ZwPzpN3A3RMXJ8Q0mu
zn!5^~`5ai64*Fu%7dJOQ!dc`v5+VJ)Jlhr<h`XEc!P%#0{nihB+Sod1)b3m!BtY2~
z?pAtQQmfZK`l26@nwbTTZrMmdOC@Xl9^RP+W{FiFqTdBMK~z@uZljOv^+TSea$aAs
z4ZOuP^F`|i8~2pr8Ni;?zIP;R^s?(4|F)mA07^OnD6zkYvul+2%|W7XMc12TFYI(S
zGD$ANZ`$&i`dm9kv36jj_4@sWha0avAga@?UPu2pKOs@#yq(=8aEnq+OU3oPeUCC3
z=ub7QN&R_h1{RlPGY4FJh2*7u0P+Hl{89hsGQ^J8sUijVQDiGscUm)JOe%&gQr+}8
zDYe|sEc)<DI@j0J)!EA;yFU0%#^30#tbMuYZtQjQJLQYFhT<HUw4{ov<u7G&Y(Q?}
z-l?`-f$b6RJv3`#3GqcJAJEXWFbc^El~p|I#@CTVgTNBLjaAY~_wX}>tmj3Xv$Y=a
z3iX$cuvI0i=R;bAgVTx%WN&l}hv4)1TC)KWF8o^;CDG*PzBR&SP1fj*6hL#<5GENn
zD$!gIEg4)V2UdK`lL#GQ#|l?27Dh*pN}?26WWuob&x&O8!H7V}4JBD)iWECXL$5(%
z1{8U3KeG$dyeM-N5_4;MTr7&QhlK&9iPB^0l(pF$8RzxS>G4~cWi@3iEVcGhreFzb
z1NswQvo&0&YQ`VDK*ONI`73}Gn_Iq&w_JzEkDpi)A!G*a7IJhLNUXSFv#`(o*Lf^(
zVdbk7M76I+n?FOXf<gKCOuTI_tNu>BotI?ne0XeYEL}zD-s`#I7y4rZO^2!bY+(ln
zxMY({GtR!gw%2wyHQNjGzVK#v_;Ryk=IzEezpCF__@rorxKYmTM^#8s=dB%6i<BgI
zylsvmMUpC9J@?R%m4<UDfY`5<Gi?DrRG0K3v~MZnJ<L)$-6z=qN;eSMS+I`bI%T9_
zRvG4{roCQ*OF8mrPjq|%tESd~c?}mZg_K#|5Ez+N3JPq%rugf(iaZ)zDx(uweh#9S
zBvRi<WjISPiTFN|sIuPDD1H3ps&0hdCQyf2&SA350}$*2U!uPQgeq+%&mZ5ralGJ$
zqwN8(eflc?6{T!2MkiS`os_P@D2AD$m2*w3C+;42RRqP&ZjBCW<eG(s5B1w%x-%nV
zY6*>%EG&`JB1Y<GjNKsS#i#A3>F?@2pRcUATjkaI^~C>Z&f#sO+G9VG&bj)%P2tt`
zX1o1@du6{IfaWmTpf~e8E$>nzB8x89mAqmWg>Qw~1MLw}(}f~*f%WxA!}QcG9s{wR
z{mWsp<H6vfK=mR-LP$YIMU>Zf)?{si{$T5>vSic8J_n8J3LF#8OWz9ANnYUj<=|lT
zR;uS8(g6ut0>9UFfS!y35)h6SopXC&_5d#~bCPy%xH)B+r3ADtfWNT*)Oj{$nAPR8
z`GcO117b-Oc;<T3o57!Md}>reH9^Pm%=2mZ+lUqs9qh)r9@=^0ypS@66g^O}tLQPX
z{{psZ=#Z8U@qz?tBlgeH7Ce)u2*Jxg1p<pwFaG*p$rOb^ft1>?Eq!`hHFyM6S;1X7
z0%&zKVNZ&f_AZKIg|Q<WLKXOtKw9IO!RWRylY!!GUpce5f1yWIrQ;KOg{lf)F{A?H
zlpXz_#4GjuTwIwss6l6-gfR3>JP4u-N@vH}m~r%+8EStXDngx?_ftDU*mne=zq%|T
zJ;xIdf&&>a-w7JZMP(mU6vi(2hrYp=%e3-cp0uthhAO9r^?(q`0q<+>pIMh5C8(98
zH+2dC-q-RTUBe%vO3Te+^^;9i{(B0-s4>M=QLZzy<VOL3MlngIjEP~yS|y!8^?mzv
z2Pk}g)uPdElW(oRm0#i?Mv~qseun_+1I^51s2;PK&59yd6c{0>Ez`?$U8v;fhjNp+
zKwGE*qN-qkye#1n+f(5(hZLvu&>@ZNx(miooC%Yrm4c=d+08_7?2hYUgH?-6T36hT
z-b+uxX^O~MSZNFvfbVnf5$PgkJ1*!;eZAe#$O{|fAA(v+#mHmaB3LY;Bo?c%Ew5}V
zSbgyu@98m*@}|^LthRB>?t}+jq2SLtA~q^eJ%D2Bog)B2>elRwHKB>_t7jO~cH2ga
z0PL?*yzu-_W3XX*W&WOz9jL=DY<8&PL)a=fKv>WFx7hQASzq}6it=JAwA+M>*tffk
zkVM%A?;riDuZ~GYaS}XlF$);$czQFxHbh55qckyMJC2n7U&pyV6Rv}~&&KT8NBB!^
z+hsU9NfwGiEn+nFBW(!^SGtC@%TD~qV-mdchKteZR#RL+fE4yF$c$UO`W?BvZLz9U
z`r^2wDRuW*;g|C!PLVz;kZQmqq8t`BU5^5Lk|p_i*<ScF++h^|IM;UGqVFXJOp&?D
zUy@9q{bkqYQI~?Zm<O+oBZ6^%{PutZl^iSa(67%VN5OWpzBD0*#tgDaBj`*zRo&ro
zUFwO1QQ5uyUXI4)a`F$8&_^nL??GBu(m8F?iCt~VbDy-#LWlxQ-22B4fXS#NDvcEd
z6~whO$HVM-Dk{giI+1(6#VYKn48!?!`OS|V3fwO}NUhB<APw=l6j|L{st`%3Ds?$N
zeZaa+jI3fM14Fex2uSC&_?>|Yws6$APyo30g4|(w4xiKm^M0&p$}#Ak_fTJ2J$Qk@
z8s!F&L&|K*6gnMa!h0~iB$Nt)hv)17|07=OvP~AI{x<n786KB-ZvC4*?gU1t-)fE=
zyIESsf_;L-2nsj5L7ktW`$<^%g}=xRDv?B;ey*}8^?SY?<6}NQi#4rjr<g|Cp~NRK
zr99s4w{;BRj1Ie}Q5wt3&XB=mu@tHbIe0=@EW*EH8003Ucrg_su3DdDjfK>Z78=;?
ze#I+%5ojrl(L)se!jEUu&HDVX{}Ptm9lSM~n%n)}Kfg>ty+pf;cG6CY_bw!r!Jcz{
zkKY#h%w6&mxH;-jCkfe<bu8|p>}ipo#2wX-nwen@1t?riW&`Z~-%?r6KHH2})f(n>
z`}V>mN&Huv1$es$rse{a2ufpw?+bl)hjO2{QO9yk+}MY5O$KvK7Ph9$PWxIf57<}w
z4f}qeybo*;uR=lfv0>i4MXP$zrXb2BoWtk&BruF3yQ8c!XIV(Scfi!BLj4aqLVdtT
z5n#Ya&I4b6W_wSCrjb%1C`0`(YrubGsmukH^bbIpn$6*8MH$)m5aWpe(!*c~--{E#
z<h6?af({`GwO^v4Mw2x)tt#jGYGc<!STtm<tG0@Z;@wt(!?m=itvsLtXIYG(%^-;x
zPuKE3*XgyqL9^`k1*^L4F{b@IhC_Me?O~&%%1kN!?4`}yWr3(Vql>b5C&OFa$I|=y
ze2H!sWcSCY)x72j)}Fw?rN^n86J$K}CA~sui>9EdX{cG)dXpFrm<7JZvnC{D`PwBS
zEk|Iwh=@=lzSK=Z&tXshQ~v|(9e-u{o_=&Sqkg9>e)%O8xrzuExa4j7(Zcebi~9_J
zeqP(H5kw%+#t`-a_B7Qu6a-?kz~Y}vW7HSZHo?g@tIf#{E5bl$*Zt5`_2x|8k0+LN
z*!lP{i@O#3G`HA2X2SrlY<_*gwDz#)>)24XA}jIGiETKql5v9Y0tI5g8u7=@3g=?!
zYZw1O(B!n+_XI%B_$4$dBgU7w;pwBbn6q$z(lU`~sJHTe#Q!7>Zt?>eU;=&9RZ94w
z0kMFU8V{>o&T>1^ys8fDC6wViae{j=-@(V;-`N~cr?r<e#Rsg#l84BUF3{^mUhDw>
z6wnyb^r{Rn-8fBWe9(5GCCcfl)PQ&nmH905cR=V-&{XBl2!STS2GGXcgZSp#qZ1N3
z2Weth90!<oo0%V$SGJ389y^6ZU`TakI>3X<ERV8xoOJvh>u4^B)En+SvM9zUof@b#
zfI>KdLWH=)_Yfy~Iy-q=CdIk+I@1237QlSJz=X(r!*Lv+RqJlsO_8@OAxSREtytkp
zRg2jXoDYFfD!HizNcI!tl#<?jf6AG)J#1p-s<ILLOj-S?NXiMa^`j{o7>km7!PcIU
zl-}Uhi!!Z;U*k{cWtze6Pf<p112OkyUDHX>me+kI!!=m%x%Y1hPeaDeKA~9FQ)4;A
zr;>7`z1)|LulM5TVAS%A#iDN*!5584?`#+!OH0ZYrR5;ZRw6)eEQ(PdsvvD;(2E-B
zlpzq{gf$jfHBF@l@>{6;c&0N*BiQQk6=!SHa~ep)UhcG)PdPoyF+|#xT%kb&YwE(a
z<<FIc?>-l1sopQ=9`r}D4Ekv~ZN6-J6KDx@@|^HgjV)raJyM8RoB?kFTQNS=5CP)r
zR#`MrOpjZh&mkz}Kn`eH)4TkP^5~rDRxCj-Vt^go9%s3Z2b{p(hZ_}r2(EeE>XcrM
zM6s<8qNiL0nNS<KG_(WV7p`;K(mu9j8%n`m3Vj%X(94?M>&^UufWh+=J{CvF_{vy3
zU=~lQk@?T^_hoPIba-yE?=BX{Ym|mZ@Li%}E_ERQ(ZW-j1hP_;H5xdj^Muyu8qe)6
z%O9X@Q>(uQ+_@>b%^euC=~dck=@TVSCv>RO6zyeZ;%z4$!}xxpRE(T(-@@{h1%Wo=
z8h_-J6H;}AOUOr5(Qu|(Odk(`iI)76yybNsLnFehuwi*m(0OR$u#bXoL=ZB0X5Cv@
z{~Gwkuwe|1R}n}+@pJXjaQU7aJ15Qq8*-a&85dhRN^{aY3TuYa^dWGa!18P6h!zkA
zsqH6KZf0v_38H#IYEx^U5ZfkXSHm?R;N^U|>8K8YLLp8nfuh{5K>fxlAu_jQYWOdy
z4$%nC7a#taY*!4iL&u!~buAXXgw*^aP%?un8Jv-QSKOp|rzDxquZD9@yq|aCm@;Dz
zL0Y2-r?`~Ny3|G%$-ZJa^+buBda55fwCjbwg!t7piV(dZRipVGlQ2S8(twayd{5v5
zOQb1WH*9kbsk&0x%hLd16#A$U<~OWVbI_^XW^fCCBkB*+*sT*HNmt<d456}~cLe=h
zi%sbo$aS9ZIs(9cLQBtz5`*c#&eMJ0h$t;$_WAQtRaN+vYJfO*Wkh)aptc*|jrlR6
zBrmPk9@D&R_pp-n$}Mjx2vsWhi97IcM0S`#gMl4pr#QQDL?*qY@`pEBky)oHop__`
zGMm|p?)!D9`XW057p4pG^qAC@o?$aw&~rzvNIEizJ?5&^0ZXyu1kmkMCy+iFY#cN{
z<hwL57U6jN7u=3*))ZsU!040>xB)ObBLOR(gcb>C8GjgP?E{-@(>|Q%Fa<eo8nL56
zm}Y0Ytyc<^^V=A{TYB{v=9dxtPTq-C-h{$4Z6^AB)Y%a+U8I~_F)B9Z$ru94$vjDx
z5>%F>07R1gauA_ed9ulq_A==5-Iu*)>|clTcH!qc-XxXRLkwE&Sktpz+x#c;Vs4z_
zRT4ksatt*3CD|7_Zs8~O4ljz#DSF38n`RpI`<`f!XxlDsnt?jQlL(b9^WFVQBc+zc
zE1AkGnGppx98o2yb*ibbfHG->V@<@=m|S(0<1}66i=Qwruu4fVfahhYL+CZ9xdHO5
zK$AuGokPW_J9b|}O!ll$aovUvU%6~yJHc6{Z_3+`H274)cP>gGv#3G*ber!0lbH9@
z>zyl+84!n`L*8*8%l1f3yV@xFv;fDGLE~zn9vHRohU|H_>l<5~s`*Bu<%_|XZ21{3
za0>-}ldVx>5nh_3JS<a=%p3cKs27D7X>c9u^m~(7!D7{CW@nRDUHq>mcAkB4Neb!J
zUE7>}z<*$~jb91!Ld~!Ee_K+WMeACkus8Ku>u~@z5#A#{C@9ZG3U>YaXGVADOYp{+
zh(Tn2{u8T7UmZn60tOV&sz%-x=^5JZcuL9_$+@2KI(+tZv<r>FQkUYuC0|v7|9d?N
zH8~&ff6~Wt(|_PWS)mcG&=@Bsh+H$39Zj+{j^BO0oE9I7zgNH<YEH%VxXesC+dF6*
zjy1plzzq!*A!>}t^z%KL2{4bhOfE6aXp51+5-{~ipC=1a!|0$H$sJf|ZC+UMdG!;g
zL62L1`RVVg#ae7s+PEA7hIpiia71&$Q<e~w!fm<K`474SGN?N<FwkPLVT=Z9CrDw~
zP)@|e4Dm;Fe^de$g4aVatvU6N5_Lt}Y)7C<@(<@#hvWFa^zbj2aihbErm=<|J4QM6
zy7~Tio^<k^Af=|)rb5pz776U|%xGlQlvTo0BTL)Cs*Bb#wfI4OJqG%-O$eg{#b$J#
z{O9c0pehCG=v;ilGQOQk7gf<s!s*$qL}5c?97&I?`Q9i{M>bf<@#6K9=N4QxOmdz6
z;)>0trg57>2Xh|hC9!XVaBy8MO^?`J$B8B%;t8GyLY}j%cDGMB>oV$MA#Rl-rqL>k
z2?>%><69xf!&j-$+2dxgy5}(iD)JB`01vy?wh~a{5?14z%%jTHmTX*zo#BgJAxNvC
zG^#Ke^7hcQVzfw&p5g4A%f5&*yIG-k!=iN|w!=@|wSiEiE1cL~#KKlf78X924ulj&
zy`PNZ4Bt3MJe6>M@~d;JmiPY~Utam8s#)PSOI!HJM8+8>1y^m=L?HAhjS?AA#shNu
zkgPLMuQf@iIQ`G^T#pC(34OBsP?5uHmZeLBN#1TyhrH3$4BL<KkXn|dvP`b%ziG30
zEw*~eR~!JNo7^S_vtz@Gp4f4T`X>4Ovts`%rRGW`SW%q6xQMSFRru9Iwr0Aq)^?a`
z(+qM(hRIzTgXBzfICPP)s7bHJlN+N#?J;tc^Dq>CSe%U|E1p|ZBexUee|qqh<f@j+
zew~cZ(@nK%tgi>v=cKb_Dybp*$eOLvexCf+^w)%sb}r&!kb8{fU9vM)1`-$sf>z+~
z=TTSd#||~^=GN*)sYPZ*#;&NNSi~vCktgjxyH%+SZVC$(+pL{VYy(5>%vVZMDUzuM
zT^b25{3>VB$mH1qb|M@yTl=vTmM^~=42dwBA#y7+9v)g%6fGd1qT;!=wW2ygR&%H4
zid+eX(Uh{?N%EMD@lL@PYQ<v!VkBK`V2|OY*o8=$h5+)Qzg;njqy8@ilNKkftRxSD
zs)fpBpzyQw$@dbmG1!pE@6j^lV$WW&@5kl-3b*a!lxqk#i@@&j)IjSU5zMehG}Nda
zCuugV!i6+KJ6pw~c=cd4ksiE1@J7cL9!=tOo<f8!*`ekf4HjM5*|}t2tQ8~%XnW^y
zk2T6PYr+rkv5V8J;RCJuim8XN)Ef2=l6D!`v<T|!Av^yhgnXhS<B~4DeNd6UzT)J0
zDUs=P1!c6boF}M6LBmiSlM$GhY?V%OR4Vv>v$*Ki8QohR*2YR|n}&JrJ7!3kVtPDA
z8PLe?CwZYLPuWO0$Zhos6>ibwD_PBxQ;r8UF-;2|Gi%-j_&+k(P#D7l{XXqV9Xp7x
zHymO4bcQh#C6zG+3lFI8+x|(*+BNul`;@QuoCg?HD}s=`(i=_4uci@Wrpa{~k1}BA
z#>NnivUvSnuyvu}=#7t*1diHi4&inksN<Tyw7Lh}vj6F*gyvuC=*!)#;47)ZNk;0+
zhCh23uzdHas0v+qe@ban#UWPo(!EaL3fwg4|6{W<<1kPFCMD2M!AG%+jEs~%$Wg3g
zJksM{*`3Ut#B?+rK5{4hXTh*+5t8%ZJML=st1Ox<>OAtM2tjym0F->mygRP<d(Wky
zI9B2lx&WSPm)Ec6&S8-U7rkUN22o&;3i_dRvW_JBIa)LUHh2S5SL3iuKlV6ES9~B$
z6fNiQnQ+B$BBP?x0=H6O5D+r&Zq|-vX1X(UqB?O~H=_)laXcwH>YQDC^=MG8ZXd*I
z26sHMqE(WZ92=hVu!8SwU>2hgy0EpWgJV)0$=p@vQ0@DZV4ueA^`_N4#g`-GW>$Wb
z-!`j4Lq5T6%Y^I&lA6=PP?|&<x}fR3<63|clzf+g%WnDczmfNH0(`ROBZtX!t=kEy
z)&!8p4Ii=p=`!qCMil`o5oS9@1rZZnBX&Sh<41209>Pk;e0n{s7LN#~cAv@6r0?~V
zC-4E3;JuGYVQQ~dEAvIj4OH{2qiT(YNf>5E^jSEfRR31D=-V{A4d_=s%`Fg*2RXEd
zQEz>%J8H}{`F!b^$A!<_qBQnnv>Ifrs+CAtM5r<op?1M!-<{Fm7P<R&^5`TjaW)3x
zb`T^5PTfd;pIFTf-&kmH=op~PKx}u*zv;5Lzl`<IR8X7$#1$F7veCmXbF6@99N_;!
zkzmcMJVRQLkHnH;4>DD;xSuv<4bI_UKq)i0t)ywrB{|dj22>V8oWn^tNt4gA{u@U;
zmBibPf&!uS>#mo0{xQUMdX%S=criktS^u*Tr=gIBxZczG;(DLliLyxD#i+TbMLW`_
zF`37s@_|2or`)MkqK*EtbC)CReM7|eL99vOR6ZZuvqnoT`w6isHU<CLAB~c>yGf3l
zeg9n}q8pq18{~DNBmC~H%z|O#H<8!3eEpVP^rqhhG+J@t0YDwyCRbg^C;pW8MC)Mn
z>O;W9G|SR&$*^+T*&>YZ31nHK%U_W#YlM;hH=tC1?^eA>5b$OI3kGo!YFh8j{ANw{
z_wmGg@(koRTW+%;+-W&9<Jsm|MhLfJ)=dR(AvTl>n|~y<YoYInhF$}C^8pXWM?|*o
z0{U82{M?|6a!l(k4wZm0PH(1Owau|YS;y`D7}4Rhbu5I*(8>2qVvGRAPvD^1O<9p%
zRTFCIm1DRA)VPq`W2$T_TL)Vc0HS8<Uk)^|WpDLM`~nX@xi$$w7k#UnKXtmZ7g=VU
z+)Km?UeRo(!Noa~Zod_7ceRCqLJd*5sq1?E@wBBz4;GKe`>y|tIkA;Sm!NA~j(P%0
zGOKzr@HF!A18arFMt&|IOo2EKu>U_;g+F7=^87!Tun!2pS8~K4=;6f4=09FiEzL3J
z?VaG$Aj77;|NmeMWwtD+2$%mCya0{GVSE{J&2lLdDd{~IL*yc?ah0uxLc0|(lIo?y
z@4N<n;3ZYZAn2aS<y)F45ffhN>k9R+CrVIE0YNgNmnf9wMg()wh7bLkjc;YugME|J
zs1p{7*|!}$Ihb)?&dxajavekOqdzoqP9FPH*pDAgmcat*)fpZitBpArDa;KOwd~6P
zDIroOJuM+A;Mimgq|R37_T7r0@JfFfumK+i9P=FbxETzLfw%z2#ID~k@GnitFub7#
zBC9=Wak&B)j+jZNwVZ~0lST(hKc`LiS!REp$6t)B7^4#E28tJikG0tTs5B7o8it+t
z-Ida<H4o=5w7`=Ydj!1$BRt%ANm@eZ=j?%?4D`vaQV&Aj^Ly>*gL?zU=WGIe-?5p#
zHZ2Cs1+YVNYqaTpbUlPcOpS1HAsPEOy3Q;UDk)%Ciq91`QFl5fvka#C*hK0U6$Wc-
z<(pk*H3nr_R3HvMOs~^zo(q(dAck_ID7I@6&#OUh-UyX#E?*vtwLcMnvL9O%hjJoN
zQM5x|^qj5Q$d@*^y6eC^vu|Ox!o`)~LXF_Q=q&EB7b1b$3M~NrbVy%K;oJ?tJ_?9%
zUoHwyARCv5+EFXiRJ0ZvhZBX?+N;XwLnZzAuBB6Cs{=}g6<N5XY7b4;YSo=@1b&kK
zB#{x<^5yvEE8r^1fCWaxFeAjx0D%M{*>4&~!WzV1r0G71IWnsClb0UG$xG#DYjN?~
z#$E=lO1`n)rIbVaGX)%<{CRKU7~*KGPw83K>Zt_&Vx3C%7eEbgcDL((#gU?uco7nA
zt0s9UgjxZ$1LmE65-26G6=+1rk)=;>e2Bl9j~9icNb1CES0VD}VJ^2ZN+{|{B~o!=
z`aRDS@=w=ak5rBlQ&L@fVwPP5AfHVa;)#^OL_wjJ_QW}hsMQdW6``{c!x$h3Cfwbz
zXmMbSyV?Flw=lfyG*LD%_WE3AB630A54OiDV>X-Zc}FeJgEGYyQR8RqDe&oOC~(O`
zAWh661b=9Zjw6%icbwszuUgakZGH*!Tt8dYa7uEZnw~n|Ln)BBrx@m3i&?Kkv9UcE
zewy6)N<5LuIj~4jx4D=^WkEURmJVYyqyn~1(D=BpGv@0+;h`{52arK*)m%qwMksQh
z-D>McQh^efYE%Oz@yaq?=2IRQzfQ9?r6M7Zc>)+%igV#`vt?I24XDB8@*7Um4!hO9
zEjbwgJT|t6#R~`)F;#}udO;@w?3UA-h0kgjLmMN$N*s4}*+TVTnk$o;-1v-jaV>PK
zWDXYv<|;<<>ulRBihS%$C7>N6ge{WY4MSCHW*UvICK~?w0l`!nn8$Q_U~lLC0Q`_z
zyLGu2u&7ynHrZkbax34PgdQ5tR9c0%<}0IQIe(QCm|0$bn*<M*hzazhe~^DCP@56~
z6)EUxT?<ph5`tx?9CT8Leg*(LOS5W|jX4K9MU!nL6^*02413r;CY{j|&Y@?%hNHE|
zkhEp;EC4?pst}JYwic?QQ=F-!l_6-mMkB{Q**il0GptDshT*2#p|RY|mv8DiDQ`_+
z>Oj1%p>K!M(4|HzQ)EzP#2eNi8c!MJr!E`)<pX<?6EirSgS#pFy?T&4^QMj7y?P`?
zT?e1(4_kR$M>D@j{*b?vX$Gg&2-$}hfW}&2*#w7?9r@&0THc9teE0xvuPiajj0}nS
z;hKu&N&MDIS;mj)>8%xE=~BD;6!fyI<MYe%8)X6UK;)XYK5S1p>i<>PTZgq3v~Pn!
z3Z;0FBBe;7Sc1E|OMxIEI0UyM#flY5(NZk9Q#4qx;<OYE6n8CF+`UjXeZSwk-|k+!
z*Zy~A&Y5{;GV|Q?oO|vwi42z_O_v(OHuZO%n-~|4+9qvoM6d&D@=V4<4n`aAB|qtu
zKSuawkB<aBx0WSso6E{sR_)Yw7S23JvrJkx{biGp5pKU16x7+?Gy39;7pcnQuuJz{
z3ziZrd?DG1eU<vL{PFcnk@K8k0pGqzU|r1ktR5H9MK8%9H9AwVfDfUb{I_knE-W)f
z-jsAq6Ner87G8q}MtGp+EFT&@sZxFq!cT;FfY9ETw&)bd8l1xk?Oo`qah_Y*&vtQI
zGbUG>y8-LSB`_DOYLxRjYOK>nSI0gkKaO|e-_u@dwhrj>|45p4Vjw{rBK|$J=PkEL
zRA~3`kI~YW>LxOmxs~WfjY3D1K>1-sqlR`^#1wpo(g~eJ=k9hmqRs{Eja<@QYybB{
z2rg+)GLX=O&GLGX4h7<Nj!z}<uT_rYQ%{`;-cXD<gTzMd*Nha@fa435YFPTp(bcm<
zP^_yZXnsuiEJuf8zq@q#=xUvSMzkb~?5~BI){tU)b<9R+L+pYpNzSF9*B;|<%k3bH
z@d_h2EzdxC6HiMsn9;Yx3H+$l9b=s|<#eSs7T-?T>$}8<(no>mY~V)$>985r_T@l`
z-2mE>;ghtZL&}RqJ_eFpBgZ&kuM<lcmAL{<mzhoat1^r~aVbS;>4_C5q`gS>p0Eb0
zx&#1=Cbz5%wYgR_i&@0@8t_RzxAW;#K@R8NX+9TXI_aCYO2b}%BE#56-ijfidi4xG
zM~4>IwIvZLY#dw_rLdujALR{_>6(j4bD4=IW|WcnZZL6qCj5O=G=oIYCcAz|j?UBh
zYyHX583BnCyxSfq{UeZ|6kzz$<dq^GgV$DXhuO!IcSyR%Jl0kwxP|TkunPfNi1EV%
zL+0M?QN%aYy9r6A^}9<C6s+&JgsIM)&P8kXoKDT_^wnn)0_5CCt9lvtG6Ijoj-SCr
z*DIG|{a8Vit~xi8JHWlF-<&cjB}F5AoDu`UiImf$=Z-=s<50%SM@5EpLrZVS;I(fR
z+WsghLSB7Z-w&DmShdfnCHFmUp7%W&J@=&>PXb3HxkaUg!#68zeO}p=YdRk0h%rnJ
zUWO-KQM@*UwPfRvTFNu)P(~MDFAR%kI!&SJDpsfXsCkxBX^!#y0K786FeS3ro9sJ7
zn7i`sR&BPACz4ByheS9{W&A;?C#;=>s1E4gd8aL(xM(E8m8#baVjweycbBCNpYvF)
z!B5OMua$1BB@A7!1qqct?L6!FXgsw7s$=;PwWEqh;7~WD)(STs{p0s}5CBjjL_FIT
zTEBQz`u96hRnpU@T;=28qoP0PvX4qn!_V#0bKjHIsbX-4C8*!};_Wm5>zuwU)Hn;{
z8e9Y@Nv`m6pjPu+2b73&LHJR8bx5f?XV{$zM!7^&Y}LLtaABt9Oa-(Zb+cYbOhQMj
z>-TC2P`TX+3UG9lTP$OuYkBthu!lcr0|V^mO5J)>25Yod3HoOxAjgvyenV%GwES#$
z6Q2G80Sj=21U|a7Ln%GcO34K-eE*84HFRm+7r8eWV>}Aax+3TP39Z%xZC`g;__$JU
zaFh9keFG;DEvc*U!`^dfgHV8(N36DU1ePgs?4DtREa=JREYd=X^6*<0O8gz5q+i3*
z6a-l1F_IijIvVY7>E&Xt={dwAZ(#X>w3`}Sm(ifnWpQ@l@YUr>_}wUp0MKuCudm9O
z`zL?g!O&Z5%ywZDUq0DUUd58!$)jBT$?CL%V=?~gf;2Lt-Noya+5Em)_^LxanLANu
zCv1}V^!`Y-Z&)dS8slAZNfG~?i}{zUr23j-Ta&ubxa(&LW+<eW=S%n)6|*cD?ae$|
z<|U66lxidE%c|ZNO?(L3+Yf1vj7?=aJ(biPH!;yxh}EzWqf8K{sWf)FL$4@AiipO@
zA5|*&dO6U6QeO{+UI|d*))wG-mf__4;7+YY5CBCXPt2w$%eT!wnRFE+VQX+HmU4l+
zgOaEX);3;-G<+iL3h+}Ct7GDf5lPH&02yI!SeeO{uUOyF>j)y)aek+(T%huQBBbBY
z7lB%($|DX2!|OE0$5Q1%%8+lr$(NkVK!4UsX{IR?vb5Q=&JR%ew@kWH-P#c_mpO_8
z&_wAxUtIa0a?r#?o@E29rRQG~o!e49&$nddlCCrbcK|QVaBk9H{!|q7uEa0YK#k@!
z2D>3l-fEP;TAnH|@crTug6Zd8cz7UxUy|=KyTn2+;t#AM?s_Bw<GH~BGLzifvc#2N
zwH(=_E-C+~%63CEM*j!9DgKMy?T#ofS^+eLR&4|=)@gvz=pX4rx4sq!J`ftTRhoAv
zwfzv+nZ&+aQ(e;-P|KFw;#OCC$9@=CB(+k(s%yPc7PA;kcRmWAs?@a=gV43ZLi?$5
zV%b`_rE@tPF*Ef)utmf5xNPZBg1cwjuk3<`kqw;{3%Laz(^o$Mv#i?>lpYV>qw>(L
zVM`b9Ps@DneHJF)J*~ZwrFe)84DXLuh)Zi~^LpoP9pJl9YSkmlphAu-b6m-K*dYO%
z_{tBy)%<ZO#z`*}!_zYUXW{}!Zl~p&rV|tiAd5&USf|gUgN~!C>DV7X|8P)g-7V10
z5+-NlVjU?P$}Ip5?bJj`)IjG``%}ob-CkIfOhTp!+b)-56<*>1FJ>*_DP$&OE{m#$
zesq?v(0PiIK%tkI38pJU>;wT}zv?-%hUEJgm+rt%7`)&5u>(Cu4%gfkw#x0y4*?P5
z0lr|Ynt%hx1GwqmvY?K78l7I^E}d-0D@tGKt(L>}#;y2i%^SRtwX9SdQs||69AD@Z
zV-44w%Dv0#&-TFQP|_|A@h8a5UCnaXlv?1r5orKM9H>GnZsBGSZbpddl#S_cfr{ox
z;vBfD1^y=1rWULn;?u0df|SWyVh%~#iw`IwI7k=gNld?=VSlG_F2b95!G=pfukxvZ
z1oH-?^6O%Va5M=2uI~Exj6o8F^Gb-M^tpzaiyP2mnyw6<?E*l}65BgNMj}j`Rg{B9
zkCOfw{Heq=F}=EAB!sZ}XAw~|=X|&f+{2{Sr**!5z5EU<=sek>`Hde$9T)xHvNluF
zFUs6!{5ZlUU&0sUHX8{-1;~uPcgXEM4P@Xhc2HBvntr>U=t}*?wI_KiNX8n~qL?Sl
zQetaZoPg<Xk6Lxktb6Zf4cX47alDP=t&J78w)rW=Uj5sJ`ax&e;!NA%^Yf%rN^=x+
z`>!ixX|aDTkYOe8#ureNQVRbPQ3faclHTonh<h%K>KBe3gwSk%1BHE%2+G`@znM_f
zY#O5aZgdY>&YlWCmiKd@h?4iXt807vaM7C<_p!gh+2=}X6<p;z+_K|xcH<({gH#k?
z>%J{(rXO8#jkWplA^`{8#*FwFkKFZY*J;opU`Esu?Ja5)h5!kbB>kYo53u#4#-)&&
zHX|&8%>?v3lWz`|JYF)dfC;NJ<2)<uB^>D(BZ>yN^*Q@A7cdiNHUA<~ej;thZ7zfr
zROZ)eGZoqvKG4;i(fF(#1PJ@wfic4p6nkX<_9@s%vz;xvMRtNuEUsiON2VGkuWx(O
zNk5vRMYD~+>3YYHBN+}8uU;D&iC;k`My7oxTldmIZ_Jb8VK9-t*<u3q&ki=d{F)2i
z4Pj)ODvLx}>89iI(MFXjld@kZzA_<C)~~>7qccs&dC8=sXcJW!U9P{ut!JZAY}riI
zCC4`apdQghsRN2N7YWH(31Bfz8n3@4iO9MhrhasVN#}VdlsLsAr%V9^#?1nw&jz!o
zcuRvmopj1g)qGk@B6y`7mO<pmeKGO9?^C%Ycl2&zyeNHR)=ODU+$EgxX*ivDc?6;2
z#(Ux&u+lT-_ZdXS$8`0uk5T^#F^TB^{~>MvVSmZVY1fE6#1fuMu9%gW8Ko4*OOxc=
zy_PbFfql0=e}1pW?@jYvv?LsJy(}yMLx5aJ!g(5>N$$Ec)*tKh_wuXMGERdO_VRKK
zovHTn!AT)DM7f4(Ejp<Wi!MLZ*Dr3AwbY00M*mB4=z+bf%aJ7~Ps{OxM?W3iMPKY#
zhz%l$)rM=Qo;Ck`Zs_GIqN#WL&cXOZ;ZN+7{C@N~0$fjn>28%d7ovfA6+58~Rf`w9
zT;`461|s?Q!gM_mwArtc23)l+uJ=5Kfq%QpsLU%6J5yeJdK^_#Z!x~I3nX0cjY=C+
z*Hf-?GkjjH;#`2d{?wR$p;IPmAdYzpcV=6$^}Dfx6@LD?<H<oAvtZbmiZ0<+>U@V&
zJ9*TCv>~qllzw4Z+LIy+EI0$Y^hI)iWOL+BkM8H>p``w^{OUv3X`5zJdOaOCQn{n4
zRi=_mafJP|<oMG*zPnAG34_)C(UXF8W!S~2{TwqBrBj0H+e>AUv|YfqN4A3{EcJsM
zqR*Xny8)TwrbQ;k6jP@XgSky#O^`QHzU&RVdlZ$-w}+(@!?_L=YY59qR5Mh+ge$hx
zj?>66WqRg$cg<t}+4(g83zj7`<gR<=Y@}Ft?~iK}4=y%77C5&Z?zju7Lg+#honU1`
z6$`~br#0B4a_HQltrnKO0GLms_x=5s8sxnUcjZu8rc8CpfjQ+j%i|xmj~o5CLW0y`
z-qu5bhB#%x{Rm?9;o2FWj@T{=IxVNwX2NezYz_XBMrS{I^Td=si!r%~_>9n=bK6hm
zuZUL{TsYF%$U!FHZROCVi7$~uMj)jQi@1d{#j&&_aZ)sTBQ+h0IkX&?7Z5dP^!iME
z`!oWk1GxV*m=P2fP>rzFTpGIlI@yNQD+7Df4HnCo(!N!f>1ZQ!m-Ah)kC?->d#p+Q
zt!nsYWhTGqvosMMRSbU6m9<~?vGE)u_Wt87O3SeQE?I(Z5?nyRjBd=@;I=ky#JL+*
zyFue2AXAj$AdRLSbK3`O;LybBV?L)E@Ul*dyy2S6+Kk%O&Gxx8WSQq3>}26|{!4H%
z!HsoCTXy`WWMy!bhOk=VXF3s$m4_^h`G*cQ7}yBH)c!>2L1;9WqZ_XetLgDAVT$10
zE7|)F!zR0ZKN{F{UlFCD(2H+Ow}F06w!~TH@YGUyp}F-D6fFkayV7~DxduP~DWuZ)
zZO8$`ruEPJ%cOM)(<dD2YgNR<sefJaqyw#rxDKcl(epUH!rRe#c+7@xVaXE~d=PhN
zgS_~UBRJD|4+65jXy4Ryba5@>8a&qG6vc{V&TGpwJFbMyOpJS2`&%+um)9MC|GxKV
zfk@`kH(hA(d^2{z!x6D@zRqS{9-&z-IrFEO==qNU;-ay(9A`6(Z80NBqxm+FiGN|%
zlTE(>zI4Pvj_7oE>UvzBI(u!Z^(sM5NW-?`s1M9VXE3j+62Z-c<K&myS{2(nwQYy)
z`POWCqzIt4|AK{`bh5I)&MI;S3q4r<9$e#0C%7Yx&+V9u8JGh1ObsO+wqE1*zYt6q
zHc(g#RS8R(9xEWyl(&(U6o5iQ6n1YyuY#51$V2?%4e+$2UivXPZ-DJl-ur`fzvGR=
z7G%5=QaAgAZ)8}@$z@kkD0X14FT{J@@;Ua^){}Osjv|Z+T!bg9KOAM3*Gy?8pY9bE
z+6S;sT5VT=y!GeW^(dxE6F53!rPB0|!i@<e{W!re;TQVp1?7Vs*5LmWy!aXjpHy(@
z86wbF_U~qZeoC3kYa{)CBm&BHO^sa9J5s1f)zyqcdQBNX+eH;gh<O5jP0V-v(OTnU
zd`p~lP2wjY&Vj&FyLd<4vGFl8H5~?uPQ;H=iJ7+V@J+JDSRFL$yV;C{E&e|l-mioN
zDELVv`u6|u8SR8D07}w?fWenq)GqReZa>KdKY%)4XR$|bF!bHME{!O(k_cbhYtB#r
zZP0z({^dLB)`0jU$Kft(Ts<QOL+SSB?^A-$Omy^S3rfQ+zh)Ik{4uH0p%3O|xe-C3
z&xq9NROnG$X_Lu%F8cr-?LNs{d1v*!Oh$##9U}2-w>Wk}lw|CzxuCel3a4w#y`O^~
z<z57nCz~3aDLwwvzr-i7gSHuL0ZEEpn2<^?iA~}KD2mZeI-^29_kYWplnMGerP$XN
zs;oc@c$xrix^v1~XaWYuk(RZv?Em`F()y?oi*YLzK%^dqsb3{*Cuc7sQ`z;j?g!Ep
zD&!1|1p6xd3OcHrJNBRbqWq>!1LbPx?Z<@S7xD_O>iwmmE5!az0j!L(-2~lYD_#@S
zo<k<<*aLunV$ob*H<{?{E*O$*7T2S#-_uZS)&nOV2==GMlDTLdo~CQOjqz*-&s$pJ
zeu*-hXvK0jf=jIXO(2z=QiCw6i%h5O(Glq@rAN@6Zw6H)33V&(Zi|xfDI@@co&-~b
zup2+fNhnK2Pi_N~20HzY+UKGdCkVeL<zg(a4Ce0&pCj}sjou?`1j6=zOilfy5d09g
z@^0tX{beB^29I!P75v4I4022#4iYnrYBC0U4ASPVN!)N870M8=xhH_k7ol1vu7EAJ
z#-T>8HR(<RlOeG$_c+kx5LK&y03F@K1cy0AY?*{pbU1vbZvm|?P>$cedP89Eh*r!Y
z;F&N(;w~~(R%ggy+Fh6kbaE1jp0`&4C<(x$F&c$yuvt5y#I2k`?A2*i8SY}buYWMn
z=2Ml_O2FsR$CJ3KeqnE&(_JxZ#JlR_FHdV^<8yWKNJ{FW2yCDc88e>;hH(WNhbl0S
zOW3038h~_<GK>xCFItUxg;8r>_5b>;A+?`H=6D&!+dyzNu96d|V>Yp9ai{Eeu<c_Y
z)1rZbY7RnAq>Q)vNkK%kUfj3Ytbo@mxw_Q0#zM?Z*mJ|U=V7qX1>N;Es?kX#b2CKf
zIF!>vYaSWS+x&Geht;tWHcmnnBaLd+H(@~c#;aTfX&$6w){vV;6?2rSMmIJ4LdZVO
zi5b$mr8FblLx7_El}CgyUwkJcokgX?;2jS9=P3J`Ea&7+doh2K8iAG?9pPezljN7i
zV8elfk1;7y?3k#_k*ax|_qSGr0D1pj$GH~8_F(>+*rus*P9|o&O`SiK)$qxN^{0oa
zif+wvaXIdm*id`$jo?c>*j#UBnvYX7mP`r5k~@3x(udA(Twwq-?OgWj%{$_z)B?<k
zGU|wYyVmGQr_Lb{%=Y}pni)M!^dZXaz)`x*w|AU4!%IjNM*rqe1GhBOEH7{3*bK&s
zc)RF~@9-!#71vYMOfv6~_Z&b$>=&;HaYGaohS?KBdy06Q)>AEi@RLs6Dv%{}$~lMv
zYRqXAI+PSF!|d@i!HOl(^n7Df9~!Of(d4aM3&tZoQWWPtpSPugjQ4v+7Yi=wzn3pR
znPxyQ^2hpdC!;cfQxtBbs7ZEM=pmarb59;Me3RyHJ6*RUrnwEz0EQs(YwxJ_@9=wm
za0JM(heGPN$LyFEwjiySwO%Sn+#4K07+D&ty(6f!NhUjtQ)xsqi_AjrhH*f9OC<~&
zWB>89)bVL(w)X#it>ag<lKwC!-ZZt*!1jc{U6(_anN>43!=~rC1X-ErYf`q=7}|*w
z-2i2m_fy|TY-d*1XoR_n>fUMee&l@uPdqU<SSx*3Elut^Hxdr*)oF%3PdQ5Y#|K+B
z!`}5Q@_iZvt)m}UMDMc^p+M`LTktnvS8C<WfK7PAJxSN7ZO&O2XVq(SxX@l)E@i{}
zg9&yb2}0>~(%}>3+ASk*jJiA;cqxlB5g=YxtSN7X;dMec9WW*eE~BQdJQdeh>H0y8
zghkm@I0s!g&~9SKOPwf!%*0lM{cop0K=AC+u(&Ndfxq|C*fW$F4#HDUBcfBbMa$FR
zW5w61ip6d|8FS1)+BDtkuTzz;4_H<|pVgOAIHzKCm2><+jTqrv5V!6%r#oAy=`jd%
zOg_16gk7-;+E}k)z=<r(J2PRe)T4}7!r++)Krgcd`)&g)M2n_a!8G8L`;Yx>Mp7M2
zfYba?ItXKK96lq}t8ij`7>G!d#9)|1`2018#t$<a{QxGgSjS%w^{?3TB(?99wf>2G
zl)1kdXX6}+I0IzdGZ<~M@fBDXeMY_taRYjSCt}4!k{lHT+GPDzbraxyq89@E+bxah
zHlHI~n4qvG)secJtR2)#BdwARbz+tsNuW9f`-IoJFt|y@JlM~8(g}r3$MLcU#S!W)
z7^}^a*joW)3i}q{*=?z$g9G$?XZ75;(`*+Vb_gG<?JJbv8_R2Fec_AHZe3)ObdG20
z;+a`LDfQW{62mvu>4q~4C{{lAC0z8%&f;+_uq;zD)s=w3gY432M0bGS(kCM!%U@sJ
z%l&KV*4FR{G+LNqh>K+W9X-xHv}kkRidkigL^0OYhFKje!v*E|CkXy4B@*wNWs1V)
zAxei6r2>Z$hm>I8+;rh;hHQRqu>u?RB(cX@DNr-9;zc@l<<k^A8hwQUXpzOLSz**6
zio_#{0YqbJ+nabbDY07&;3(ngJ{`n3p`gkT={?Br#Imw&ZfAJtMZO^-*iW3a`l{+L
ztD9h;?1DYnk4gFA0~qU8%rzkg*Kq>?qUhVd-KK~!QctW}9^sp*ACFu9OE+~e3tII<
zx6G*0;13%p3BbHHIO6UA(c-cgPScnkK*r%dw|{?;2}JerI+uz#mbg~?bbbB_r{17k
zx+udS*NhA6{mF%US&|DDWv5_GgUU$ZTy6wg&nYtxMw=xX69}tvt8k3(md5=&dXu<3
z#}^grH15&G9kOC8V#^yYf<xreRq}HQcqRnc|65(^S~%kb(2n*G?D4gv&gyVEQb%Rv
zE@h9_VBA!|R1a1*lu@=lZ-Tp#v3D_thZH_b+_;yMpci`plLTHJUj64}KUR#k{BVI6
z|NFP-b3&QSn<v4sr0^c3G%)hGIYL#|0E*8&4sGXDuy_{kqjwwkiJt#@mcKlLG{HhL
zbOeEDXJvjL#K7>0bE6<c0CP}GXmWT#>%zz$Pbo)SMQC2UC-PDwgJVJxhxGGG5NM%T
z6ZY8NU)7O!k-43rOa--Wh-&N_{~_ogx;l#ed&Bx>HGW=nlIpX4Ffodtc12<3yy;%K
zBgs6!H(EUVXOf*_r9~#nbD|ES+n0m*p6%*V`nBKTch(`h0{1loy?jq<Ta!<{L!>_+
z>h()JM#AR~K`jTAz;IQsuFvy4`<i5?gBW_m-D8L-Gq0ACJSV_A8htFI40%?m^l1w}
zN374}s3?U(xk{-;b*5QI5uTpABrubMf6RGe5!1cIG_;OD@AQQ4itllem~PEC!f_7+
z(~o7Z@>N8>2jF*U<%aLIwsF@*swfp!*Ui0L(Jg&F17`NqoqW=GEA~OFo{CH-wzms@
zU<eT3!e9e{KSm;Y2kT9<ry@&R8OHUPXsqlxFOi>^-j`rkwy7&LgxEJQ7hny=^(jGP
zBf(xwwyu1D^gA=cRTwKrp`H&NtpPTmc32``PO|$2nWa^oe(JKcD!=~n64O7&%Zc}7
zI=Q~$NFWTw5rS^a6ivyT2<SC!()M!!3MsI4J_g61xD|L9)y3?EPX>vALmeNolcAT-
zbVQx|Os1<V_CPO)r@oK~t2V>p;|47~nRd-3*#ka)luwDY_lQY`zWwNq65(iyEj(@a
zK>5;&0Gnc~)BZ1Of#DE(3HY}XT=|#WbuTyXzxw2A3C^^a&Dhv@wPLj|Q`5Xc`j`QI
z;44m$du-WSzE&TTtUeuty52$UZY4YzIW|7s6wV8O{eUQXJ!LlW2?<1RaFLfIHS){b
zIZ$9)as*PN(g;%gItBJb|3Dv7JfCNg0(0yc<oi$#)GKwY9z^<Aj_(0-tO*)fJFT4X
z1WHnR_tFbqSe&Ux@xSEs)*Ui0mN=^vA<n<NFf&b{aVX(BJq{eTHhx=I->Uf1#zuf2
z9<Vz#1;cxJ(!zPLtD!?&1F58n_~ob+^f#cn7wd1&M(h!!k^N|QI<?e<U=bfMj(xu(
z_o9Q?UjuarG;$KEdlp#~FhAAqzQ}?Aeog(^jJDn&`nu_6NQ@43T?~7fs(a!DzQxjQ
ziY=FTM}M9M2bp2m{q(|2yLB*ZnjKC3jFJuVlPVp6%d+YIRz!69eg)Zst|D?GoiR)m
zu2A<*_QZpme<Ny*J|MT^iZ@%KvxI^Nn@_JL#GV>Om}fg>5Qa{^M+&V#r(LN&^~wDx
zV0{`i^mgeQA9plCq+J2!`n!X7ijfY@KQ9S8;YTnfv6ALukFGD@8>xibV!@7zDroij
zZl8v<&9sQUoU}ptCW^JbCx6x~Y<J=aF}bR&pzBh()i!l`Y|v?1xm3#u+)L%GBeflT
zF>72h--;y_-<&hCG(EVksTb6hIh#6yj0F8Di{sCk`^H}#S@f6J{b<pf)%J1C&rj_L
z^LEX8gmIa2BX!~vG%f5c>hO#MdMkK5q=~ux2Gn|vKN=0H({5=okTEe`{IZ<|+Wp2i
z!4pel^M$B~{di4oLCPdjpCw!z|2wB|rrhf=l?<M}FxxOQ9HKsBK#qhQCztT*Cu>yO
zsp8!wZQVdy${(uZ`sW%~jlz_oXE_QeE~c)@33SZA2dz0%_u5GK7J@xmXxvCU$#qR8
zh1&f&66j68(5>dxSc;2^)*)XCHk!ubGj^dmf%@SS`tCbL?EKh-y`=l<=@<P_t;qGu
z9q}kno$~HiN1N=+M#L(n8NMwRO~pv)d#VUVj5o#${fu*bIRO;%89O1p@C!AWU?r_g
zb6Wp@o#k|lKW2T$?F3x8f^7q?-F}B>yzB9;i46#J9_nPP-g8aAD{M16_nCGVc`*o`
zPi>n0Glgs+9a!_s8x08d!>kb)GcFcVt9ND|ey+q%H);+n93AC+rkdARuOVFSEdD1T
z_CT-7l}d5!9knSXW><PHt0NM2`}oy7I_&4qB)Ky%GiJHG5kZIjPBooYL%s4F#_+m7
z&uchm_Ujy47UV25HEWzni2`SVnL8nWJ$h7>5WK2+%JutGzWDMB5As3WtuNF9?oGym
zlVi94InIo+17TN^0OVOQ#7NL=!2k<0xE6y-HT9zWZH)9+YIt_;o<6rao7l~pb$AnU
z`BDu)-{<Vsum#Ea>i(Xtkvw=vk=4!jZqjUi#Fmat#hvQwr0s!XFn$-0MPBzXe|%Cv
zMm81)!p@Z{LbBRnH@nv(=jnEw0vPsU<vt|GD^X^J-6x5j*M1&exgjYv`^i!=rv*kv
zI>&dy*)aAWP+)or2xqeDjI3v{c7Sn)e{}dlw9TZO9VG%RPI;!iWldVi$uxqMz<yI;
zJG~2_S5_`^i%~Hmg;+YVRO+vB%iG8mTZ##xdFziDd{t$YRe&_XH%x4r)B}`07khQ|
zZ`u~s=jcUKykz#6P_$kv`yfC>rODH6ahTRQ6<D(G+B~1b2)Z6aP{N*9jNw>0Iq|*Z
zQ@O?27w#Ow*5?iTWU`t#Oa9{q@NBX2_k;?}BW^q3-kX29Ah=61j9vNv3kh9b*9`JC
zSi#ce^x^E_h8gRvhyCmW$I~GirRIBmoucF)KbT*=dX+)dS$nPeB_xEkN^?f_;NGT*
zP<!R+6X(E3zcJlOa44H*zVl!!E&T&y6K*;fS~Y?O*?(BOg+|-|*bw}$4M9U#x?|XT
zeRvru=l>vfGY86SA(S5%la%zKSb0&UU<z>D&9?I<xAc+of#T_qyW;Qn4%FkDhHoOQ
z4}LelVj|<BGqt%67<Dp+J1pe5U4KQ!uK(49!X5T<uEs4`&vJIY`a~Odwd}l+c;RL@
zNwk6z6uglxp1;e6RpwP?N*AnUxI7DqdH?BuRWeQZlP8lG&TAuVx|ARhI`IwP5*QPp
z2(oCv%5%FO?1l}ZxIr;wZR;bY2u&`xIjx9J-1cIaJY8A_aKohY<m0IkpAns?RvydA
zQXU+NBTt%b=ddvR%P+V1;1L<A9~KT%7>|+ND1Vs6k`;%GxJso6L8D!1fWvJThPk@3
za~(AoQAQ&+^!DxN7g9hcX3ah;W-H`V>fpmb&4MVR5H7!0U*OrlEj<N8P$66Jn|xmC
zUr|HaOhhbsGm`iAUnpA=sFId6p<9l9XL)nhcH4FfZ;l=}Eb?g;=&hjMfDh~ovmFvE
zhY>_Pi5F!heil^$tpo5|wAfIRX5cHSQH{p>k|X{*Pz71#)g*CoWmTJ;GCZB7a1p7I
zDFl?av$g>%JS?t?Ei5lmVM=_Rh@(CH`2~K!)Q%L0Fu@jRa=zj_Iw)kF4y=jQ!`n^)
zLAA@B3tShq>Y!Czt;La#fQkh;SLpHB^6B`o8cozkal9C@{IuBi<Z;h*xFt?D$-Wf0
z2)|CyUi}y=&*;*nW`j-_A5JrxvjvBQ;*a2g3Q-c3L9R999O3khtcB&jbRE;77z7V;
zdE(Saq5K#A+ozu#bTR!+QADzjkc?S2?x?B2nf+{v*vPO$*CUA~kfY^Xt!~-ZT)1XP
z#Pkt3q@c~ZhRoO8Nwa$ZrLJx@+bFs@ldpo)gLLbbF?lRbzuAzqhz!IuVjt2-?5igH
zbH_oEDX|0X$zSAaNh)Pu`Zo-f5*Ct$<#fnm^M97DUT(eI*@9ODm?RKO&SB;IXqRqk
z$9{l){af#Mo!_Q<+IxqtmE&O;iyT$Zwyf+IU(pGg$fTeSbeYxu`z{MG|In(&!qwo+
z9F$^v#g`WhiO42kY9ct#fa||!X$W0_$iH29fW4)kWxWYWe||*Pu!xi;I6KeXtfZBi
zT^Uq2-hQaPGgv0)8D2zcu6`E^b#xTFs3y#0TYp*N`e&dq_a_THE+U)6m>b>tzZXAL
z<mdGUWvcBVoC1C8*=;-M6FR3|@0dH!c9rScERJ^v*0gl+XMq<l03xcXAMbYw7Hnqt
z$!fC4@EG7Ph!+OZM@_siK>@y5S=6YIc>z&=;=AQutZS1T)_7vC&G5ZhZ@9-h$^H=I
z38sl0T2p!4ORHzJNUv$C$`!!&Y?>4hxcr+d$RzK3H$w77ydm6;;Cy4#opJO7cy=J)
zPFcH4?VN8^I{Cuyk#t8J3~S)5_f>2nt+ehjYr^_eslS#0^&-0?29%7H)`Bh3JTXzO
zTLa7UakqX^@ZHy#^vG_u*c_0khGKnAq?cyG=Xo9eTd}U+x?~VH1skd;3FM>%=)zu5
zaZBK7_Z%R$NXPm5%=<PCgXioGhdV*kX4FQl%!kH}+VT-uD)BMcKLWg`PgQF5%hbO{
z>7RJMzId_T`Jvn*l}5U#u7!HiWAw7nA9_8lL*YEE&fmcrGcg>O2M_CjjK792p+UJp
zv8cCcd#wKL%hJufK8q;i{&yWT;{F3(Pc#+^u^GYAX=>5&>wlxyxc^44!xtGh!~u2-
z87)sg=rqC{;tvjJ$&wrq8P`<?6p9VHqB5a(?$-M!%1ck@BQBF)ul72<$<fBDfz|p^
z5<I24M<yyw1!#)RBW?bx{kTHQJzPd>=K0$HEBE++*JjMu8Se>`fL)0Ldlt4H*{C)0
z&%WSv#Lf&%=J6aKXa07__vL$&)ynnmUOS~C4RdyJ|L~RUmE)Yz;-3^TYX)8E<J}e}
zJ+=z7au^*5UCw}I<;u_#88fn(L2yHyG!Qs~jH`1pHZSnt1LgqG2SQ4#H{VEYn}MJZ
zcaY>)QD0P7&i8#he-DrPQc{F5U|8Y^!fH*uQtxb1WO7TB^6m2vP$8G_3hq(`ES<oH
zJ}%c9B8IQtEdF8=UG$m5DFZyUwHa8db@ILg9TUs{C>0Cy+$N&4tFdG<_@;B{yJcef
z-L+`4oBcSbzNgM(>Q=5y!gu`md4q2QVqtKOOPw^PBl1Bgz#MTA061sXke)`O6a|I(
z8K_4``3fC{;IRtDK1}*nGFSH0pS{KHr2F_*ZLH`h9NS=vs!67@YN(;4_B$6y7BQlZ
zuFD9==gJX#IuGh*Qp<@}z8c!1v_-i#^sARUF_+`hI|4ghM3OY2Iloj-ODG@_@Oz!4
zWT#8_VSY@94nmU!!z;ZBT%!_J*B0Z>*an1|6OP{mE9vMc@^-x*TIxbA&?P9$oA;*y
zam|J!-H&wE5)<x=q4^f!)^W}N(qtPB`N-sTHl~Q+k}`p;=6;(XXv4uCkGKJVUl8C%
zej+3`Y5t&P<eDdf2lC8LI{qzeo2kfpCbU{kFoafRfcUorYvI}0Ar8z~G{!<T7KWgS
zBxr;@y6aAJE<RxVpwBSu&fj%ICk0CIl=lo@gX$2(MPO_4<Fd97l#4{*pY`3R$b=Ur
z6wUa#@(FtX3=*iyd!HMdRZ|!nuwlC5Zpj;0T!qSACr2x%{S>N+YKm3sgeSe_%gcw|
z^YNr5>BO96kz%#jp+a^iw2=g^n`@=7*WF?hU?#Lli)%5blrhla;FK9cGi1azRZ7#H
z@v`niA#B)$0fnwr+IN61ghJ{|L2oTQ?n^e~j1~@>=q5K-I$$R9%7s%%beY{fN73M1
zCyME!Cr>P|#?;E@Qo0NJK<ZSjbRLDiPLva&xm)axd!$Am(&q^4W?r;<u0G~pu=}aj
zc_+TS#?P4*>-JZ#iW&m%O<q8~vtb$a$fgOvl$yQ;=Fr+^&4+#~o<dnG##xN$`w@0!
z*F&D!Iv6zs`BYN+PHo~?0Iv{+aq8vWF$vZWkm_59cf9>t#;FpbN;4p|5MttfDkFOF
zja7j7sjk0Sq^!{IQ)NjnPFpgz$Y+FE<$gY*vzR=mR5b27zux+5wm!@M?r(M;*m=~b
zSvccw!i??Dh~W(he+WV==W6q~lUsK@A7|9Vcq*r9tkWVRy=SK?BUL`trFcYOC`Z$L
zvsL_ob*#wzz%U+~{N*|7V<w~$kBUh*H)a-S*MKYhj#_CcMg_*k_Y$gWTZni3wq6m(
zkMzWeiSx7^AqETDR46VoF3!d^Qsbau&=;86B?$4O6%RR$t#xLvk_!4m+B+;EGF5A;
zj#JNm>Km#f_(wfNhm{QfRxX|NCHB+`?_cv$8lmcIra}m2scMKZ=Y7L*exc*6GbH;&
zv}9fDugvRp*`_M52G~cmdP9<Q;FP_6o;iS_^dygkHNFw}%l?1F!T(QzSqmYJ0kqf<
z@joKCtCo>F&SaCfS@3^f=<2v*c6Cr!2>rx<^Nx`W6Z53~TYFU3>Ty}^BTB88!i*_b
z6wIGLfvVigy?Q^81_2~x7&cj^IwtPzTuj#<cYjOZ{PBQ09i5#3zE2xt=&q{{C-FeW
zlE|SR=flEDjFldPUWUU-<?N{$>tjibw}OoliZ!zY#vqi8Kcw4?68e@#t}3aj$SY4$
zRBHdqqqdN$;vc=P2KeOq$JO#6xlWCyvUFc5Gd=mz1nPlh2}i8*sK<<YoX7!pk)v~t
zgzS?uOZFIUNO3B6%VW5{u5;dYJ$#L1(_nDB!$|uUcNG%1prvT6$7o}#wc(dLiS$-v
z)%|PeWUoN7CTB<`un<^}@ETq&h0$>O*m8NKt?dVgrc_w=rKx&4i)BmrP)H28Of1cH
zb8s5(y9~7vyp@`Lbss91u3DiT_0;e!-8iLS6?Dnk$MMa>iHu5>e~K*s0cpTmxyHdO
zBcTL&W6O5#dBvrDGDMAo1*8qp@QV6?N}qhATE8&R)u-8eL7&2vzA@=`30bx;XS^nm
z(=9wqMH;SwjXmrhD>aQQ<(R42w;BG)tdBRx)t<^uRmfm?=9&C5jVzJO#|ZAT=@N)|
zJu6iQB>U1t@YXzTx91bdQ9N8Q7Zg*$CubT8Qw2RCA8-PR&pZPLjGGF)!$OU67%bra
zu~{%%(S{$43$&Zkf%wzCO2b~hC!rVXwH@Wfp+1aJ13BpetX}|4TEQqv!Np?~G5mtA
zFrECkS8|Qu`lie}ONG`XN5&LJ3OuX3CB?ITPa>TablyO)W#2DL%<^WNY5~RHd%DwD
z(QxK;X&Ex6Ej=`}3tbwp3~v;>m+@U_pY^Dmh(Cvg?cnZX>#IDM4o|vc{+dz-Qx`2-
z;-~2>AjE+oA;fmEK?ab7qx^3`y{s1o=vYMVK65ESoKur9i4Rc@K2&%CZLb=$sdQO?
z|9LMeMW*h7EuDzDChOHr&sh)=wP(!Iha?w)nxSj=W21nzD-x}Bk-yn#rLa}WE90;X
zQk=gp!#-0L%Ci6LsNs7vy0)Qd6vj5wLTeR#)o_d;QX@X8(+U_mk&W!*zVTVv;MQ9D
zMUv#8$`XE8348NS$Hz_S5#wF(|C-vD{?;8ta{a><H<z~mE*6A6FK#dOr(#Jw&My67
zK?mDANqt(QIB1vSdGaZnTw<^*AUQYA;6F~-6`cGGAjTo)^MW&y6aDv&098dTg(^9V
G_x}g|c(u0x

diff --git a/modules/web-console/frontend/public/images/summary.png b/modules/web-console/frontend/public/images/summary.png
index ff88438b28f0667c0d536c43b41fdabdcbb536b8..fda0abf6cd3c950f78e685d047a37ac62eba6be0 100644
GIT binary patch
delta 33103
zcmZttby!s27d8$PN|%%jARryW$k5&0ARr|I4lPJ4b(9u{lEI-nq>&bI5Re+WyAkQ`
ze8<oC_j|7QdY|`?xvn#F_L+UoTK8W2UTfXw_7d|b467214PfBEQ#J9xz#ydf_rb(S
zNu$BQz`*rX(^A4+BBI12dH@|#R>r_!!cbF`d+~l|CmrAXh2eD<srL_o_Lv43e#A#6
zX7R|7Z}t)DanJbTzFdFFVvgiYY5HQe*X)?rkiV{NTmC2P2VBFUpeV8KrHL!F;uJY?
zU2zUfNO<f6&xqP#m>GW!NZ#$I`Ryei{~Y{~ah>`XYiY;5m|DbTj^3uwJPq@m^q-aA
z(R3mz$-P{5OKneAJNvVfetb0br)+TZ_?Er`cUxV5UDCm|+8J%o8BJFRqZ4sSgSgCz
zLgyOXPz`RYo#<0#SNiMsnNfRtgp*JHV0T2P{^aBVepXKfEDMmoxU-v!l{p<t?;spJ
zj$i5Q=Y#*4^W^Y$BEM{>J)hkugq-*K=QX%h6GCz4=3^Is<)yO%?_+M_ki^J~#SoW0
zgxbO5^Uy_elZ!EHpYw7)aNFsdZwbA(QG0~0k1Mt24&7F5V<33=Cl*uLPhvX(E)PH|
z^702TI5#NeHGI3niWnK)+xt85<mcwym0n?O;e5q{M<dB(qR*Tqqo#<`vDKf>ej=z-
z++zui=hs2m9r}z=`@?TM(hMsPp3s=ehQ0M$Y*#ZQ`k=XZ7ASbN{h{tN+6K0q5=+|r
z*F<+3Qd0qNn~-hA?!g{|LdSqQvh0w_kOL@CXt23bp1_oD=aepf`s4x3Wu-GZ^%O2g
z*aAE(r?V&+`Q81PTpb!UDE>rNh9N6;z#VPQYWI$d<QnT{PXdXW)IAQ10=}C+bZ;J#
zt2gGHlTx)XhH^GVi#+^qB=hz6L|Nr(M%-?vzLj#(otzTYu#r!pI=s?Ro027Sz(21h
zz{M<7l5YlyAeXp!g1Ah|(7g&B`A~Tx-N={C@2vtfIfC^5dz=C;fkqyS#;M!n!i>fs
z^VECLxrff}kKF)L#I@31z1zYtP<*`|j33&S>ASUZNOsHaGS`q?(5GtkXYKT_b)Jpl
zwz8-3%<UX;bNTN;VqU7xz@W#Ez(+2P!ac7k&}i$^MWaVV8}vW){0};4Z2d#e|4{XR
zpZY(5`*-R;z!^S4Vf;g`!~ccA|1<XgfD@2fha0wOiyb~%>+B~$rM(qFn9>WNFV;v-
z=;e|QnQJgnNKPob7n3;r7DFe}bp^4r`!~^qp0ZRz4Oz`X@;`W2FCe6p$}-B3nvTED
z7N)ZCUFM20q4#B7Hzs1qq&J!Ps5`TY5%?{_rn4+eU-^|enXPtbUXO92Hc;RX`ruZT
z{zyU)vwt%~;QsXw!;S{coE@$!h06t^UFO144Fuotw@)a2qtEPFPkxGj1}f)@^S~w0
z6NSExp1<;GB$^ri*XwIN_0K&I5Y-3w9>Xws7F44mHi)^+E`bBsw0i66$OS}%x`v-c
zkz#D!|GG0b_*BW0Fes!v3*<4riI{J2BQ{_c6KtTXu?wU~Y-%I6OxjY~Ozg2}EBB-%
z(Yzap9VBx(%i9rCU;)&d+=ZvbZc)=}+LIiB7|orWpY>7MS_3u>#DJ%h&w|=}lev%x
zPo#z3`K{}lfY<N^X|Hc5C=<gSyT2}ye)lW9Fq6kMMMNXz_OhOhfHxH_Yv1Jzaai&m
zs?4DF&L)U+p2`5;k#TqQE%qwao{4@5?p7@WEd)#Z9M)t&{%-&E?wnWDw8)hFz;Wb~
z0M1YZyo5AN9^0Tu{6Qv<P<hhNQFX-_FFnfLrhkxWPH1D8r~tvQM!MCg&4b~$?x!=v
z80G0o_spXiqWI?k(j_DhggV%pLWimydPFw9H0Vy?5<StWUH!h!UeLSiqX82RyFzTs
zj3I;dH{5(?_9>A0%*Pxu6nIRfmU)(G{(d6?4UyL!{gluvX&Qa%<3a5UD)E%+oU`AS
z5e+9i05=y;gBz8c_vuz`Q0@k!g7Zj)CnlD;r!=<JC@`Na`N2XkN7K*uXsuuNnqw!1
zVC1?t9ZcF<o?hbP=B^?S^6eETmDX#R?m^6(M;JR2x~jv)3N6x_y~qo8?hp(EjTDpG
z@YUWvUGJsd+*+5w%}?=M5|b}|{>&6zKW=$0mn_Ff#Kj}w1AkAm{A*Wr_p?*sGI+A(
zMq6!q9EfKb%(-8ZVX2^wfc$-MPeqc}q_;hQkha&N@w6@>(8sx_#0qW!$L8$W>tr?=
zF@<F|{CO|}CP!rvHXrVMuI83DXqa~BuwUy<Ta|9-QBs|*HXe{br29=Mqi*))fCqm)
zrp1hxtE;(};RGulMl}<NP3<{pMI)a|A;#U$0JwZH26nr%MoR%#^H%u&Amo0l{1+pW
z41dD5!n?_*DAd4ih*WS3A#Po9Ty9=dxW!ZWay>kKbpV#yz5@BHU(?)<nX=tq#wRiu
z*2C!m$#Gs5DBx1KZ0(7ws^qaaxZzd$Ad>(GbwN25F7itzh7jqR_jY(U`;A|!a);sp
zy^(S;kipkLH;%y2($NN(XAaR{{uUTaDnLWc<hN2yk7!~~bZWF)GNfx`2e|Uy>;U6I
zp?|k5rt!hMR6T*ee?%lfn)wuvnhhW7h;rfs{}@$)ENz*)UT%*`)w$=nO{*<eCrr9F
zf%Mk>SCw0de@~6mX`?)GV7)M|CNF?O>HRw?-z`eX2sP9|yH?>vWPGZHfAZli<hMZt
z#lWmWWerEh#fXBA$&jAF3~4Z<?8*;zJc=sHcD1%751%DXZN=6h2OAV@AHm;<@w|^e
zBc#P*6AQ+PJ@D}1!^!ejO>Tpt4)1TxYn(e?HWHT6j2F6I|DsyeZqCI=9hd>=;6(w|
z*^K6r87ZH0cPEkg?)8NtEoT6iebK!Q_(1}`4R>+T63(R-CUXg~Ffk^~rwSQFHoM)g
zyxCD3YI*b+C_4YUR;J9*rbs~!4Lk33DQYWEb0U(;({pFY%HX*R*+V?J8eB_Y?hm}k
z-ci|1G_$@+Ot7YPesue4TLx&s+;xDj52b%B&9~eMYOcif?ajWPR79gn>?5T3HbS``
z6?e93?)h70yU_CK4C$K*-LMrf>m<e|2RrcLo_Xyd6g<>l5|sd4!hFf1;MtWs!fKav
z_A=~?K^i*anA=$F*t;sv`q#qell}J)Yo0V7^Nlg`)dw=mB*O;;z;h5A!?LBWs5g1-
z{-*`^J@n>NkJ4ZN-O_<}`)WXPLOjXTXEditO*e+wB3G;rXQA5v_Lb87zMe{wPeO_K
z4PZ32X>&_Xz;8hL*#0}mU1u$5+lwKJuLMf?Z)|QK8GS;Oip{`#^Ui3qhv<JDZW(Yd
z&@80^e*C*Sh({TuAGWRNaY@R6Kob>SXlImyldFM*SkoAq515H{W?G_eVKo56_~8GC
zRQ=yu+JAiO>|DQ7Z>_T|QunMibj-Pg^a+2!L9S?f>7yPDdpq2t!4xy9oM$UJ!1Xo5
zA3gTDo5g!-T0>v#?;2%=ni9kQ9Sec!!A(t=Rw$bOge#fOGMMZA=Wb!62uSvr<nSGK
zuHQb+R`L@J`>~VI_rz7}Tge;w*s~n=akjLQHA8+rEd$I$8F!2K3xAb<{xEs|_g?z&
zH;~?M@b#>}|A;H<_Qd}E=gu++958{XPy3~&fOP9}ygc8<>ZWp5zk#3>?fwea2Ba-l
z=!i>b{4p3O7B7MLjoHET!Cb1d6YxP+mvv%!9RBJWYjc}q$g;HW4jG1uVEpoj>0dnH
z=e%I>a@UxTq8L==YXp<CBa^C3!iySw@gR{*@dUv(711wnPiBJOdm7yGz#W^lxBjqP
z!|tF!Xl5}byj4MH$bkJM10#ewJrN#1k&nYXz)E#zMoGd-6_`R4^7l)mbTC}X_$8h(
z%$adFN{wUS*&{6j9NE;FN;lQyrx70FSY9QCAn1nsmoXvlt6fg%)Q03bj>>7x<h}KK
z>YOggV4?eKtdBo$_oMI40Zk3gmsghy74vIwwwQxY8NXE|{1Du>j)G}xN)MUL6Gf%e
zRW)qgKy#espMVd?f7hM|PdTV8<vZ)if!3D77bof|^P}I)tC`vnhqPMOT7GWVrrbFD
z$<|4UyLG1K2Acib{wSD5{REDm${s-VHW`9fZ)U5|pnSvai@TS10Ial;>G~c5T_47+
zT+RwL&2C!GU6+!n4rNZ#aF&CcD-z^P@7E5&AJ|xXxq=PMM5{D3Lj>_%k%ae)p$jwy
z<~n0`GS&qut+!R7B8%jq0jV{BbZ}`xpf8>N#8%Xg^ujTt2Oc$EGZ;CvKZ&0>P7S&+
zB1^H_i8$M@@ve0(fv<S?tzBC=b}zL-M7^dO?Ui+j1wWS+h%?P1u4zarL@h30E4Fqh
z&spj_`3W^f$Ai_JQNHE}WKc#~ls~20bi9Rl!|OhEZazM~5DNM)VR#L%zMK@1=00;1
z_a8+fm@s6g&dJ}k7C^K>D;SrM4^v8LOA%dzeu6$d_>>v~Oq800!LtKuuB#3mC)B*3
zv7oNzntVtxDqf{5m;VCRIjo0{pPbI8%v1R&eB6V{P=ux48pHCH-lV^{B23BJ$_H~O
zCZlIl1bOm)p8AO)%N0y5d~IQ48HV}3EeM>j0{H=EDHA|ZBAsJ7&!P5=qsV6@Pm1)+
zMtM3P7YG5J;fNh^-_*$`jo$0%Faj5=_#lJQKV!KmxOP#V#=X~Vy=O%|pveRAfufe3
zRpKd7c?PHvr_YT6`?MX<f`DX=U=F<;@KQ7hYZpz3I}DOrR*gfnT4llfM$|j=3xpcc
z6Odf#*14oX!%G1V9sGoJ*2a}Oh^U&9XGnCb1`?}&ckJ!$Ss1Q!<@U-&Gh+B<vlxH_
zRC0)#;}DL&rXRs)6ij}#N$A0<lE~BUG@hxcb3fz8;IB@I6jsCN*7`;moO_mSKOeR(
z*<{HsGSWDPf#TmIw}~C1w>jJtVENK+`<R`I+SU3xi+=SR8!GHZ#YwP>8NFg4;f^z4
z;Q7n$uS84!ZY6V(!&8naZ3Iy0h33+_D9?SM32~ujlLAp{dOru22yP0+NT8@xCp@%2
zEK2=O(%U7ruYo=Df60PSEM&Q<V~?=n^S^LQuWYgdxTIaVSaJSKM8nHzYL#^UB|{#Z
z(9I~H|BhIp-|gy(MWMi4gIFA7xq31H<KNgWJ$Cx%<b)poI6ViHtja#1gYX~SN3V>g
z{wFh7|Bgvuy_$Rf-)V{{^xv$-PoR(J>=T`Kn~lo?4Xzv)<6RDZR;RqYj)|;sk&(Ee
zrFgNu9%{4{U~<h;Hp_3TpyU+N<pP}d^NTq?y1z+9MTg&3<Es!Q4XlB^Eia<Xe!$t*
zldd<&1HSVna8m4vl>gf4sG`^vk^ZAo9LqD;*#9+YTaO>btiME-#Kir?xSFVUKW~l+
z)(F#aOa`G)M*$uK2p<c94{ymkbH0xI1PBB^mYa55gq6H;u3i$$mpSM!A_Yk6XodNz
z^q|+j*RI!vJ5`i9c>EDTKz=SD(s6F1SyxdZNV&sgau5ruG+932X+8*cu%v#mq~G}7
zhYig!2EtXe-6yWA2xyWY8NF(h3G`fjVTJnWVWe$SZO}&vZBR;IOQM3a6{OuGPr|g|
zp=HXrW^lT6Q6~Y;ChuZF003plcE!+&Ip~#_xGeA#SD7?ue5Ny>mbNMZCSAfG{~U3x
z%PoignS1xasPPQ{yu~D%>w4;7om&cTZPJ95V;Kb!sJGUfJ*#<$3K<E{rsfLvh%3Ef
zC|LU?^*H|`rzkfV@ze4)&%*Pd3Kxi$HP*v3-V27uOEVGreuTvXE}pyC6&WB>52>L(
zLbIQsSGz#KQtCGCAqH}os&*?CGd&b}3M=8hxRloHff{JxE{1S`nh98BnD#N>KCu{B
zK1G~jSbVqIX0n}_hr^&)AMB<a-<QISKT+T97SlCa|CM~6wNpVkB|EjHyT))siO5d6
zfwkCrS(mkky+!~wD%tCNQQam?I`VcKs=LjTOo27H{M8Is>pNq9;w!81gqP#k#BZf(
zXCUs@Z^%H<9m+Sxczt7My=&@W*9_I!_^N}Xp#i>$NL7Z!orJ_~0-YTEfZ&B*bsjHG
z&`)l~wU%NT)rmlvGyEj8{;q|ew5Kl(&eEE$d5;%Y#sUCDrbmAlH%gxm0BCqqRM%?k
zTQUzyQv+vqe5kfA9hMCRc?$8<LK!Y>Y>`BRh$NHjncXmWD;H?*YaokB{VkQ=5wyX}
zYA2BNM9vO67W?Eh(i0Zjb9xde_!7G?su{P&>-~?j&pj1`zHx~LmQvIM8%glVxe!0B
z0FLCMcL3x2X-k}nsc4oXV$Rupgeal#L1g#(w+u+b^?A5}xY}QpRme&&^AX_libbsF
z_uR<M?a=)w!|%N298=b}WBn`~sfiRp+S1H_eBKMkH;(_6doP$3+|6lInAK$*S}xQA
z)F<2+MtzXGefYULM%v5sx@u~-LX*24neaiOW)R4^&2NFr4=QqwmW4e|D3Ml%L*J~0
zWesayz68h`towd!v72GgnQSU7TE<4kf}w8V$ke$WBIryL$kUmemlubmng}N&9bRSL
zE~kXhxhiUljnW*35}>GCjjD*a5^RN2L?Ni5XTK*c*XB=}equ>!Qm4Fm_a%heSJq4?
z1W*YIFOVWrZv7EHA;w=VuExL**AEjj5j^tiCO^d`!VpYd;@iADrGjSz%yHn{zZ|(l
z?((LEJvsY<w|{Tv)X-q5w6wa+9x2{8{5AA3y+EPFEV@wYiK4jU2v@iKeC5sYtfd$%
zOrB|{AeTb3uLyB3vE!$RbnX05<=o*3tOSt6nq!NE>!aF^JaR{k!3MhQV-hKK7LC-E
z!Fh~Qo-)wnPfmjAbwQKc>9L2AdZHdif?CNguLf#c1ziq)8N`MBct>2*7Xf+ir8&o>
zr;IQ=BE#yJ7gc&>t_)vRWgCuz_(xQRbAgcRsDo!A2CRe(@8dn6=*(Gub^ovfK(e37
zw2QF3Siqj}{pzZ$9=Q*l!G8Z%TTSg#I<wUyKaT8YQjBlu#qgg@J?HbI8gzb8W7UXl
z5C$Q#r|KRUM91z*uruGPA%0$X%VL1&k>3pV80KwI)D=Y1X*sDXo42EfP&~p@H{U?_
zqRRLY??IsuoqS#_o`rSRY>+e_&<V}92L}l);95LjX0@co!KZj$5C#XCKmW8WH&pII
zVA;vu!<+*C6tB|xK|8sJiiJMKQZGYOl<n)^-Zl95jwT07+aO3XxYrJjZ)4?evw-N~
z)X4NYpRqiC6`f?5+0Piv@Z3_l{Hg{Q{^qNH+=5zw31>2IujVD>qEN3FfNy61CLTj1
zXTdaq+tPVyo|AVdp2bn+H6*MP`UPyGKsaWj(#RE%Y@PU(tpl1gY;t<~h=RDARr7It
z?}KR-5F)_&Yn5?+2&EKWDwg=MkBO&`+S=iJ{es(lJ;ZiY)64EaSshZDo0I!1Ls(Uf
zJ->X4kRTtr#tJcj4|fCj0p2Dca_E7<*xT?-;a8xZ)v9?}?PPDif#F>-E#0)thf-gf
zai8&}`Ur`Y3#Obh5h{b0MP9%mXWEi~59LB?aygNNsfDm!UD|g>9MBj&k$oR>L=P8>
zhotqy2R(~H9oqObb`zvN>6#lzwNG>(?yV5n+dWjgefcXYMjeRkwck+Dk!#HA|KbkI
zCd<;bgWt2@6QtOLSY^;)n`G+_1!RRAxT-L&o!eM?1Z6t7yhv(|)ca`sxM48bFMLm)
zcJogZy&|X7mOOd#Bf6BNeli(hy<|1i-ZO8@_a!&`>Spn`Oa^k{6rPO1LR_MR_Hi~R
zLf!l*&M3Ld_B7z?8DUaV<}qTecS*usSW<Cv?6<!q9D}K>nm!i>5q&sMvVR`+GnIcE
z9D!)Y#Sv3pH2;hZ&r*<hgqo6ACPlJ#MfjE=Q@^)zc15^F2{u}Gy2J9k(r7}vD5>+a
z=xrRN2db;scJ8_(;KnAT?_$0qj{-ph{Eg}gQH=~ag8+Yy0G#%DgG7~PV<H<&sUGq)
z$@8h~9-=k$JJ?jirUG|L+WzA(ah1}=9*5)y%itYc)4NMsXKl-9oLH#LlpIOn#h3ic
zoXPiICc4CwM*8{<TlFq{*<Yo%U5#XabGn{oQ&pCna=fnIWbj;xF!Vx&*acXth-pf_
zKA)h_8U`pS*`B#?gtpPr*{$hmE-Xz1tXlnjn(-UI<Otpo({`R&qO#q_+3fs+)|_Si
zh)}$+)Ukr*huiAEs%$?ss;%zl-oNT>|0g&;SbP0nA3zsnjco4u_gg%=jJI(?|BqOy
z=z9m0{MQ#y(Wmy#r$86=R@YTg)&O<xOk0=KzdnJOa|d4ZHJVBN`(o&W(%kv~L$WBL
z4*0^<%8z0wsd2v6C)mBMS8+R99!$L=ufAu}@-ZXXpC0JcE}@gOxT}2YJiy^ZoTK@E
z_`*B+2^Y4k#4Hx{RvfpTYk-r&RPwXste_BZo4CFd){wdEI#yrA;g+hIuGmsgH(p%%
z1(JQ9K$TCFk!c9W|LJqS-A6cTOp2-#7mLag*JZCsk`=D6Dcp8?lRH`YrRp(lqY)hP
zJ}|8+McKsodm5FE?fkEr5Jm$ElD6I*ouZ;t#ie9K%GFnbXZ!ayukJaS{NFvT1u$m;
zi2z5Yh$LEcCgpDZV!4Sq|C#Mo{|fVk!tb5DHjA7uRY<=Tv+$2p$3iJC*v!G5znvP1
z9hk=gZlg20bQ8awVyiwSt`L9UQRs43Q|X3Pu;;C@7H;IP@rqS=l4PVthFzWbeZASV
zhp9@oQvI3M5dEUK{T<e$i`a>iw`Z()fO1#bGEQ^Gq7Zwa++4<G3i9bMkM1O7?yS!P
zH`l>QdXHn<GCL9_y5&Px;>gTp9Ij*6I$3e5%g(Iql(IpVSe~)a20Q~_hcM3|I^%&!
zq5}gFQHdAWi>ju^A{TYHjjsgeHnMvzTd8a&vs23BJsHWKbG{lCB6x7}QpD5rG2o51
z*;cK!<az1`RA+6^_Wk}^4dH%7$<_65zFU^d&}LaOf87&QFHWxkVaQ=-lmq_5*D6&f
zF+waPyM7w5<Usqy^3qInWXsc9YDXBhAuV$5NpAb59=+$Hlb=OunC}k|ShA13-K7;d
zSG&P}#Yqr7vm)Yqc8X15_lt2H5U9vz%hNr7_F=PKyU4}Na86Fo@mJF=$4W!1AfJJV
zeKA4tsN$4pJ4f|o!WZ!I*7$@LXM`V*deR8rNcDWS*przWs>%8&29ZV`vbR#NwI4SO
zlj;u*P^FPCYv-QE^@tRwYQ4)CVbG&K@aORSu<^kASIKZEW_g>R^cD{G4G^^aFu11N
zWhmQnTcJ#RSYh;-LYRc~&~+kw4;$;1I)g}k84ZSn<aa+-DH#l8<iexwzn-<-fz9t~
zCi+8D4`c5Y*>j(rb&Vx%9BXPsU@d<b$|uHYyYM-95eDwP&&Ddbd&o7*iXvy|kl_eU
zdX%s*M=8K;as&;QTZwpMwFnfR4`H2rw#zCzPN24of32jUrv?fs*3w9{YG8gSE*HR2
zn^ak+?OCGz*0fPjROgkrI-EEH3kMCVvr#*lrBa+Nqtx-Tto998SrbwtimsBF5<3lq
zZK{-6v7PtFh&s>gvRvD?r-ECy4TuZ{oZ4|8Z6yCUZ5uM_Lg}chX291|&hLtph>b^i
z1bfCexc`FD*WNd<O?l4eaTTdFkw5-N4zjVK#0<vztwb9b@2iiN=}d35A*eBpY?fTN
zWM7A~g)zbpABM_)9*l;Ly-=T)QM9st<+$H^wN9SB*arVe$={}u(}%0Sjy+<j$vwkL
zz}KCo7)-yaXoFfhIR}KhrS(_dWv$+K4$EOfnsgiwm7-!}kjv_pc{R%!mqP(+#FfgN
zP@Q%$hAmPMQ+O^s8=oYkzRPoaVqx4POe6Tp&+KY&bE4W#XOoHNwSXfYf$4Czw$m%u
zGSVMwbb2YuQe^t;iv-zVXHXt!iQuvS(`S6`QcPtc-}c2;uYuw0lcAHFcjoKF7TgP6
z{eaG(%VDh{CfO@hkd%ycY6%Exu$e2KhW92+;-Qr*`z3_2Z9yVut@rMew99?|K24s)
zo@OjW1(YW1RB)Xq#c~cU;0i_@2l`qnbBNgDzB&NJ>g?1!hAOu2f8J{Nh+J9IXpp!*
z6I!w`gDNJ+1K@V}hx(XKBup2_hozOp!D7E$oE%rOC{$XCYmw%GSv%61d3&tkNB=1j
zcJ}d3nKL)s_OD6+Xjd*5B>g%k1L7X&FKo<+vTosecj@x9gnj=F^ED<*wJbS;t&1-}
zgYCg%djt+v1tB?f8CU7_!smprQ^!!{m5~}8|G8=cz)2Cb-`ddMl?E=+U<SKVX5x9N
z*<<~s83@dJ7y3{fqcro(v*q%M>^uMCCH}WcS6Dudc=MqLuPD<(%k{AfID_3x-n;{|
zpd6d<4OlJN%for9pG@yCT@6L)uTwuiAe@l8RHj7SVysq)Hi#-tW;rMN?meJHC@nV-
zonypJ0Bor4?9EGbp=<#+zA%GFPkAtQaD7W5`K+et`p_xok0#PS$!5m_qxtyA^oCV$
zJrNzn!{N88Au*aF*A|~4WCmYqG~U3UpvxlS_HfEV_$PE=+#`E_Kb*3x71GGskM6zh
zASr(S{3AAZ8tU+rso7ZNUhK&fp}OZ-A=giUPuB6HF127yYQb#Hm2*tH4Uv68p7UHr
z24h)*e=^OvuuDX3hS&>%2ZfJ7&|h@ElqqTPu6bh0zQN$vamu4qcY-cZ$pQW^3~>`6
z=szf(h$+tn*9IeR?bH7`XRdfM@_U-*-J;E#&AeuZ@T{o+S4*(Jw#YpObmC~vyKXK0
zy1OPNvJ$!AKH}RRjHN5=TYv{B?Fm0<&aEPaBCzC!j-SSC-NbNK=DWU`Go#{DwET38
zZp&7SK+jw-$~OzdT*hnjAOGy=i#9My{MW5Dz{#jZWxverK|Hg*di|PPNMC@64O~j|
z{qD(yWSJWrfAVot%l;N1YH6#-TRP{HgL*FZ3F=ElxigAze~Y5gx_<cQI7&aYRaKbg
z^W)@zTsIugZf#a}n5!1)ctugMmJ_^Ve^jTME@ZH%%u&IG;g^LFW9W;xs`H%a`v_Mz
zgiY%_De;=fa4{#;DQ5BAhlwsdWT`cri~YS4XYQqB$*8XcfjJULF+fhr7WMZn5+G^C
zF)s$wmTKI?)W1oz7xuq1COg<8g+o{mbVW9>XFOwfCd<k;<`eYLyFdSw5^1y-JXuiN
z<h0<K_m<jqCv?K$$g6sh{E*oBMCPTzmltqi>vJ|LV&Q^P`JjM{9G9>!%Uedd(8d51
z`)maJ?u=1WelTG2WsJp1P=P4S{^7X8iVQ9!bnyp5e?9sELo0DC@;m5+?f@9(ndC=}
z32b6c#Dv8d5WNpnt2nc$40pQ!OfzJ&5$APCn6!`qlGOPfm|Gs)NfeRi7-$6*(W-Yz
zS1O*hv*h`8zR`$b)k~ANxB@q{Lf29=UhV|WW+3~xwSd5vJ>(Y^-iu8e?A|Zcgz;pc
zl!!+?A3`5oZzz3>@;-6AuuOKzBsiN^Hh%7g>{M5i@|~T-e{J1lnmL(LJzdf@EE99K
zpIZpSlMHE`kIO1?$W6FE4_-i=_AC2rj}yoIh@NG|IWf&j(U;Iqnw{X-i0;Q63_o$9
z8=H*X+6AcGOmZj6BlL>b3z4v`v&~6@I6RYPIB`(nE@gqBA?Cevp_Yq(q_NsRs>zsM
zx10B56~ZHU`K5#Mu^r1AQf<Oc=BrKoDlc`mp=xEUrIBE$ljz`A_jzfGozZCSC4X0q
zCk`N6J^k2)@k#8`Z|Pz;8)-3eqcH=QIX)gz0924X4Nn2m0K6Mqr^s(`o}0Ca+gN?b
z98R$DiS4v|n}CkvfuQI76bFWecGAv59K^GBqkZfKhex?Wy!^$Oj#7ra#m2#B&rq}R
zQ)s})gOzP=iB9Mrr=wE?M@AzuM7o-|-Y?FmcIK`(?ew|hK>;^M^5=c7k4s(4hjy!F
zfhgw(&Vl-8&(yY-M~He@IOONr<H5mkG#3zB{IGh{$w^ZE0tSmkvmBzCeA{#-ouhF&
z?l^*&{fX+oH|v<hQnXT_S6zu)yPu4~#9{IJ6ZF@XXj8)V=A{$t_0H$%HqU;%Z*c5_
zc5uAS>+^5SHP=E0dVR1KzFha*iANX$w*Dp$A6&0>+()xb-npK*gt}zzsfm!I@<LOf
zY5~u^eQj4D<YWYNb`5>I*jyCfUG#N8Secg)c@wfj2^gff-uS9&-8nI8>;?`&gALAa
zDg0@0n>*7+@*wa<k`_;=T|X~P@8nwU4sH&QwIkCFsl9+<TRdB_yMHoOPylem-1mfh
z-rAG`Ju&`SPiI3tRyxGb3u8icD>!3+W9J#xeKKL%)))F!f{^HHj;lZrF`nf|%op#{
z*?m!8md!%ys}d%Kw~1yY)~Rt<A>3!_O~14*xKs3$PtOXtavcLdwjA{!A-_0n+(43L
z8mSG>OVkYtZu{ozcEW$0<^YB%{%N;Ed%@p`>G)szuybu{W2*kI#;sdpjM*wZx+_w=
z8@6ZJ<8}R_Qb(k-Y$reY>@&ejmLk`@PdH+6i(ta7hxar3teDx2avnQ$Y^Wi#uqJI#
z?y%=+f;((SBPmxX1({9*ec=ig`{P2ZRnGx}YW9=Xoor2*<TU~T%Al<i!7nyxbsr6k
z^0@?lnn72pR9+W6yzV^BaTal6JAG?=aM|DQ2LDv70HWFfPE*rU>R;)DotJZ7yTpcr
ztLxp=q(t_5(IP8k->=OV<_VeT4oH<-sR90@*P&^7YDzaO75T0P9X95Ztj&i066F|0
zj_}R8?7!-a6i<Nl-cminuDxFNYjUl-=OAd_U8N$(kg(;ovY%y-=@mQGVN7|xYtHQv
zr+}mkJ5lx^=2R3ZijdTp%aUDGBK+<4Mngj>GhDCa0d&0Tt8_xri=~LLzlCHw9;o8=
zNVooL24P*>PVTIpF42GnMgQ{WMU@8ziFrLDYM@6o23i2J5#*m-NB5o-G*%5(UtE<K
z_$AMKixll;Wc@DYLR!qhCM3JHW}wvEVS3d@Y7<AEp|<szc_FG6=}pE5W^+b2PwvQ!
zqiRnawm(K*_4I>3ri4ofZHcP}3tl#Ojn<xQN(r$~6%Jx9!436L)`RACcIHNyTGIKN
zN6;4uV4x5-Hc}Ym9SS~nX)$>j`_*hMraIL03UP?AXwGjkTiCq`33k!P&eN8T?tL~f
z@_2#pLGozIb2TZw-%UdkE&Gcmi6$Hm^d|y28rE!!z9Or~#J?snd3CXg>Myh_{ED$3
zvVX7ozGVJ|>%Mi>S7ctBH4-MB8LitHwPB1d-UHM=SZBvjs^K-W98EoqOoEw2c&nJ*
zd+>}u?`;fkMu>#)mpuW$dJ;-v+Sgu3T+lybBgTU}-K?iC9_g5oer(FB$D?xt^Nx>9
z*2gHQo%TI^`DZ%ygv{Z{>?GUw+CHK7q&FL8HlDNdYdhY%pWn2ir(Q8R=dSX_Pg8A^
zw+%p?K)1B%xBSI|x2dj!gNayK4fWwUcGTZ-ri@iMq129>?fDk<E-5jZOiEOTHC}lS
z!aOVO988Pz!L`~7Jry_|q|-fl3C=>Qrm?~%8kq)0>71j~=?~6Mpj1O^W+%`i(<@uf
z;JxLC0sKxHwI{*Xdzb4wbYb<_6D0noj=um(Xo1rv?#YyzKZxp@d{Mz{@){zajP`*3
zwjf+lDBh__4b4rJO+_p;7zMXrpM?vaebG7LRxw#Q>=f3s#nlR&h-jUej)PRiDuT%A
zQ%|7_Z^%E4<F~q*AQ~F&84`BF^G&aoJsKHlBNylnlyT|N`WIYpbj?L6dDHX?=st#a
zYQXiBB;Q)te_W~3_C@tgj@Xq|yMAyzdQ>2*BN^Wb{jC!}VWBxOg3Hs`^ssNU{lPUk
zRL-lxIe!a!H0WtKJ&G`tO(3lSQ!j!-167}K`J1pg9NEOOQgLMtvFU{gT~^NWtTv&e
zh!xpH&nVXASI134;uJ#Op^!76ZRt%8tr2$-raYP9Z?n>Tu;%9F@Wdx_=!-{#rj`;B
zZwT1_-BDrLsII=0IKy93lbHB`D2m!WG@7KHU#m_IZ_LWPtklWG{wV&<VNk5ZH`)KS
zj^&wZe^cEuRnAK72^DeDK_5w3eynLl!?(=I5df_XF^s8-oE)jwOa^S6HWMb^lARu#
zUd<tHqv6v{m|YViKF-^7|8xz}2DUnu!aLAJ;#Ao`W`$=F1iPa|nQcMgj?N>pGo|Y)
z$*Jz=4ioXY2liZq63a+*=Kwpv<jOclD}w%ex6VYw+L38}*$}eKF|ato*gHG+ZLlpH
z^jV21CThvS9mOC7K--chrAF!l4$pr>y|emOC~LBZkOAX26YsyGbZyPioRuYKbPX=$
zh_a3XV_R)OuP-6RJ-Ak2_J8g8wkaRTWSVUKuO#-C(g*V*biykgd5?k*T{#FsM06hC
znLyCpe%ZYuJ=9Xgw<lQ&yG0wF&pn;WM0FhlwD7ru@`2QVW1=hCk3m%gR5QjRelynD
z0VkD@wXSV)#y<KqB}x@~<wd>^O#!6D*RkqI6UY2WK%+jM*D*|cyO$|8E(@hzKVDmw
zudDny(&OT}>c{*50`xT)ksg2iZ^pK&74#h3N5l<|x1$boM_-7{iWl5+I5Qd*$z*jk
zW^+DK7yx3cttY8gKTYBmTGaeK?~Pz*^uO?^BFh_*C?y?^wk_z1<yv4ZkV8e1I4&a}
zv89Fxv9SNVG@3Y3xULHQQ}|nG6juNhmd-iU8W18PqeU&5Fs<B!Se{^stPW+7De<gE
zM101{KYPnr6T8MjO}Lxu`>wbl82xY}X%(wDqd<dZ(RyJ<=#B#0X~r%||K?CZSp(AI
zq+vK9$bQ?s70GFQ>_@Vnr(Ao|Q;$Uny>Z|Tuv*11UY26?3dRg<jbj~YOewz1x4yX$
z92ao9et0^id5-C6gRyh)doTXvMwG`pU(;-7FVG0u)c~_UY7J*`{H)w7U(h2{^}v<x
z3=o*$55zmiOQnqvyg?TyLV}exi149pso!i@2VJ_As#M8uIdh3>DfH;`Q;C1hj%}bQ
zXSzPcbjV-J&tvNS%EENj&q&o{H$E0XnF@#%7(X!{{L0ra>{s`@k#qkmU(G-FWUnvc
z#fG9jCb&N2L|u%UUfp3U^hCy=e5`W;vevs^QDhc0n&8&B=Uso>==}Kiu%2P#(ye+s
z^03b+vxZq>g1R8ZKTAqizd<9P2#NMkE_RaX)TOGGo12@}QYzRA6(YnIsuoHPy<SjK
zz5K1&l_@pLSx96>z@G;<EEtj|@q9)-gL~(<t}w|e^ouqd-ZOp!gEap`6EN`3K^Jh<
zxEE7(1=sRD-sv5`xp?WTRh<KqDa&UZVhQPhHsI!Y757_rT=Y2P*x-3dj0VhYC>L?^
zNXraFN^vQS<I(GJFq&w(Ww}LSI~^s1SyFD$1thF|qMyB`U>iD~m0oUL`^<0iywo!)
zU&Wfgk6|=byj4w(2e)7A0%}tEnp$4^|Bsyz{-6R)lg`Ys5%ke6<}G@kiabmhee*B(
zcY(umF3=7=%hNQg8RsqS{<<apRp^pw7f!=J{;|r6^SV{=@);isTuF1D^~z5>2cF&S
zNPGs>{w66E)9W4I;Oq*S#rx*UQr1=3(3b<Q&KZ){1DeeEYVQ9}HlHwd<Bp2R%h3OS
z68ZmDSN8fjJ%juTq}btW;$dO!iIq8Z=qe<~lN~Lqc*!5=o}zCacK&EX*{Cx!jnS^7
z+|6WD$>)4l?<}hK8@r|A;ADu)#*enVJoz;r)w5uSBZrvyz!suAqYJLwXTX{2;igPH
zSR=^0jpHoBDklEz1Ji~RBDiTdF5M@0SP}%PKX)a6t-0$`G$gIP>4m9}9m0f;WyP0J
zxSpC+?z6qOb7W2yd{c;$%VB=K8pW1O7Vzqz<7TWGDAog(hxVN^h__Qm#~{<@w^SA#
z2)cIDFr5e)ZIVj4h5|rlGGC1)X06|k@#rTd>-BXo;6wdngP|KrmN=($#WAoKE@ss4
z8UO-9r+4MB&_TI6oTu0vJ7#?L7;-m<7K?Rq`LQBP3Er-y_~!<2SJ(*NN@7aXxRgWu
zgbJVSMqaliSfialHMWL3d%-^`tA%vizS3gP<oLxUpn%p%jFm}4qeNtIkbUA>NaExs
zbzgMcU4N;Ce9NV#oy@hX@-0B<(r^%AsG2}(xs%mX=u=Z%&%RjcRl%^0RMHv!Dy4&A
zc+41Fd|3fwtkR}s>@BmST@MYR?pL~zkqYJgIa5*63iqnbJj>m>tctWh8rJwDbos>*
zY3g)DY<3Q8jpM#)D@SE;V9?fj8xAU-Ugb>IjTqP#d7X)3Q1>B+-IUH%H<_foi~rV@
zXY+KbL8F=?g{BaOiV3ln++{t87sunK39eI(#<A=#^_l%LFv{0sxtSC)C%b<kOzhL8
zNo1?~GQ7mqaOEN$1}B7suHF1!MWYQeytax^cLI*s#{In})_P4<RFqvbjDk&y*B8TK
z#prBmC-2Wh&nm`=+??jvQ{TA2cZa$P9<Zs*)1qZDWhbG_6S+r;<7*nlms_Phh}sx`
z8u1`Cl;$<Wqb#=zQ*|=-G+^;UUc8$?G_6%=6cPHk!QZDD|2+{p?Md-J(s$63Ht08g
z1te53L^GV8``91-Df2tsDovT18E1ta94*tjF!KpaiwDqS1enUbYiDoulFths!?&_>
z*UBw*{(gO!zQlh3HL^dNCeBfJIx?ZI97*v%)!g}TkE4Zb{^y<pb)7Rw%%YT*Wcu9C
z6Vt00-J=9IW^(oQ$b349P+lA4&-q56pqxf)$0*o8d2?2oUFedkLGDF4*$9xuk{?Dg
zxyEB^C*4<i>eSovY8MYxT$?<xW-MFnk_^d7@p<u=F_hhMxKRnUQe}A8FaEBv0bKI!
zKX4lM-xH_!N5L;eMm;4uGvitV(7Ks9gxEWl$u)#iE9Wk)r1EHTm(f`Puydv4<0)Wz
z3z}R@C_0<6KO(Mry7pHCy!_PRH&?X;G=DKo4*ul5Op4P{t4)57NDhp)un&_*d09yn
zRGh#l;U%Ya1~pHp7+b6M`}>ZwU>7Y*Jltg#gT~}(*6-c`fqvghhz)gqu(!-O9&M&O
z^&lR4py)MBrs|HB>~>87vj)^9q`b%2@kk6BJBQd+mKl3v*(r3Xt|eW#6W;~paPl?(
z?`vT2d;!-gH`len`iZr1VHayxx6ox+ihs6}#}bG!a-!mC7+h<Iq36+e0qEclh3(__
zbKun!e+4aR=cU;)S6Ng@;^Dk^@aN-Iq4}#EYGtRHOE=XWV7vF(?mzRZp~t$62UR(~
zHBT6fzOGW)>E(_tYI96OYn;x}4KX*%vwD}clpFUltQ7yzLWh}y^iP4uLQTAU1g4TE
z30%1;HxW^L`+ons`vUC*TLn$0qaw1-t<e`WbjuZ0Zj5wr9g~)U*7<oyR&?r<dDgEr
z%L$;T#r^P=KzE%<D=_tIV#?9cqrhe^mKjoeCr2~`Ot~R&Jo+yGW{$!0n)tsYt%66Q
zW<~H;#py~op_z-<KuLwB`MI*fgDJF(uJK(fF<rIwO(~{k^1*6Lf*5ps9O0%F_)zHb
z<<ph<0Ugl|oF=5n%f%OLRNW$yT2`w7QD)yOb<_#6G@c*Yb_EKzl`dmcoRXDve*I76
z`2Ot`zt<nRcmpHe05n33H4}7#r1mF1ZHVNlC~WH+F)bblj+ZfkZ+R3!x`VgW88lB6
z=VY;ihur5Po9}SZ^9%C*dj|U>waUlmXSye^S;xG<5u`lU(Dx~KWL}a$@d7O|{p%7=
zY>HXeuQ+;+vXG5|SLia?C_NghW)(e?i>{Uu2M#-7ryqF~hY2jt{B7jF1wp3U-LMy4
zPR_>?=g_`lC!%c_W3K?T@cK3Fk3JozoaZtS{sWPb!;xEMxICNN?ys2VeB0#4k2|33
z|COvOrTKz~@VB1=ac@G1zp?RsfxU`aU(D^o8j+iuGCBD|OV&9V@vSe9UB->yf3vvw
z5(#rUDjQupXOR~4fEC<gQF@b%oJ8A(8>O++qcE@NS*9TOxnA9f_(uy7j+-KEt1<7|
zb0={am|nkDF@h}uC-scA#lt@NynFvG_9j8|1lMwCBk(1^)4X=tZKGeqN8-zW=yM&w
zTF^7mn$DLcJCU5^^{})p`M0QrR1#d{tC-qVzI&`9VRZFfacCh8qw#t_QcR26$KPs)
zhnn0J%7yIRkZ!YDgc)CsE!$E+6%)Yz_2@UeYSkxhMI}c{#{~Eb-9ot}ZcH!n=Oaw&
z0R2u=FfrFXx$N2S0J^DM9SzCpiM<_+Hdy2(3LxUDej^StM$7AjD`=9>u<fHVY7!yY
zCsyAK;fa5)*TZB7lUGtb7(SlhB7c;e!aL9!wlUn#bSYW)6bAkQ0!zNSerQv0JNpa=
zUMSfAE)BmF%htzsM_JoSX-?}hwjKh;$bHrNh;e&Hs~0YTsBzVsua*|W+Lh_hQm>0h
zBbc)d=<QcN_GM{M@02tQGNKVxw=roi-6INR=hCoj9u-D;F8=vze*8VGD&pq2ZR$3#
z?bg1Mz3}sM$2=XQNJSIHyZr$tcd3x?EThIPsOISB!jpy~q&O;3kif^|!7PA&V0g3A
z2o4<H8g24}A+E~Ycsb4+t%G~!Nj8PynPeh%I&NcW(3<|rgJKVz3IB$A=DM7IAY*vD
z)@#3m?#BGA5oNOSF>oJjI{eVdC6M#%tj{Qs7w)icd*e%4UumCLOpl`8Jr92UdNanw
z%bq4gO+qJ!Xe9WK;IB7|hZ|4^&ESeOb56!yl`BHJhRfA0Cmw|SaSbE(*YpW3Bg|!D
z4p#|`ePu1&(3UO^DyLDi4R#qTLvGnvcW~yyPOIqu2Hs;fa~~vkawQnB`=2$ER)#|6
zo70?v@!8+6bmm<!6M2>d2yD^m;rx8ECd=77c+iihm1(%h#79ALT>#|eWl1jN!1~E?
zH!nd46)Y@=W>3ZGXxsnf;l7ns1I>L4ezCP4I-=0&V#$t{HpX1vdXJIZP|T>QnLL|)
zc_mv%D=3$YL_{j(RDd1hG9)n^d<<SK!`)D8aC}_Y?>3_f8VyFYEwBT|-JVwtAe3aA
z+v3244^0Ekf`7C{05{>+w<pMFWAAwKVZyK55cxKi)Q)Wk-qBq9o!+nCg~x1&zy4$o
znC?De)GbH0h_5<QX4<7edMeo&<*9dfoLQjnumTaJ9NwEk!&uOP0p_S)Yhgi56HqqZ
zN1EUU=l?V?VDD?_?pZIhNop$yxu~EjQJe)6mVFGrY5;bq#+m66q1K%C=3gnu>u5Wn
zKb|t(gsJBc)$?^g-{dxCkirNIZ8_VUt6t?qeVT@^F3-_%OZSRG)wPh`biPGv0+!F9
zGXB|e&rtfhG5lvPouw^I2rE*E@Y=QqEdLS-YPos@gyZ=xX}H5g_c?N5LKweNC>M>1
z-C^%r0ToW{EBf4hRmL>szJ}sL^sF${$9Z3^JUTB@f4y_G#3|sN_ak>@-_J>RNod`@
z(G^6a$q;OV(Q{08HO}<!MLo1ARvUyCdDM5MqtzRZ;?UlEE3H?&*5X>{g*rkBTXiqL
zJ)u&C92oF6*4nkCmUO>;<HjTOjXSh?CJNxv0IT^<#}~<uN@dNLR6aa;P7g92{-s`B
z3j?4kQ#jS45^IC<c&=1~>H=n#p^HEq4i!&vw+BXPHq7ye6fN?(h)P}M8r6OQGAQ^q
zH`i^78nRC+)V#2Q*+zU{i3?=3d_Sn`WZ&q0QhCHh;k&<Qw>moTM@rXoUr*n*5rDOf
zBY3x!liah0m(&@Jl1p(%B<M^uKp1|!y0*67L+%_ClDJvT_^)%c19d-XY_8156l(5s
z(QBKeE=!~%3AoO4FJ?KbT?!<zP@0d&>(e2Gg-g>jEH3KT>~*@G3l1EpzP=b`DzB{x
zy#iyx!nb}iwkXIp4&VO0fum~!Ku<KFu?vvlz2(becg9)@mXfi!KzQzbzy1<A6#2b0
z?_kiH4O&Vq0aEcVlbhswF<wMSRtIZUlFGCkDG7LNp3mUQ3hyXGbMNvZ)${C*xp9G>
z?wDDM*y86O>KLR7T=DQ|B2o2CIlh<nQY<Aye5Wes*80nZ+{)7_U-EJQ3vMQQHFfbv
zN`d`GBvm;fw_jiFPHr+DzOI^(+?M-aKS|LmZ-}dMZ&>{D4H1?v(UU3?WN=#M<1<P*
zVyO)pJi4Zs44L0l`s}4?kGS;eyfm+MUZ(pChT+(J3>4ygD%*Qe&3u(NB+cWYqY^uI
zArs4e86so3IK=qttQ4TN`b<S8%>j?b6nl79iFx@~#qXn2qi@}r%|2^&u(b2X(~lYK
z0n}Joq;y&EVTPoPC!S=Rg=DZW>oB?i5{wdZoSydl*&YuPv`(Z~ds7eR4OU+5^Kuuv
z$g00{^RCkuyaR3xntzVQb{Y=v$$uURf9}2JAT%TR{Rcm|71t8ra6;JftW;Hg@d;&7
z@<pizcp62e>z}=1>Q0+@U|2v6h6XP=NQ81}8DjpV{#Gy+LwV(61kK?N3aR>BW-(dk
z$UPEZ&wu`r;f-+V{@(XGL)@ilBvQA&`YtFu7pLgA#3m+P{Qn9wAZTwU6^lBS+{y{(
zQ(n-S6kSLrFp@cBJS>5g$hPy-*AtdHYns!iS@wavUeIy3IO7N1xV%!~uSS)vHHid%
zy!X<1KC@ZKz_lB5)9mA;j6<m>*j6^z=}TYCWMelHvGdfWw~}jbgM6I}ypCFpr;BT`
z^Z8}Heu5uz2Y`5UbO?6^bmUcHCoWXjsEO72{YXhM0g0S4Ix8ut*-ZEle-<N)-_*yO
zW1CQiVHQGRVZ?$O7Yiv}oUg2Qf;8bLs#;d~ckUZ@JKR*&6N5N^P?3mQrXrJGSQnH3
z0Ee}$g7FzAHUx9*?GgFi+lT)tWi`jP?iFpNpx;lHF?(CQvHd9gjaox<eYx(4Y7hPd
z4=MpxHyQ*e^1|zn5i8%&Z@PV|8_-FC<xw{UYGq|i#~a;s2nqb$`o-f-@q$4-ONB!}
zn%W=?<rAc=mA)xm8Y9WI858-VJ({^B0i22*S=BINEv)<9n)2)?p>^bTwo7QDfAH1<
zpNa*rPHm=#(wNE>pUc33CzlK9a48J_+z19rP6Q!&J8GX1F_^>_B4_yDR@Db9Iua@q
zl_{ps(B>z>l$xkL7I?|;EvuuG<8{=7r~P`6KzGo5$;P(>&nLg*^CrDru8PymHEAih
zW+V4}m4aKmjk8f+`YQ}m>VJ^#5+!xnSYC9jKGDA2`Pfv&Ff^Jc_4;87KC}iPXnF1K
zT`9+!859ILW5xOmk;I`uD^G*W=QX8Wg92+DJ2=zCqJVcnabhMq$-Z1g^{|zynvhn;
zy!Ag8s>BZ#J`@$9E$Q=-A8Nw#MbDhQfyu*KI%M3sa5YJ+#!N&@#VEN>g!@9encvTu
zl%Ds-jRK{6vQV&j?#XDHk^sCa8)9mY)rA*l&<<d(66|)PU*-YnQ+pItKHs!pG6$<&
zQ4S1L38--p$kOjgln9h@S}nSU*<a{B7w{Kq5%_<_1hwm2&<mEl!q>8(j0<#>p!F5>
zg*3<TNVN#Ro%Qz%bWst;*^rObuAhu!kRaw%(0Q?`xxLo33(yF*q!#>mad}!Icc|da
z&6TTk;*gDQian7sT5WMTeixu6YC7J)%(62rE}l$|rTWPIJ;8|;cEVaFV=mDvTQaGg
zbkBSR2H{^@+@t@jN=nCs4ik82kkTpQe1ZMXrv9&P1uN5CbgE;!Un?0uMKRfGQ<?IA
zQCU_6kVGP1zP=-~Kkn20DjIZAWrr5tPFQoie)vnwW}}x~b(-`C*hyZbJRIp9<Qd<d
zIgn7rQ&uGQIvPCMA84$IB)a@QeZh9t-OKHPBw{pO?)|uFdJI9q{7qDtqA_z}8J6!B
zr54iya=$Mi9-({%9dcz@kD;a7cUU)bL=Hs2YkG$`PV_A^U#<ItxTqe_o#WmbJcJ|r
zA|Ki2wiPrkAX*1a?k?4)V^7}><2Xnj?w}*#YpE<YQwLHae-qZ3sd)O|V>@yF$y}(Z
z1Uy<Vc7!Jchhh_>n_N@=`I}$l(3c4n|C0inDDJ!%-ob_SVdXbGGo%0iH1;0AaE0&R
zcO-g<=rww<(N~Wa5xo;VT397IL6o&h1hEJfLG<23^s*A7?dqaO@4ff-9QpnJ^M7XE
zdEa@Bv11(PT=%K>bzR@j_ar6}G}FKgoca<&)OxWQD5%fDMiHrG?-cZ}d(NV?-;DBM
zzM0I)Lf{b&xY0<GmXjN)`g)zxjuCV~tQbx}gIwAyG(umKAABa+{p30^V^vid`7)Id
zllbXw(2-Hrqt?9M*LVsE1xrX(t=5_1&b~SdTRqICL5#1ZLe7fR--OHNOA!dzAip_c
z@bF>!@b$oeXN&^@D~hJGHy(AS;!0f@Cw*u~RiX%>YknU=wq;*-h*6kEuhJ<dZ6G_e
z6?JgUmDzU$-b<v<ala~NF>lIKNr%5sr~Y@*wXb&KZ{zf)>&*s$CB)a-unmZSAGSm}
zbzHb1J0<un-4nmC2U#JoEhWr<U7Z+?haa*dM0X^m#UQ?<@bq+xmvJzgipy5xXH%67
z<A7q!asy=r5OFh{k$#46m9(`jv@jIE8TsWKc<U#Ryk!rW3;{_>PvWn#sfA?{3|3*m
z!3Y{=!%FUE!M&p9%2kD6#VF#PwvLz?V(MCj;hV3@{5XGlVVdVB?D0=$$i+IwS(N0o
z%+>bS#RtOM5kIBb9!w|Y<cMR*pTF_O^G&d1?Kj{vv`;$@@J9-E0%TLv-Px5skWoKy
z|KL^U3k~X=OFN@5Q=rTkzb4S>cR6Sq4nAJ{5F4Bw9l(f$Qw-*ZoY*D#3j4spnZ!zM
zi@|H|a6K5D&zmIh8`uYb%i_W3{^*l?o<!B`TAyripyL5jonabr#{p6f_oCACpt!}Y
z&OV+4G6)^{^anrP;5XT!@1K;y78D=*FbA5Qi>MUkwH7rQZAHbJi{}0IbunxCoMzTi
zqkB6wa&29(CDpq8`h##+t%OhUZ}&%{kRCdn1rt%&S{wbG)%JNir1&aD_nQ6rS^t*#
zEiq4MWyW)uKH&{Yay3AQ4I1<aw9k_vmuMf-*WZ$qH&yZM7R%mQO<&|P`nw$JV}mH{
zw$@-d?bT1Ab3jNuNk1WAc0=)lV`|ilg|mQ)@=h&0S&ak!p^G#SOP@-Y_ShQKlovc7
zXob}@n}XDk*HV}JHB$BBb(fieimnhD&1cihLWN;AgkJ@>9UdYKc|YjU)I%K~LWDQH
zOX(hzH{I|14CTm9_G4@konS+ymF6J~9p?BmOTO_N921#_qfI&G0eJcRATQX@@b7O5
zsp5-UM5t3<?$fGn6)4UJZZ)+oZ1bzXO|T~E*DnlQF=p+IZ;qoWN$%^LD5|MlhrV+T
zBD%+L715h*!P19-QxR10E;Q=4eU9Df;lY2^Lb0QrlnGxy2`@-j?$c)+ISx?Fqe?_~
zFC&<qeBE0y2I?fBh@_q1YR+|=*w4b@%BZjmll|)ZMcthmT059cJ|&l)UFtPj(=aa6
zVaG`)C3W@P(^h1yG+UOqyyyvgJ|ks_z=(v3Y8UzK($Yw;6KqO{w>O`Vqa(m6XL>jq
zBQ`U6DB1l-Qsp@5lXUkmUgOKfkeS`^V(kZ5A5|wqq+ho-kLRXGq~U3qUyVrtlw@Et
z9?pKpa$(AE*X}=wg^56WCrL~1HH>k~SLKgWJv-gGq0rXqQS26x*DnT~(B2u=Cur#@
zyExixcz0>xhmXpA$Hh<S;y9L;N-!K0k7A^RlI;$b5q37a+U=?%ymes(W+YW;q}~gm
zb5oD!nSa0QmrF7J$i$iI@}rK4`(e{I=2-k$oCDu|>H@R5A}KZKB^l2t#0}OiE$MyS
zLtPXQc${RNDx5+0pfbi&LmFEV99q57{f$ao3lOePxj)*2%u7FQf)a5{g>?~MfOF1X
z+;+Td>VWaILc#@nl!9poJB!TtU+$coBubgHg05Wf(sW&-h>WvRXe1q!sFJT3b_Q?4
z<w8=IHig)<HuEWGm^>OBKEbHFyCIuJ06H6bt;$NI*vWB6u6`?VMi!qL1+v~pKuZJO
z9WZe|yBhk4w+lD{0SU@W6IRwh(iS{Sub}t%Zd!E!ezK6BCP7mCko!V$$Zsg%Ey0@n
zq-6I1&PT4b&Og#;s6ie?h{-x8-8{PKz^5fYOXW&XmHMI&Yk?3}A$aqoYWXr(u_O9@
zH0z@b1QYajH7f|b9T?l1L;AHc$U<dE1wSI<p_!hAT4sc@dJ=0bTWx6J(i{FZgqvFu
z<XQrchboE9Ox;jVn_uF{7$20!YMb~k>=5qY0TaQTJ(*cZ7A@fdIGFxyH;HIk-Nmmq
zO>$MlCO_jbEQFII4s^5LO={=wo@Zz?OBff4F{RkZJ~4B%w8_6XxdiQPPBP($FoZ9`
zJ0-X16Y-LybCD+Ib7(&@<ko{V^bFG_3KU^-QRe@vi-Q*1^2hIWcqJcm+L%=45yUYP
zN*(_#Wgn>4JWMenD#LrKCY9~L2Osy-wEfh#L@fYA_UzeX1psXcc<L0LIBi{e74THR
z9h{F0F38Y0fPnwAd(BqRUj@y|<q>9fIXL4PJL~S6yJ4^G{lAdv*r9+@z=ZoF6hC%}
z#-5=Z<|P6Q;!)wi4gJo&Yj%&PnSBGq!sDR3G3KO<k^-awbTNKOhcOy)Ayq~%)?w)H
z4k_KGN}6q~s{u;s1<a;Q?D|~P8f{oC!%GjyC>LziM1j^<=KJrhiAq796Jz)VY_rnF
zxoP4U*y^deq{fYvkuK$rWl=6CvLATywO5|?n9Q;Hk(WlJt+HJ54E3Ysqjbv7^4WP4
zS2jVwzMTg4ZRyLsNq^pG)YeOC*x=HXdXh)YXZ#a9h-haWUcf%@YVHu7Woye*)R!WL
z>m?7F7IP~^NGeDQs&fEgRu)U+L;iU?QIj*pp5@ObrVyLH6QjyshKS=#fGdS@0=;>~
z_!bqiG)@U<+!EEpT`KZM%l`&0eb265QZtGe8mfKK7x*2FoN|Iei4;#P@khUA(`Dy5
zvlnWHz9HkW8?_N84}7_=p<z1hV9)!&4#|2D`!DoP)%2NVYaBCcJCVj|UIFN42se4J
zmHNSb`-Qn5yB^~cszt1vdI|gdR973y)GkzzVx;FH;@Sj(b(OM8&hWr2&m>t>pgeCR
z80<7M0Cdk_fQFR;BG~c^uoqIjNE+>i-N7P+2<RSf%$@5)EKrW5H2u4=#HBZynE9LR
zj%CRvXBqWZl%sac;rhX&to!)MNOU0{PD~KLkVhP@#G$s%tajk?HLPco;ptV^uB)5x
zH;}uR^l3^YBIfz|D|w<Jo+g&Eja|~lqv$0VWrqDjsjz2)LCb|$ibKS&ilOd{Csf+l
zrlLlhE3LgXniQNdKPU^8W8{g;=(I+30<AU%AI5r=ynGkr%6V|%ZbDbv83+ETvVY&0
z!x9h(z2@KvR(EvX2x*d78#O&!fBEiNF2UHzr=6{*H+U|-4xkjDi5hYi6CCDK2yhL-
zaD#TkVtFq^KZKWY*Sq>QakM30jXML%SYxVp_9qWUx+%MAe<{96pldwhj(_|n0J**8
z5vX-?{Q;oIed$2i9tRqYjSMYplgiYBF?tf<9{v-ado$kN-2UJ<{BFQd{?&_q>s?1|
zs?^^R*sT<%P0_R>ICrE^ohu9=t?yVOJ=wceg=|Jl&w4H2u|fAxwy9S_oz8U<n{9V2
z5izHH$#qz&PwX9i1gxgPQG?5~rT4C}6qU{8Fm*X-H>Lg$?atucd;t4ka%V=?YvF3R
zuXG1(M`gkPbV~!0L@F~Ns+GsU+Wod|b|hezzamqB_De3{f%G4nDd4CBoYKFaJwbTc
z3A$Hm#6ObneM#GGlqLxn-v-LvM=3ywPuTlwMR^zsTh&=rb^mFL-D>l@gSk+%;ax5+
z<!;>S(=UpW-=~*W^ZB^xDmsp?))?J5(}9y|%u%+m!}4t|hq%O0FV`a&5-GMSH~eFp
z7!W05=WGAy!i-S^2_OQ89+MJ&*ZJaT)q{6Ro2X*w<A06sDaqBMb8ZP9)k7I<9$ocl
ze%rKg@i+Vi-n^TN{F)GRFjko3Lz$i<xgcKg8E_*x4o*s{;?=tO6y|4cRSJqp%NB9a
zj&ERleYmz{htTfS6|9F6D5u+sA;RzfEon|!NTCf%5JFPB6zV1Hrnu^si-1%D9}#7=
zv_jvzsFBq82>5N56ZyHD$Xq<AE605cFy<SZDZRa?Pu3rN+|~Ajd0<s}OgFV0+_C1r
z-(5X=8zG;y$mO%`TO%*1G*J?Y6cfjg8r{u$OPmmY$@jDthG%S5sgps=GAC|MG)`xR
zSh#%Vy?=cD)k6b1CbAm%iu#SuFYK2S=k2Y?+YKgo@7^`1w0;&*npTId$o$@oqNq>Y
zTo5D6Vx=JVngmp#A+i_x<$mXj!3Wp(a^HloH`v~qR7pIQeW)P-k%g`c+7a1qIuNUg
zY?zC{j^lU;VCOFEXBGp8*};lFc1d%gW3gh8fjgBTKhH4qj*ZTjK+Oqyj6g{NI4_-`
zJ};48S8hoK1OBfFPbl4e=)xc^32~7RX<T11Q5&5JXuRENVjGivC(O?3a18#(Jf8@W
zJDP~>3hDNgu<d<)=D|Jak{b+VSVjT)2dKP&V%f*Gwq=FX^7}UyQW4H<|E&W3Bj-$~
za0k_vn3AbSFmv}epV1^Df)>-ph$&I#*{;OFLo^C6kWe%ligok-+H(D;OV<i_@GBnK
zZB6I4ju#@lW4fYC62fCF4#&mvlIDo|4Ndn5M*fXo?`<uCjswG|_sLZ_vAk`ouufr~
z$z67dP3tA}+WEYURddw`YUC~B;yb~7f>&HSR-|{Na|=EB@lC4)L@ctqvS-A6+3Ne+
za4GWh{rhM2twarRY2YUNw>&q@r_5&+8+xHM!oDAtPS>zRB8=I0OL*Zl)DRbblj)^B
z0B_esBYv5+9>|i;N6f!PXk)`>zvCXalR>tQ;t{!R=ez?ukqrbQDH#OUcr8SIR}~DM
z4gt4{ss8bTv@z)i6;*kbmKiCh{E||eB^E>kIEq*l%NOgR3Ex(%J(J&mTt-wxMz}?P
zX48XzLMSzJ*~s`%a%0hZp&m8S+mS*Vyv4?_<A|tXM*J=lD}i74dkbLYN04`QH9y6k
zZJ_C`tj;!U+}9sekr)Y*nCOXYf{f**Gz8T?wjn_v5Ouk7uc^F_&Cm(K+1R&^Dvy)C
zqfn68Py<li-<=ZqBYQYW0&!_{IxaRuewUm@E2GR?D4GG|5yNrW54M)5s;<W0u{&}s
z>2Bkg5cGHwEAvlgM%X)a&K0iDYM0_;ar{f?o<KqD=kI$16q?{unkPTa12jAN>E7Ft
z=XX=ZYjp@e(+_6ji1R7@N@;aJ<2R*~8T3|0Js&8$I}R!t3!LW?3HUc-OlcBfFaL%Z
z@+~R;{mC)hduf7T=evW5sV^%1!oe$Ot3=iA#28`QO*kNXhpIh#==!m(<;Lf9-e08H
zZvtDN6c0<~4raEp+CrPZSZ1%c-xBUf&ISatpAaGZ<*C*47}iN=;OE#tnZqYz55med
za#{I)4ZUWUoYP}YKtOS94Tv00S%xy$8}~oNw+m%<<Ait8PAOB<9qk^>D<+zfqPzK9
zBMSK_dSQN58!}@$XT-{gLVyUa&&v8GyX8sc3{gbP*OjDE@ozR2l&#uOCH@LyeE8`I
zAdjm&l#?`h@%z;6%B&z<Xk~A?mbtdTM<zQQi{K9>bwg`z9LBHso+4z@GpC}g53;VM
zE)2icWs+v5zjtmi8<!A3U;v&d9KnloOgT?NV4Hmd?*)H8|7$3W?S?`~^5MRbwzg7s
zP1+KqhCF{x#>$gTJDk-XcBSc#xVt)DNuMS4nn^hUgs~dZC9fFei7II}%bQ9A0Aa!U
zR{FCAy2X-~Ba0t9fxM`I%-1YFN<IDOD!XWxN83fX%7=gn$FV?+CD3Zg0@O=8u`opL
zd$~^n$Cg!T$GtD88^#-16Kg)@p>yb1l#u5up0ST&F=h{v6+v+{Kl6VI%a`;`>^<&X
z`tFUqyq<g8EyA*BX;H%K_wRlwV%hcph>|Y5m@%KgM$Nsi)2TD6j2{^dHKciP{#k40
zLHj-aTZK=0uE`D`l{UAo);fe>ph`i$On4dt?JMplb{4P8a5UypP9O=Mi+atyM}+NP
z@n;MEJr_ZDPidg7_7Bb_LJb7pL)V6_3r?p?388+k2&-{2w+;h_N;D4982crT&z;*E
zSgJmdd>E_YA-;YoO9a3YezC47_~ue*s*Bl&(cXQ89eWWR1lt@Ft7}XNs?4y93>@$c
z+H2G7t<rea2337&Z!z-c&PXat7lB0tkgV1pygU?y0oc|&G7kzJMid&`2|~SS-s1}A
ziQEp-^eQlo5>~A9&*>e5Sd#tbdp7CUr;siZkXS>NiK&TGKyZI%%YnWsoh!~HRjt-S
z^17Q&wZXU`Pk65RO5^NMA1MT|8=EM5LB;c&@<!47;KzBUHA$5{4T0#(Tf`)7-}zkA
zpBtROft8o<TG{SNjUNt&z?1}75_tAe%J{cjdfF7VVE?Y?iiT`QrY2MnTlY7??C}Ds
zAIezmUXf*UEvN!*m+lvL=2Q1?T-B7cwQ6F)Fbic~HGQIc{~>TOaA}GrE}C-g9d0$U
zBZXW!BCQo)VfRNX49GeU*HY({2>FGG7W~6(51o$%aYJu6x#_fOa`i7gzErM`wG#!-
zr~r+xJm;{65hunT$4B?E+wB|V1OH1857sc51o*I>OFiB;eUaM1SdhsH(-dhG;_&}O
zYj^K=v!t1OPh?c!opR%xtSkW$_@JCfz0r2@6#ZW^ce!xFb@BG8&$1&);;9Bj4OtkP
z6g!v*sXofnt0$WNKTQ-*OjE%A;`eBoy|EDah=6#2N+59OAR>TV^3i%`y%-la$SO%w
zuHEJ4cZK;Nz9bMB_hZlJAB6&*R0@okb|2V}|JE??{X=avMN;M@a+}jCPfQkkygIV^
zy@Hr|NH_ZSy5XUSUr@b1O*~qhI{e-9fVoeD+*Uj!S}P-#`94neMCGQvy2eJ0_jl)+
z%oPN)2n35t@jrmP={7L-A+5fFjP6FHnR2r*2Ta24w6sdkvN?nz&ArVjxoFzy&`6nq
z=t54S&wdp8vZyM+AIYXWb}hZDB%dq6f9?P7W7si9{hCALq(`)#C6XbYs-I#Iw`<vH
zNIY;%WiS>O3XLPys=251F=#<(U;sg>C#+jG^7gd3_zk~*odO(G7Rm-QGw2qv1v(Uw
z((Vz>9xUHx(@L)yac#FLFm2q==p-TA%)dS-to3qU_;7IO$VNXEe1YsCYU2Al0%IJc
z#oP2fiC+(Y4B?zZ4U=k*vS6OoZYc7Se694iQrno;hJ1hfx_=9|qgfJLjDJ#667u!c
z<h`byIWOhJYI1Lv`k$q+$x^C*hhwE|9}9Neq_|^5Vd6=r3Kdym?o)yrMj_tam+u(&
z`c%|Hco~`=z(Hu582AGC7giGK<s;QyDJvtG5H5FZP~LWvSJ5qIRl<f-PSkL3pKuP>
zb$e@T4!~m{$L44-WkPIX)yV=x|2{W<gXh82_L$_w+q}6?^e-e~-;ow?go(8hCiq?@
zSssP*#n$x@GEa(T;7qd==rd&GlrX$;8md#BE|Gl`{D=fHka)hkKEaKwd1bgV;504x
z;n=0n(8lI&Y#NKXV5grjb)dH3OCxVJklOq^-?ciMK$qnU&^{yXDlz4ceF|Sf^9kAE
z;RFiz6F|aQ!eG3<@+;_#mpP-0TcLcpHMGY3_|iv0>JeyY(%wq`v%PtA!J);E{ZHd8
zyv04|U{%A%S7QT-TBxASE*00uX4s*5(#H_Or)Cr)UiMF@V=xAk$^ss-Q=M0;<C}e2
zsFWA*`}r!oGJsyZg10TyuJA2#>%3!)@k?bCHnJpM^$6yV$2)M&Ri=lqP(#+(78@H7
z7nd#mKg#s~Tcrp2QSa>lW>AJYTH{MBs=yi%rM6Un5aIn#qRb!4jS5Eh@58&_|1i_b
z9V790EbW-5EL!>M=_!P750A-}I**_oBArpEDoKN-kEqO8g^52G&))f<o%m_7GBQ=J
z+*FXq{_{d52U+6rCOzJ5HsSAU;<Z26;;6@&Wl}sTii?$xc%)S~KQ<Nm@MM+nY+?Vc
z=I?^<ax}%KHq{u~j7ru?kdG4CAO^92USMbqsbPtJlXzC_iO^E>|Bom)zH2E0Tdd<x
ze0M93ce6f>=DhNEY)(#V*8MM{)o1Dz2W3^??~3K-Z-k%IFOP6QHn2G)?41xs(qBHh
z!_C#SEri2BASuZWh^{K0y3j@+>(glvpHC;klt$DL1ksMtlKfGaRw=(`z+thgM#|!-
z7_~nYZM5zVU)1Zc$3AP;%qO+74pO9foHzH8=^o6u!_T*|S}Q8%FR!uq+Enyt=IDGy
zqXNE;kAIbixx$gUn!a?Quki}3O8o%x0>yvA-0NOkhU^(~5}LfiN$?e@Gt=1-0-TB+
zs-I|?tIQstzTq{e9Co(?=tQyOdPu@s3bkp~;lxei`a5GrUpfxo8f<u|OU`$`fzL0Z
zU`ys9@rT`xZo#xpD<xT3H*@?nh(j`APw@_{2%#_lb<k*BK<?!jJoQWBIy7T?xZ`hY
zZs}*q*`xq1H>LUZ)n1mpR<5wr(=#Jv4UWrHOEBUWK?22)EI82DNDBC^_p8n1fqefM
z0pA;l5$}f0XjoTGm&_H8{3+Y>&hgwUnDBd{JV2LkneTKAGH$9@J_Gw|jk2)rgr1~s
z$s206Wp}D)HnX-j^Fw56@zr}B?G`x5^Wasb`3}x71}W#_{m(J)q+fF8cGp_Dk{X|t
zY~1S5HzXw_46yMYKbfl*6v8lF?(FYHU&>;AB&Xa<tywm^{lLWJG7|FV&mUL;?1b&7
z-SMxMuf9)&7gW2$e!aM4Mp6owaIE)jWxY7EpXZ4eLnYt46>C(}`6OuCn&ND6E_Fvt
z3ryo|YNM3Dyi$=#^4gZQYF8X#krgub{e!c3lF?`b2Ukt}sLuIV#k!GL@b1ya2I###
zr`pQZswf|nQ8x!^M+?ilO}BOhm|8|jF`<Uuv;M@=>;0Fzr@wUz1Mk0N+iqoeHgCE$
z)Gycnk%RBXo<Zmh-nN=(m<W%?G;5Sf9#XYgkQN<D4UxlAT*#bu(-1xQ0hk-EKb>IM
z36g&FD5Y76K@=kH=q8Y%Iu4e2m8HtIL6ky+JCrXUyxjkp_-P<Midk704`wgFW+O;^
zjB|VzqA5&VL-a==ykNF5HQi;P_e<q5cBQFGmj6@8wlr}k;i%5&J#>hO+66xBwC7<|
z!rZye#?<NARw)wki+nMsvfYI9^a@N$aieoTjjH%xnVNO{#ckFL%d1sppe<o@6pJmw
z{4~eE9k+gJ$}D-K%~TH-NR&8tBlGAJ_n=xNeGMxC#=$)OcDg}j7n0rXaU$VtVKjP)
zv+iGg8ZLsXND9g(P-15KgwMnp;5zZ;F=AH1%(FcUcq%p-<77F=J<Or`g=pUMgUX$>
z!3)fk4DF@c$XUAJ&W?}gwDRc(c3<xP^1IQ~M`@lco=IF@!ocNCGrO5~4fqdFpnJFX
z4nG;t+&-E-xYlM}RA^8*aI{yZ9y*2bJmVOq=K!fb4+L>i)vz|_e6Oq9gOKZSH58B`
zj%|Zl`XI63FUN<hb>^7-Huw+Y$PjAJ?;CUT5WL1^$}GNWiN_5Q-ex||x&iREcTqBC
zqbD=X++mf~Fz5GLD^$JV7vzY+Ar=HU^A<Chh(#h=)Pw0T|KoirSTQ$*BT}e3`g##8
z%Bwxg!?y>K0#u+FN08kw&}BqB4^;oIK1vu|VYu*Kp4#X%^G`b|*z6PH6hrcPW*t|7
z0=obrM6DEu65cSe`x$H|OeOCi^~VNKWb8@TP{3-G7QjcxWY0G4RHk<AR5Kf-_uzmW
zSgQrG9f125)fGrno$H!p@$4NreNuJgz(R_~?1m!H8)Yz^JDA>Qd({2^Qg{Bf&Y~s%
zX1wT6`7|IdZ2#u+>HyNGAf-kz^bVsQFftiz2e7+iSR9+nO%3_fA9c2uzU{I5NBtj6
zaG8~7o>uzPmEtJ?cVUmXwcVtWJ&3q}<9i1cS!v}YfR*7R?tqBjZ?AqJ4n|$2a4Xj6
z{UN$P^J13XcyOBAJPHFs+Ngj!Ocaz4(GQZKkJr^l6|lAZ6RrPb5|dXYAm1WIxkR~=
z-h(Zg^{M2Fb$sj`A_d-Zz%YOJmIIgyf=O5UqI<J-6)?d0F3_Iutc+BB1?nV~pQV`r
z%L`bszd-y-n$3wM19v9^Rp(-%_Wa$s@|<=A>o!TqD!#@ntC&3NPNMNZ#)m&82ur`k
zsc7>z31afY{(+yhR>Gr3W=3eVq#g1pVahIN(ey(eRTz-PJ4rT(W9m|9hrr8nWGvBe
zvX=*`+OniYWBe?+``Z$hUa3X)Wb~d`N~wQDZ`Mu5^NWxWbMJ3;BYvoWHV)%s4{}m!
zJl1{2D?Z>=3v&Ao<i`(b^SK8YUN(o!vIBH36T&W$4O8#XVkOI8eS+l}kafdcQFPnr
zqEHC=0z%OnT`%2ry~u4DmLIFXBy7hkm{U|NtS0A6IhZb*F5Xk5o^<j^iKj@qm_PYS
zc`KNeX+?ng6)v@)q5x*Dj7z&E7KC%M9s{tKh|D>^NqPM!o1j2RQmk1$E|0Zdmm1Y7
zCF!&FW3g=R=jEV~T->e!>I!C@dYgR@k6EevRU9HIentEv*as*Qq{WuRa(0BZr)s8N
zjTb(oTR_uSv!&}kGKol=V^)@1DO@v=H@x_BKj4|z&`Y4<!PcvBXR)jFT60C0J-LKR
ziC?a$R^e&Er}u9WnZ4LUS9_WJYm^ss_q+o+Ylw<*mFP(Vk{tm@VHj#zWaGJQPk^GH
zZOdTozHw-cQ_l(05b*<)W(-o-*_}y`JlmnyRe!qX@b-LLuKmH^;zG7`ZNSk7Xy={X
zZl^v}B^$fxBzpgK8bbYQP+187z8tZcE^(>cE6iug_u^mgV<C-5E$`<oR3nFsvILET
zUz0p|C#b6g&2Y?CGlr><GsJ6eJCZg}hR*oX+WR^iPyIEGJAuXBirMrfO&jW&yHaW*
zXJ6@!Pk3*i_w0K0`K@}r&hBdBYu-e6@$L1A#a{LK8`{pQ8vaHraFLFR=UWb{U61yz
z-*$o?TK>>bY=Ii=lIeZ#3<1XprJtgIOWa#wzHmZ>E8Ry_?!)H!J7*Eg2P_Ix6D8zX
z1O7{A<AflVv=}rn3!+#!=)rJ@v28g%YqXM4*?o>^AG=AtxvV<KD&gL0GZ>{#^9u&+
z%?=n^nI_MuWfea`YVi`BpYw~L@*KIcV4^l<D;mZEgfTkdmHomqFA0E9W?p;lc)IN3
zWcN7q+90}N-uHUsj#9Kb*zs^TuUV&CBeLE|9SXZC@Xsrdh_RQnq~1J659FF9t;KGJ
z#_#cUz}s&_jB|=LDYU!?U&@crLuxG2J1hJ4me6q5xr$OjKWe!ZU!P=W0T`6o_y<Ro
zbKX`6mvt{}tn@yZ)>5L?n#H&e|AR_Pvs=cONUtpzmC1!HV+6!$)K5L>E6&e1uVy7J
z|FGkQsbz~ddwkux#G6HOR548O7Qt+n=;{dwQ&QeUtIj0DCpR}VgP+G&h8^Op*Tyz&
z!eJMX)HwjeSt-O%AOwHz@#VtJXmU?sgUe?^m8ff1ew0y9P1y8Sz_m~4Sw5bkb7s?u
z>KtHSWB41455*vF>F>D{L-kP|QG#rEk6Bo=et*wUC7Yx~bVICKRQ=Pha&pUlk*1r3
zRW2Qhj8#5A@~5-hveiHq2$jE4&OuB>G9RX)ow%kIN#zY~)7RXMaP6fw6G0wwP8AMw
z#rC&f`|M!?_j6|b=kOz{x8#(PEPBTrPqScVY%3Uil!J-!7nT=9%G|Udc05f~O$f)W
zDp-f@;r2t+64>c<^Qfxvov;rC*!uh<wP6&_<)y!2r_X_RskhXY!lAIC)r<h-ypdWT
zK<ZKi*fuF6s#aFbx+dMw&B@Zevd7_N#}4Xe+u8Vz4isRS|7<!4%uf_>38Tt3Ei{{`
zpGQlwEVE+x)vVWpoGSi1@E%0wsILAK%ck46QsI!YRIDHLU=3sp|G}0PC}nshnrm13
zC+t$d<%bFpQFR%;5-Mz#W793ouMk@+$x<n$RPE5_j2CA7LueXn-k@hN++frbMUSC@
zghMYKXdh({27+5)onYMikeet7QD8^Sm4Q30{W@Abg|hnS8Zw@rBHw&nCX;luM0Bfa
zPq0l>(bM8IPXV%b#p2dI_Y~Rk)lr!>`vgVWK^0zkTe0N}k!|pya@eI$Pa;pWgr6Y9
zKmr7p+l9Hil&GU^anNSbgW72S4$;5A%DhHyK0}^tcN4+TQ-{|JZCCzejT(9Sm6T=Y
zd^>pI>kLLZU9+!Ga?AfVjPq<ASfh-}a%U?oTDGf?rp(9nFX*029kR5=@j@@Jl7bm`
ztX|6>S#~C>>yruZb`@1Qx(#Q|Amb{FM7U5I1+}wy2;H%%DGkH1!==#PWLf-+&}WHR
zO20m_%RPnp&L@x2n)Itl^L%{^T6>}ZO8M~h%_Cw@?;jQ|?M4@+R42`axwAzUX6piz
zX-1<OJ_otZX0R3WUN+BAzrz&Cgj@qd(DHY~MOlP)y`7e`8RCL6{)weOYQh`e+p<%;
zL*g75YU+HqO$b$pZ;UtLvGPf)kOuDs`f`fJ%nCM5z9Rkl#C0*`JGv242KWd#4}RY@
z;l9=*yhO0a!$*E?i48IPT`nXAf33GyXfha{LPqnn6+H{xkNWzz%C{N1Pvx;66^m6@
zF&cw#)j+nqHMIF~P`?g6)JishSdFuVE1wJrNqb(6?@Dv0{4S~KffR4ic90f{rFBpl
z7r=67lPzXbfPvDSBn8GuYBNflx7PBei;4O-xLEvxKd$L<Yw(oRk5BO|u634>S8JpG
zX7EHYX{Y<txTS<t;{5Ls($?kmjK#i$K(SSS`5dH8aqF*TEiXO85*+Ylwn+4vIt6(9
z+PCAUc?tf-pP=;UmFOy54XHus<yLAC3=)i(Xz_o)b!_oF4B`{5=`{JqIbBj-t!ugB
z>B9k`O-0~@O2^a{VY{$_SF~*|^(Vj0oc_j_EJ&WCMiRf3YcDk)Prb|S?E^o++)T{H
zMdg^xT;mS{9u|58myW=pqiB;dlVAVA<us@H?}Rx>3MPzOb%uUAGN8ymg#ki-x8apb
z4}6VBh>^8;4?f6HoOEP!-sP2{U18MGQt0x&>+|K@Bs2P)59@c^GxdH;m68=iIN&&E
z3I8|1+0DBFI*dE3a9`I@ot%S}(~q9k9PtkU?9C;xIePRKT4b%IGAg-_l2O<(PgO#m
z(4&KD@tX2b3C2bGpo{)ygFyq?AMwgNo@tZieCE%P^T_vp09ioTH$%0LY3ZN)ih@}D
zvrC%OuGDi?fm0GNjA*cExke;yZgb*1Mm<KWzH4D`aR(hH2agh75|N-YoXF6OxQO#8
zFh3GQCZk8O=e-8ftur&;3_91qpzp6ELycLl#U=qe?>;RP`3qvW3P4n7$!KEB(1Yct
z<20C`T~?=521z*-ygb#qM#Oo47Nc@^J1NdlP*U$}q|mOI%a6qhcL|oAP5=XoA3O=2
zQZTG=*>Y*~avBbOHxjhBv<<W3&CldMLD1+`2EATFfK_n*&S{F|s(v_#<E5*r3azvX
zrh_@1S<}lUi-9}8?d4TcT5lG-C5re(_{c*6^^x7|d__~#jtDZk-Re^$+kOENEJ*FL
zdj`t1a6Io<6B7%n6rqZ^ivh47@77lr7I;`IW=ZCq;dCba*03&WF0_akJ)=2~9ooPG
zX^qohQdt<iI*IfjNHeT*I-|I*+yY|UHA0MQpH#UU#N})$)cwlbQB2xfQus4Oxh=Ee
zrxh|k<pFo~1g#rtgBNVZ;KV@<i)#uGBfMFtkW5|uh+lhzGXBw!$C!b4x6#K(w%Tlx
z=j>06&dZrH2!i}Hl1kzjbZFkJisGXfyQHbJ_m);i)v+Ouml#lW?6+cSmoQ~4;5SOt
zdky~$U$2|k`@nJxh;BfqQ17C3_a|Qz{P{Vst}EHQxURgPpCwy91IyzcNCGD+44oMd
zgJ1xQMf)47M94hGL;(j~<}t?uAeKE%y?yzV^!UEn)Z^q?hLRd7{)xp|(iO?|`SJ<}
zX4~EO=9{lzH6dO5jyzQZQIQ`iP69(}sF_a1rJ}VdBpv`fEwEeJd+xh&<OFjJ=Z`v+
z4st0cL;nyVa(7N&zr8vrE)n8nxv2KOe$dX9c>88>Bg?Nw&60)QBdB*^J&*&s?S|Fx
z*(+~?;*+rPd!&?Q^6H|%@Y6>+^@yc;ZN9~3x!|9$Fbz;#nqOAQrOIhW33qQf_vel2
zl^WYmzkdX+oT8kj@DWl1u|d2-W|m8Q>2z!yhZ-GSlINs1u8SBPoNJ^eOnFgJ+}(yQ
zpE)d3&i)MQM_vCtN^x(_9)&obPcnmwXHs>&f4@YW{-~5vNANl!$Qe{DhhN5}Y3zQh
zw;&ON@!2oX+r)7iw64N;>g!xq-iU`kQ~i-UOm5)iRLxCi4qaFvnG%lA@&7(@q80Wd
z5Ta^Dt=mH~^~)QoWaTqo6k-|X;ZCEn;^|O50bVQT52<PA8R=QqqtI9;70C~cEohQd
z5mtR#2%)YaXplzh@8jldsTw!ggrT}3)}7g5qnT#pd5-w!W6w@xHU|V7*0k!0YZ~fZ
zkC<J)yKD+4&I#2}ia)*B?-n_(Mw4^k=uJ>eDjB5Fzp}?aZyH*7{B{EAjPELqv>-!?
z9c+b@vJ%B1E8$w-=IOKb%iii3ZX$G{0bm|vF5LSdPCD)AjW1|;b4h=xMA76rg+>on
zX;y0RrIe0CmlGV(6~9}_QbQ{ie{eE3p&ad?WZ3@R<7~$k>}Z)4o5F1dhn$vQv#Z&j
z^x>9ze-{3fMtYk^K^d~F2F#nDZ1m)F2}Pu3?lu>0J$W1MBh*;u&{VW#Y^%=`Vw`Ru
z%2;z6XSF-5)XuqDcZ@EPH6!q`V|IjvWN(l5Dn#<c`NwE{SayCV1G9YD|GlYjW0^ds
zvyzO{#P?5$zf~`szt5n<S6xzpeMDked1Bae=<$0{GMU4NA?nX(@>!-yMNLP7&A0>=
zU)i85i10;sWZ^0Li2S#cXi1ac?;#y&f5<N~6`5VpRi-clYbhxuYH8mEOfdnN{0uLt
zO<m`2q;fjwXrqSN9EPhjKK0rYPp;cff%<gGOq_}tbSa`U1&Z@H;6y%TVh6WPH1UpZ
z`&An09UP%*iz+8MCQ@6ZCb=zN&xk3oZTWl?dU`Eaf66udnq@bekjX7fd5S+}(NCfL
zvCp7KZvsM!cZl<tYVxrdg`@(^2%Cq7OWF>)p|-hVb7t0yXxF48F<nQ~@H>aFp+*O8
zjYHMJ7vdM8r-LgKmPY{`7p~Z`;b1AiPK*-*e)_{HZAqS5{L4pg(A(3$9HgDAe#0SP
zQ~)aP(4?&&lduRx7U$n5>yq0eDy1}GD<&%;PR@o}qTZ45a_UsHae?XL`}bY(KC`_D
z1MhT4-!g|NqV988VL0Zij&_&N4L>6{=>~!9P~mKnDsFCT=y;;=hY(U>wgzqC`PKw(
z!__c1N3YSb`Gcp7UzM~ipr235I)->#Y<O$D4n+#fQW3TIwbtXG_D@@Vh-f}ISj!vn
zs=PM+wv}j@P9w1w4~HcQ*!1D@>rkkSS(q$207G7xT4M*ZrG%b~Dxwo7o+?_StcN||
z9=Q&cY45D9#{YsW84W23DH>PhlSHLfQ_2v%)(VvlU&0{#&D-x`tD;1|;=!kj!~JQZ
zGC-0i@Hd#JvLggcQeC@lthqu9+-Pqwt!@5n$YNPzm_pz%pHx4Vgnef=kwda@4%hr1
zI<$PVpE{%h-aeeTjD}iL@78>^OgavCd}e>WrL2Y#_Loe3^vJCPJfp3~J+W-$EMl_R
zTj7uc(she~(1YSC9A+5lUmq({#e3D|8mxqaSrWXn;PQ@=!Q1wqA!!@}QO@}6THhpH
zgLX*Kst8za*}2$iFhjs+XAaUdKQ@~duAftbM0)JAYl)?r5f3XT{*aOPzNOwxOwEXK
zrg`{oA5lNw*dKHC!sCy4M}$~HjN%0E;x;SSuFD3lyx||$(HFuRdDC>t6Tb@!p`Q$F
zb!roiD+l}ynRgS{xjlAUUmZ|vQp@wunQzoy*}rv%p)S75Kk@J@GV-$v7!UhlM$vf*
zM47!dPHNi%@4$lWf|aNV%X1hVNiH?Mh59Gm>uzxS{0`3ldC@&z)lA0WR@BYAMUMJc
zCsjPVK?_&A0Oa7zvj;caM@~C0jFk0}&I$TAwXlw!5N$Jf7MO6HTcaigV#^!g{3}~M
zdSx&~&&&i@0<|T)d)G#nEq23X)p?Ln*h1vmvc}I9=j^NFf&zt75`PDkaD++JLp|v2
zvN8o4-_bozQEmRzY$r2f|58a_K>gaBf+0fZg73Ly!uN__NGD4d`<lE7@XA50qre$e
zHa9fjFD@<F-;8K}ZS!6P=o0Ku17|9tvV^hX)cDtThCxs&0xc4Cwj>OJJ)-X%J`x&s
z5VJ#4)>q+|5I1;C3uuf-?`H><h{KPrONda2L+xJ<cX4DOCuM%Q;M*h#nF|Y}(8}u5
zIGZ}d4&3wi@co+z52g0LCvzv1G8u_9bsT;9mqM_njr>LXbU&z{rdOt_7&*^yDhwxk
ztMrwd`3_b3w*Ce_`e$bU+2+|K69WUopHdFX?$}fbQ3#H8g>jqI@XyQ|cr@3|bEIci
zXD5zF?cHpI7Bbv;Gf#s&sgRvaj{xmDPmoNHrj0zCA|qf4sI>Mt7!z|&^zh_ze-d}(
z_V?$H!4a2U9hB#IRHbk0OzDenu%blc42K(pf#<66<T8t-mNhw#zs55uVf(jc!}MYG
zYMBZFcay8pCj_|Kjq+STrv6D3lEQV9-SZvot7f=SKjm$WDw_F8SgZiDq}x%{8qF=f
z*+|SSqb4@3o?9jH6yO$dcTnmJWp(gXXCve*Mk_1de18m4AZ;1e+n<JMs3i^R0&_Vv
zX9IPsPo!?kBK7(z=`G(c&zVUocsMupK!!$19-GT3+mp=%778!L-;zQWa3D{7eH}F#
z)!wN?1!*vmdgk1{AZa(XrcgIEtvd0r--dK~+v}Je=e5$r$H;lB$-{Zp+n0LF{v(lK
zyTmb2T<!1A(G{0os6H5hv3!R@(fg~2X@Su%=6Ud>7Zz~XM~-6a3e{n$^J#{Pl9l|t
ziNrre-)uve;!+Mn<<6PBFKgdL_q?jSXmr^;(eDoL(YfmV{O$UkYfTt1SLOVm5{J-D
zc(&VAEKheP<9$ns9x^-dDnGlzy>`~RLSIHYu6w$zPmM+H7p=h(OQ{`g%FdrOo6>Y}
z3~wK39WGc$M<<EDsGDxbxB6<vpms3E-p!8FgOF?T`wg~eK4j{u&EhRLPgv0;LLB?~
z+!G^bwoH0|McZhT)iWOad6$c?L;59<3f>$H2a2F(kG#25&cxyx=ZkDpECxX$aT6gE
z{Z^mHHk@^nsjy%biEl30++RKykVj32V|^n3NX;M(ORZI!>_|jPy@L)7jV&53-ul`6
zzO6gf(mPKyRcNE?PyNL5K7T&Sl>d5`RKBw}F6(0Iv^RXik<!Vlw&!@;Yb3X2Prh$D
zf$d)%?BO>w6z=F9-4xj*Z#3JBh>X%prOtkTB@Vp=u79P9VcC!T^n@q#`8@@3=}q=2
z{g#FX;X9$lLxh)fY^~mwT%|W(nhzH}tru;O3eD`y9F{l!`Rsn{_Rj!O$I=eBA*~oB
zSH-Nqf3nhkEMbi6?>QA$VmuOR0T*dp@jAM2{Y|Z$<dhh_!9X*?n^|j8H$Cln6Jj<x
z^ifL+CT0#4$VyD!5AQznLu0rNr&$_q;HaFr=RAiD?K^L!DWKH31~&2eEG1~RWHpk0
zsQVvWv|J#~<as;9s=Vgmka?$yE&NCb!v;DTq%z~*m&B)xN*F>;2nJVvm|kGmABVo3
zV5LNT&4lu1_Y47!Wpf4Ip~nV!I&6P;oIb<MZbf_XBH3|RA#zon6nzcWTZ(By?O>Z0
z2Z{Wxf~x5;u4ISb;Q2Ez@1RmK2x@~!*9qSLk^hD`<1UYKPQ;!CF>EFBcN8=y&FeN;
zm}y!cp}{Jo91i3UmAjW!isHbMA9aPGlnd$;tL27>Q$keG%<XgIZbH@-0}5dc*y(l(
z2aBW)0S7$_M0<vC{Qe!Sg8Dc$eT~0=;+@QgV#mSVq&%2OK9Jf?EL1j$G3Fxm^!}_}
z&i~P-P;v-+SSrP+O||VRP`hn^SKUy})XavR8*VcSoP*4FY0Ron=}pnCHC_qHCuk~I
zs-rIb%Sd_rQW(Gdi9uOOKPk*Ow(W*;<>GN>2G>n%*`Wvl>b3CF**@qr5_l#wQPY;4
zCz(;Wp}I_`%$*XYoiycv*d!r6Z{LIik@?yNWF7xaI21YJ@8GT_nU-eSO(ZX`l9UYi
z&2@e==bD^awlJbJ9tBa=%3YjQO*#gp^w#vOQ9sAg&s4CwyGg^&SrdreT2iFJx=sCE
zN=ok=qX+bnC+b7NbkC1)wdJqmsW;KpD#(Lsue7Y22vuT)#|y5A@wAD)&H8@7?Z-!&
z?RS~qU4jOnA#vqz+-)OItPxn(=Tw*;($~~WDk!VetBs4zU&C!FTG(9~PM?#?1v=)V
z5KG5464r2-=#-{Nd^xDNtTws`+LJj=fb%aO{K_tktl<aM>P6d>c^o(|I>>fWF?<82
zXbigIp&Xl!_EU`Vo}d3D*hu(-b%p>^@N}RAE0Z+lalnN8E*xygersj<)~}QfW4}RN
z3R7mgk&_Te3xgSf1Rsd?e?MR{sd9#o`|e|&Ysm!WFj3`W1YH%(VUr9zhTCGx_(xd%
zBE;ANL_sn)+OqcuEoEdd9%G&>$;fn(1L-d{i#i4-kaYh4{U$2aG{p(JGI$d@687*N
zt~?eQF6<yBh&D)(k1?3B<t>HkeOC&`uJU*HfPG+t74$EszDsF=)cU`f?H^)9>YplE
z`v><Ao@}vW${@43=3$T2E|)eY13vnGE5GW0G!tDJ=Btmvh*_;{IL#fZce7=;c>339
zJI-}cYS@XQn!bI(o>tcNB~t+e6$Ep(Z9tMpJbj&Tn|cj3@umazk(s9arRnjQn4Qg$
z^lN)cF~gBq@SylZ<Ykhc{|~{rl02cx+Z{u+bcx-17c~>`PPGGjq3snT`hktO4-So*
z{su&Dg!Ue&cRQ}AGP~nNC%e$kJhP=E8?ff<kQ3b(FM1>uuG2YP<nR6!R%9sHP+-|X
z(jK4{|J?C;@FPn&e@hL2cwWq8{(HO*3#oqc82*=<#fbfH(H19!*WHbf>d0_lh8AvR
zF5C8GyF5ttnjHvw7u0CRB@V(?;sJKsJF||n$Q~-YmXh+d#Ck2yLt!1kyJZxlA8mXI
zry6yD4|Y+tE4-k_gn7#u{uC09y^TaG`oDyn=VRnuLf#$)Yqj71MhgiS&3ya^4C&ob
z7TE3l(R1lqz%d_{y?-&8?2C|10FMs0jCMZ#Tgu&HRCG%aHl8c#n}eHA7G#uFQs0JJ
zx3prRUk*J^$M`iSwFr|$FU}YeUvIpt?qL0<jr$IG6JpQ{YS>-n1o-@K0p))_{`Z*w
z@1Y0IfPzg;c}CoW{Qr;Tf$9e)>Kp#MuQ#Ggsd*xpnT5e%Ud9yq@y7(-`JaBS|0F<E
zn-B(K$b6~Ax3zr%Gsb~9jvkr&*==j?z5n(=bJSWg>QBB&_{pLuP6h!Y5Uh!f0yc*N
zE*aDd!^8~L`pXbm&e^yyogpPeyLgqXrQ)mHyeIhkPTc3!lizU0!zodVP1#(>J9VzL
z5sK5IWp!b`S0T6O4IzTQAp|JqzD<G9#M>zMA*Bj<EAU#a3%RWW3NXIyHWZ}8L+9qa
zMoJf~*0Op-z4HX(v8dkSwfil1r)zR<u7O34c(DuzdnSU#LqXYo=ekVTH-r^}!cuEY
z`fYuWcQ+z#NDL8KG(sb*ba3&$+7fgqY*{svt-tWsH9S#eU1+}FmPEQweJbW%Go>1a
z1j)|LXJ^<%h69b7{>u{SFMgpb*STZ|#E{#<={%hTzqZWlHuaLD+$*^9e&ab7)|Zf&
zep@{T^$p}e6^vD@Nou8hzxww4WwEqsQp3u?#_<*JLP8$&KwRYoT1R!WURHqO$<mLb
z<OcR5u}Eup|GvbE8hE2g@5Vl?Qhjz3`j+}q0R*#bnU%C*I$y+g`O`W01jUfJ21I43
zomY~*dNwwBAQI<gXKJ<n#Thi^S$U%*pX63~_0Ghx0b+DM@{_h|Tj6zrnPl!k&Vluu
zNoUQ{u}J=yZk{o-r%+m1u>={3nKA5oAJxIXRI)g>zDm1&MO)e;buH2CW-3~h@O6~*
zJ#2K}PMDapaq;V`B5FuvzU4Do*|;&0G~}kJ>_%TmmBd91YA>tt%7)}Y)H5aRgr$Gc
zB{Zd(q05YHxrW*1OWspJh~p_?Il!)$lvVU-o}SDY5*>+V-6b(x>h=-WLW;@q3`=ZT
zS*{r-T<Xpe%u`!>TxVD%&_D809fAq8^IM(iKJ2IJ-Hf4L#O$<+RlU@4IAS@i@`*hm
z&x<F@^Q-yNyM~a{EcpO0vMd>NGMbxa$;>EmM5J1XVRCG~LUm0Xv>|}Tl>cCa&W`BW
z7&9olYwh0B5In|nfy1B=aO_qhuJ*&+q)vPEeB<<0gnyE^WGv7|TkcPLD3@pnZ+ilF
zxbdbTGtPr!ws<9a)RB$KvJ^@WKp7|rQIoO!Lj<?{IJa)Dde7mNw>X0IpOWhDshiyc
OzLezDUzNQy5Bfi++D*X#

delta 31437
zcmagFWmp_d6E=z@5F`-XB_Y8jEbhVG-7QOScU#;gzybk+yTjrhAlNSM65QQo(Zlne
z@B4SI>-^j9sp+b^>#m;an%Rpmq$fCH4I47Zt65e;RKs)eXlX1O*OYwVe3u>ccbZvL
z#R#1dkn)c3Q=zz|iHgLyB)`Wo)i~Wf#vA3#yght<l{LjynFA9Jd^}F(sHkRvaQeNR
z@VyUYe%S-1wQ1INQ-gQyUn$%&cgB6Pe^Ao(o2BC*2Ef186H8iHk|XR$a+oQIe@D0k
z6<9I%Aev-hYM^}mfun(PCZvH<<ETRN4S^9^hqnZi8X*~xnOp<q!&wuPz}>{`^~Mt;
zY5<9*mJ!>2U-XPag3w%c;nZQ_hCCKRU5Ucsfq4g#_Q?Tq%gng33&Jl+Ab}xhR<&@Z
z#o4Ys!AAlcwT*K)7X3<st<x!LKe5UfG;TlXdqF%G)@Ve&)zS3z4DqoZ5EatT%9SGo
zb1cmj5$qeN@3b$pTV}T0E~?VMEjN5JO<uJ*usQY2*hxOT?>$`6C&5AxsXD4s8g{ZE
zGtyg45hlf_Hg36O?rX@fMrwK7>%&!E@k+_%H~zj=&bE)`A|GI^sWs6teD692`ocw|
zaVc?7Y)_ovuUaJGVIVBHvppG#G-D-&xmo#~Zq)0Fv&O?jGZ2n7Le`*vKi}U3CjFA6
zy=5Uu#i2^_2h7pm871ot^puZxJJxc#l7fQdt+sEXi#?2fCnmq2)(?Lb!Y`*P$4~3J
zS0xe2Vl`ALHjK(v3ne5~)GP)K5xfE}32bN4zIb?+>rDTU7-bLL?neTCS154J(_lQ&
zc7qagAtxg~J~vHnC;GEzx?5T%r3UKkE9z-ahc#V`CMN|qSza4y(pP_|%`TkQZzjbs
zxo4^v%o<5-k7K9FIG8M&QN5Jot4e5F`BmIt=UuLOcTb40mhKynEe--AV4x-=zR3Xu
zga$(qkW@(i^oRlx5KAyk)?eok{=eDq)j@*<IlxQo|3|Rw2%Gpf;-A^%zDDI}YK{Q7
zRzFuT;vf9v^WzmW@*i#pE)*cxJ4WPRRRC58Vd3rfuU^jTq}&`Wo#@=yq1rAz8X`Q3
z^d$Q&?#BMmUvtIKE^<E`NcI!6IHO9duzk!K8urYoAFoJ!v1BF_IkV#gv>ASQX6;9K
zrdaV>kO2P0w!#op(dEt<DHzJulA6EKoI}T4TV0O_CXDx6>FY2U;egD>3G#~b^66-w
zY5_)}-B1}R`@1~9kN*m-;%RaJ@lFv<TC$l7h)kZ><JvCZ%zl)bmZGm!DlTMHM^3*r
z-`WW0#mM2|p2r4LBpAAnN-^~a{XwwMS!j@D-!3yZirglz`?J*3p~UvtCeLHrS5m4d
zgo(+b&e>e7{?O=$dMk)~-=+|G(`tbo@5jB??#1{?=`bXQv&8+TgV&)Fhmm>qz<0#9
zZ=WO~em!q5_RI#~eX8<g8BW`q_ez|)>Iy64x(p+r+LE_#CsT$s#<@<MeNPok+vBMM
zfLm6vk@a2uBHlC)GV{jVqf-1mQz6uQxWvsnb_ESP5l*Pd4bro5BeqF%t^RD|cE8`4
zoK^+6-b7~%Cdp160<8s|NK94R{s4W&*G<c4v%es&ZszI&C^a$k12h+)MJ%k87Wju!
zqlCdd<2YvZ8E@eOA4-m{9$c<UP)IqlD*RIVwttV90O6ygl2RZ&cv*YKFzxPl-76Xj
zsjv$@q78p3kXDK<`gy<?o_$ALjZ@2Izk_$C$8~+WnrZigPhU{JX4F>bx6=}|X{7Cz
zv1nnGL2}#h^-CKbp<pYf^5z+tIYYG}&I)lRB^dGIdf@j9Vp^vBF%1TShNkYJ@0BwJ
zbauCPHIN1}9);GSHZmJH?<JG^?L^c}WJR0L_U|(pkR6w3@`OACKisAK!ZA^?JUO1K
z`kEKEunEnxOD~=seuMl=6u4d+^ZRs4K~>Xz!HdL3C$&j;q1%I#QkS6&s9o<sn<o`C
zVeY$FYg@?ed0^}5vy@R`SC2|Jc6I2{n+)fVdCR2lWEp(V$8w|Id<z7=KV&z*`t9DH
z{u8+nbh`-~Z3bK?goSov?zkFBL7lyc)J@&A8>(6!4S|9hy5}3%OzMZhL5%wDTL+nX
z{dhew%5Fxq5p7I-d?_<7r|PaYs7&RP-*%E`H`bxfH|Ezel|H+`&0x!SLqc9f3m97N
zJ%k}C07MHHqn+qj3R^7bO@A{atYOTned7@{dobX|QpK2b3E)bRY_kZS9OUa@a%Rr3
zk7YM&&mS1Ngm~8*J7BwqGe*1d!$;pkW>pj3j>3A9uUoa`Mx@^dKPOUY-Aq#dV(;<o
zWK(x4<gHvPuSsm6I7u{KNU&~(tf91hr$|nVDK>LX_^GCU+P8lVJ^&t`!rEI0w@5+e
znOS_gge5V5FW|n>DZ~sg6JUM+-EvIrqr@n&2REwEg4H4eTorghgHYEQm+<$U_G;?2
z=aHvf!=ELvp5#@tccEg}eYU%H^&6&3`0B$-{5S-uaKk;UQH3V5UBop0#tt1yM}3T;
z$qo0@Wve~X3AYRxuA%nY%6<Egbq>19FB#~ViAQ$6&Of@^;47x=lJfzN(NKxhe4kPn
zkDn`P_??-Yf0p<-IAzK-ieRCi20BLVo6(x{AD@l9eQ=3uy%Aq%1*o}CZ<8`><DHBb
z8EqZBEu3oXiy+i$tWd~IdGF2v@H!6Po0#TMUDya=@I4+NargrAGStau;sa&4Pp1@H
z4Dh_|Q0hV~@pXgHDi0huzsO(E%%W<bfb>WPUN#<kRwydIzC%{U2R^?PF9nLh%o*96
z(S8R<1ue<r5~1{Ln(KzmjuiqtD<CR<1@J;Ok_s&?zp9tCKRPI6hY-?@`!SG+$&Y?w
zJD{T<e1#acCI!S9{p$6bA5_O0T<H*#ZPU-iU{3O{oH?m%bvGUB5TH}dOc(A~ReI}>
z|A*@#wfk+`o1WMDKosamo-C?DLM{#bpn)>rRqIhz%OdH7@KQH5m8?kk-g-kbC|xZ+
z-J*Qp=YFiz+m9<7s_)Npd;XdioP4IGRTbvsG|LY0t`wW}45uX=<^jbIGf6-q){upn
z)m~gY?pv*zbfpGI+4SvOeNJ9D<v~v(i}gK#z;B^73e<O}mf1it$-p(^+KTmcLF{@W
zmu=EJ3|m(LL`-3tcW*!$M=5Dk+jF<pFu=tp!g66$>m%t!hcweRLP%}9?cF~?>bAx`
z^pD*!n;i04s{R+Z%pe+~Ix-<k&GVkipKaCnCTAt@Iox8{TOQ_f@p*f-4sX_Gzpul$
zf%urW=C$7Exhp|h<D9!&oHh{so%^#qJJ?YyPgGEAdz@as6%2yfkgv`xz@O`R_n1Hd
z!L-T-YUHO)>sNPYgp$Uh@U}d`>g9KDqxBn0ihA5E@ZY`B1&I!+)Xg&X<5lWQbqc1V
z%SiCl4ju*~RZnbRr-0HuGK=&HOLI*Y?Ls6~hm7R|rxoMm?1ds_CheH!dlR#_iRzRu
zjC)-~@}2XwfFua}+4V8Z|4_c$ra=LeLO6fOlebR62FY7ebU*tNU%u!y>ZHW}oc~*+
z-bzuKo5q6t3p6c&{R>$7=_DJyWwUsIMf_;e)@R#>>D?QWA=SMaU<y&~LA7@y;|<~3
zMO0B`bxC>q#$Y+@^m3BLrTO}o>=lZ!jLniEX2Ga2qNJX5VRG~D^G;0VNxZw%)-fe@
zxl`BKyLoZVlw4G*u1r#CxGYvYCALXVMwo}|@-e#n9SFVQZA^+|9$R#^*M^Zf7A*@+
zgTM+wlRXJEv&r?3N`Ob-kJx~pnKLwN8~O7r=cT<tqfNsfB4$L21Adu%AOvuFt_pNF
z^t3dT<@`&tClRif{VYLYiC5Yd-u2Nvmt{PawwS6>j;ZSMXDbsp#OA%o?$@Hgphd{J
z12*{hpb})g<LOEnp!<FtGSfQdrHo}|YS;tl3=3KCEBz<}?yz{31r)Mn{Ji7&O<v99
zg&|9*&2(G)F5Af>@Y{GxiteZer~zp+(*6^5Rk-FKW}9!YtFlgJT!=|`vnu)<qzeM5
zg81N6&UDFcO73S*)n@d!2)I>2l<0a5Kv`@ZR*-zK^e&<bi#t0WTIf<Bmkl%~p_)BJ
zNRDeM_B6Q<@<mZm5#cR{>!D9ij^Eg8K-52CosSejai-Gll!!%lwPT2$DwKLQwOM=|
zDfu+)M=a8M{pKpQ(WOzEFxlZJr7k2y)Y8c?z%_9HY)>$NKh^M0@FHZm#TI4B>I^&#
zBG?QQ5>GzH69lFbw5wYG2+ToFu~M4WuhZ{8D0sYdf|15qozt0;bknjle8&tandGaL
z<!`c8^{_@;Rlf!<IP1nghms0nwrDH@%*n$z`dwe~q^@McP<5TB)Y9*N0BEw(6yH{P
z#aILDD=bBJAWwy|>uErDT~}aGoE~V0_jh)-h<<saf}#Y^hc)LGf~ke;5MWKW09#*O
zgsp8F64ar>31%w_eB6B}ml(78Ap=`F1PyWWqhAdlBgxP~W6##%f4)T`*!C0W-Mq_Y
zM>&yR=Dqm&g--=GN6*o($aU~7nETL`ht8rAGo@-KnOuS4doWX|aZn@6A!zZmrl&M!
zT;gnZtvc1s6o&bubZ8ebaBXa&lH|+bY$yu{yru+yb%50}7<9edgB18phxV>-Wg&zS
zN7CsLWs}&-fu`VZyJSdY2(?5gMqN&<gg9BTTU$5r&Rb-^BMXAiz;Ur!MHLO)b?9Iy
zJ{x_B<^(FC=$kOf=saqg2+(MU%kSEZStdxe$Mg7r%gOHt_Fa$Bjvth#37ypo>P47J
z6~fQozCdLP(o|DGt8K!(S)cwJoDk#D&EWz+Pbx4lWj4!z6J+WNm_eHczvyWaEM(Fy
zwtQo(V?Iw}GN`kzE-EDZ;#x_8KYrI2|Ckpekxl`O#3&aXhydV%B0Fz`CGMpqUKdDw
z9q(u3sY-p4y)UMR{Wc1Jo1Fi?Q6t$}j}QH}Rkjg7_j<K@lqA^;N~u^|2(xuClUKvi
zcG!pVaM^5>0gVDnMRKpoKhr2yZS~%6bw&E8Lh`tdCkVK5%aqeDyqB>(9dvRo1#kR{
zTTk_~TcU3M#e-B+CQQ+2KWHADKTIkG%BtP0;v|(R<=Re6R4g6orxRv?-`aZtjX8u%
zY-@{1Zp|pqz7|jFk3@<8;QCs%QFd?V9A{CDCv-W)h@1(Egx<OxcT($bYOBsiqEV5W
zXzfq$qlzeu0uQmPtTUUh)WNx8xk_-AcP<Oq?}U&VK7*h|huG@<Vnq%1kZ8qROlD{E
z*dZ{%^%9$YM|K{w;bx}hQKjT9v%arm$F>FuQs%0$utqJ-LjLtxp8kiY-*F~n&8Omd
zXrk5E?L*-fdL5c=yxGJ{S#xiIw@<0%qI7>8WC)TFY(5t#%w}OMb%~};k`{f2es^m(
zTvchx0fHpwNdJ+2JnG1}&G`QDs5_N}QWBB%q0%UhT|JyyCvIZ}HLsYxez$qI(B{X-
z#V()ng@j@KX7(P&c81T<e_EZ`FvmPfmWEV`9FrU&fmoS?G~W(CjCwux;Y38dpVb{b
zK9thL@d!fOgIU(R@pWf|(llPX$3T?%iDl;01cgAu@}~5=#M%Fl^mfBc2H{Wt{!tLH
zUMK{Wf$^lr5d3+AehW-jM2i+7dko-T*vGL|_NKb4Vqz2FF!VCW;D~*lY2TEDaFkI~
z56LN^_fP);ea(+s{N?_JHsLSXpMZ9CA~4qn$6_2;vqmQph>0uY;>icL+k~l*XM1`u
z6F&ng1p`szuBjV8L$3tb1lBY>oy+QXlMN^S*e&7!M|_d;O02$gW9;6U_Zx&gC%ULd
z?(hQF63OxlS59o}s?1o$520c{aUzye5xci@Pr@fY=U3NS?oc_o|8S@>=c5xQ4Ycah
zW(|<uz2Z>a4&7Le%tE)LKGObZRt@7d2Z0-Qqgjbd%J=fk!J?O`6!`(OPc9^XJ_}xo
z@8R>re69b@y#*o{RjLkW@=RDdr?t@|yW8TpNBuQYWP8lL)SSHjX`>9T{_gBON7gSC
zq7G%eAR=KSE!Q^v))DJYU6J~CEqE*@8FFpcMcY}DP)LI^|Ee?;OcK@+SY6~V0^0jK
z&3R~qf4Pm{5VqKM0$5O1%Cj9v?q2V~hvNDn&!RLDu8sg@#{?1pKEA3SeLpEaI(f0`
zCDWG&5V-t?a9FSLZi3O+t~S`l>_zMNzZ2svUX-LKN(IbDkK?(<(x}|bX^wvR8X^DZ
zR00}+=f5Ox5_wC2K;4d(4*uON0U~)}+9Vurl^}84&>b+FSX(ZiyZJ~iI;tqf(N5ng
z#d(+GWL3wte$yP$Z2gSiAKz+n;9j?~E}!dpl=2UWYK3%HybitHrr*xqxW11&^MaxI
zw-|IXuA@PtTn@}&)9RB>j1i<Hx6D+1%~;f5Du+ofGYb=yN&CFruc&Fg18M_bn^0wu
z?Sx=>4;aiNYXYIvtjN*Az~*K2xDR_#|0MYg0KgU5LSCh`y-z~Z?-h$M+f|yA&?vMY
zd5vt@4m=3Upn2w&QU}pEce9Ek<8H@Tan!3J)vh)al+kBO1r=Lo(c9wfo98aN;-1_H
z@88LS5pl>FObv?^_AO0DL6&#a??tY{w^MnMgc1HZeuVBd_$o@vFX8rK7)TgsfY8-g
zgH@vp+z+l$UT{Ps)~zs{Km3NNjiaZ9a~pjaylY{8AHsk%6#$_&@kZxgSIk#qCF7HQ
zd-%oi0--)N*gpV(&pguhO1Uc7{__%eeaBCaHHjg~B`p{%8trTg@@9QVgd{G3&h{;7
zHv3y%WL|muE6M4N3_u+EYCfjjw+z`9OyM95XI5km5)WM9-g6+XBG1gkA=|{e9muO&
zKVgrBKqexnDqpH>8|RcrDw@=?r=wX9GKtwiV4sxTmdnCZDR?~{B{K&j;vJ{tg79y_
zlpzO_yL8Ck0nkZVIGwxPNP$9jP{R}Ni#hmsq0D03=FvqLGaf&Qp*IN+<X-fVzSl9!
zE~WPqNM;uzkI8UJF?YaS`T6%{_cwHD>TFot%2G#ugBvFjo)QU}k7)tsDZcQjTi((B
z#iikWi7)FW@LLo04DtE~DsSo#I5+;szoi2x*X47R4oFs=EcP3JYHgmAPhZx`o<W#=
zo3kQjkP<Nw4o9m<Xo6govAF{K9UgiqCI-Had5C#dI<nRWbX#N+dtj2tU@FEBtv?(M
zJeR)QiAWP~7q5&A_Lp!ljprjL52|oO6&wS%rqRzKe94LhM-1(3KIG4ExlRRLE&RQc
zI(7bPBv9NOF;`bVy=uzlwK$V-84@AIVNy114eRjg1ATCOu1WZ8?d#Q|Z#k#+yO(~w
zPg?~+ID=eLBR701D0w()VICJRmPWA&@$SG?-Y3mi)wM=hf8#JNsrnNFsa)m4yh~DZ
zSfF#MGKoeMz}uV?N!2_Cb_$b-W#3?9p}G$P<@68!*du%AsZ(-a4FSJ>|Hj0d4AIxP
zD8Q*RmHX{XK6k}l2V1#i4x<9)YCU{OWQ140?7Xk{AbeF;nrMeu5dD51^3vm><Hb&N
z?V+EorNSgi`Irwy6ikeA|9<htzZBwt5p$KQUUU?-4R2K0J8d%hEirXJBKKD1xsLw>
z$Rbg%LsmsF`=L+XHzMKL9eVN5_wYj*)aC~4n&*ZF3FRpw`lTIK7sW{2v;D71@W%%3
zW9;TjkVJBdL%}rRkpNt)BJg&W(0_G_u3Ac0qaE984Kb1k3E$28Pf8v->46dRUtJ=r
zd-B(Me1<PC<|?g&BR(=e3bc7wuJprWfdyl-$OO6K+XPam^ILSV97D;w+)&cMMzDC4
zgcwqhNkIfdL{4r0C)$vFhTNIyMy(|sal1<Gj>BZnY^x1F3B0FXn)6$WE)H1S5>VFz
z#yZM#fWNx5+CDcNzS_>S`V>pccCuyDqL4%z3s6jdlu5r8Pohoc1VK5iE(EN=R+9yZ
zQfah`{&XG;OOA@~T;zK+931d7M0a2cN1y2ZFP(23B&Ug^miapPU?ZZ3y;T+*vh%5b
zIN{~r(lWJ)<6(kIB>1f`G@$-$AASZtG~_R=(0ozzlS{!bU?47wbSl6D%!9LG`A&PE
z_>vOXQv~}VPO^gvf?A;adBR#>6o*Jf5^11V%sTV{P>zDkKT|uUi|+@~3MMNBwhw?!
zTl(HdPUxd<96i&AAI;_!dbkbi=~VL7eBC;a%7uk^pE#P5Qz++ZQ5&D<8@V#k96F6!
z8-hi7j$+K1JftcqDFs~QPcc5x>-!Bz_Xb~uC$~j;V@Fzppp5Zb#eX3}PblK1Zq}~y
zow+skEj3%~`M_fJ(y5gL$9{r(Pd<^o%__p1<DVrDf|0wqLdN>o=o$H3VMkMWxYWEh
zB^6BCGR(t9qt)lgTu;cKqAj^K`VB|T?)cginJq7KQyv4`X`aHMIu=znrM9D^Krgz7
z_r;3c&R1n1Z|kkymuA|oB?++PUsdYCPWILnM-z^*<{sQ0$vtMC>B_?XTz_%5X5phk
z@7{ojz5AN#x&c_5;;g#4zw3aGiUYljB$Y+!5D$I<2dw}6zUuno6)0cLxKm=|=i?r5
zLs5g>Q^)t-L*-~}yZ5>-FH~QamhyCl(b!8WBZWXJiy!~pr!}vqW4YIa43S#-9?UyA
zT~jDi2)JI3@uxxZ&~9cs_}Aw@vTx)ZLwSkasv>hnjIx`oG8t3R<ke!l{8HsX@4!-|
zX+t713So6O_V@YEqqiw<FK$vR!qL6t<N<D8bP`{_Io_a}*|5w4?`z6kC!$j^9$LlA
zprs(37UmMcsn4!k6v{;Wb!sEJ%GX;3z$l+OfL;bu9?tS)T^4C?WDqlrk5Pfr2>0L-
z*|n1Zp}U0(lsRMW!Wi3Tq1#ogc#*yxHxIiUD?Znlc=i>d-z*HuU#j~`*+oo^#cd@F
z-rj$O8GfnsOUnyI`DgSy3eN5>ylH|zwFPofQ<qxwKS+!}N&9Xf;<%vNxHB-Bi}$I^
zT;a#L9eC0@&GB$5<}Wt3V2L^QlN?wv!kvt=S0uG&Cbmh4y|%ShVhGr9)Y^}`K2Ns_
zK=;^hC^&cA-eF2Umc^X`<7imPw(i3SfVp^vUiuqx6MZB@HdBd6Jw|nu&IHi1DbPYa
zLN~)KULb_LNI7eoPQ7QIKNb=zfea{`_>KR8$4Y5(HBCH%_ANDkXzm9aw9i)Hoz`@P
zrL`lnd?MrS4WAUV8R_o1%PgFws_(<xND|l?;OF31YV-(O9U;yNUzzTk<f2KiaZElw
zi<~N-OP4Yr%l9HrIw6WS{)q!Tfu6vFHLO#st5=@;hjw=0*~XVEZz~3R9~Jz?B{`m%
z#U4XxFZsNqs^r*@=)jK>y6@W7Z}Pg)RNt-XQ7bK(+cp(r-s*KvBk4x=iqUQCm8Kt*
zZ7kt-Dfr$_tI}X$M)6K6@5^HaP@@C2r?~uqZ`i*TBLkcNq&Y$gYrZa7f=;mB>EC;*
zNhq7@>ZUm-3gJsB=bQ=fCoY^hM%fJkS7nwsukcg}6NV><Zsv%~VqJTQV886%L{K(P
z|9djLOy=d<p{s-iF%-)ADd0#$(J$Gibl(}}4fBf}Yfb!cLxj3_jsz?qxvJx{06D&-
zhUzT%=xw!Ng3k}{pNos_KpwtM()XeC<_g3uG~at027%Vl=@#2)melnPTK9sG8`wTg
z8{T|MP?yrPnr%<ug2s$vD0u+=)KfHI3qSzuW*~V~&WUMdXO(CwEH|)0>SUQ{6tY?+
zR-7m>m}SK-YhXicd5z1Vat>y3|KpZH7><N{c#V4~U^w3DLiGuh0pT)sd*kaemW=sp
zXqmcVIIrRmlE`&SHHXGdk$Wp$n=?lM-B6D3Hp9X#5}r_8E?1|=vrH&bq5bO%liFM3
zO_eqH8C0jnr!+YkVr)LMu2W@_E55kocAfG}PMDPaVJeYf<6CA*-z1%}Y;pNGJ*uM&
zNhM6p96AG5oy7z3CmYb76bjS@D_+RfTtQSVAhr)`lv#c>#;JbFhmiIfa)u`E{VP9#
zyH@7RSs*cDom47A?`vXJz{bOqici()>qeD*4hzSiqM46G`%Vi!d6${KPWP0XMOcLY
zW>fBx#B@I#^FFKg<Q&o`R4!fzDUcv?Yq`pR88R3dNkIj?KcfIr`YwH?F{dwF<COXJ
zBVo(ff*xv=4rydDOKoPlzFzhc%2-(BNk@hAllo&H5WuQOV&bNAY6hE+<TFmzP`!9?
zhd`%JF7xSU?2x@lZZir|+x(cfJO+X5rKGh35vaZ?q`|q=KYOsZFNLIH{*W3L{TGXx
zfe7JxQ0d)u+a@~jS2nG2G8JFN9#M1ft`g3Wb}Pv@7A$+L%dAh0eH)0`u%1Uwc*|od
z<g!Z^6*k2`rsB>TCFu|cG{QYzI;~b15uhA2AlfPJ;OaZw?gD!O5Ki`6+dbwQx_*n8
z&0f8LsZ@((ld_>nV8c@e<xWMW(-Jq!`D`lCMg|dw{Cs*ZC%l1Ng|;2Enx`GRG6ZzQ
zXj^GAEavVVh1SFeGacBnf5-l10GS?eTrw$hP0~&KcpNY*1RV*1GXzFeim-&9ZfM)G
zlNX@^Q}-wD8jGf9&g_qv8GOD3mwpl2$dYC6Q?{F;%!3D!t8iZoha=8~dora40_H&5
zgut&yn`?<!?r(md4mk~}8DHxU!0#!TH%2Ki!dl&W5>I~CqtMi~p9#@#qG0>!yp?BC
zT2}kTfbx1fse*?cLJlI3RW^UPYkRHqYV?Ot6RXQ|QkV6dld5;X$P19hTrybEcBD41
zTaOD^_o^x<eluhJe#=5&Rd^mu27*fpr2GDAF)0my5R3U(Klp=uLR>*M+m{^tmMFs$
zOUTVN(XRBF$=s40oRCREM{HO0mqCyFly%IA4af_S(dC|<i`~T@^!JGvORA&NRhd%D
zMCa-HDiME=J{Ng*ylL~+cYM;N`-x?1b&~#);wb~78E@ddnl~l{h8f&$fvSm@Sk$rv
zZjglj7Ez_^JYF8SkudL8k=;0bNI8oWg*}Hjtd7>O@sR5JbahM03&~O+O8Lxw$u^wi
zAgMo}rYdgKny04;4#)pBNxUo|CHydV+i0M*pz*1tL@7g$Zct4m=dhhwa}S0YDa>!R
zK;Fi{(F!2<_nB#B2J{V*fI>5<S};5Pd@BPoe)hS1SKRm3*lU-Q#!7mNcL_v0Ty<9I
zen`;f;1;~7Ky^+#%<|39!p7!t+UjJiJ@yxLWSJXi)qOQKr(N>aap$wosy{1#p6URh
ze;eprEtUEDQ9U{E`@1S@-kBt_Ww)|jm?B!va3iiQ5BbiMITUvh1cd4w4iVbl&)YYM
z>vGGbcxSooV>qV(`~3(~`lRlTXTayHganS4FmQJPlDddOR!^earzhhP$87;;FpTk+
z<3le$_7-4H2;L}v@N0|O|HO784J!@Oj!=cm?b9oawG>_6Njaa9b8(zcLrPMmOvHR$
zz-S+DOZRWni<Jf>vt?kRkJe8YuN{U5yx%G8&QX|bX?Dv?Bc|=9CP-42dU$>VgZvqU
zZ4!~KH~kRZMnRKONul00Zib7J_&z`+G4_4%ZjWttOsNus-4@R3PE?C#Z3W?tZ7_~D
ztjNi)gC-SFU>(oV<_9O&TOSY?l@|PF0xve^V+2#mJu!%3pU)`OR8;0<GoTYaFXmGa
zfb3-um=hySHw5l>h+koZuI-=#&yIruUs8`v(tYKup)$&y>77U(F_eSADz6KMQ0LY=
zR<x&Hv2shi?@da{FbjwI0nxp3n9Px0)x~jdO?-d*rYc4m)a5mQr5#2Mn^g*g`UBs}
zSNO#EfOxqvMi``kqRoEidO=Ptu@Kz*2*>u)@0|7&eFApSj1Qt|<2)v~qdx{;%{l!A
zXog#PrI|lK5$M%tft-~tc<JmgkF5a6xH3^o!$a<WuFpTc6Ot-fp4MqQuvMKPqBv@;
z&C6Tq#=3p~z*{p2Q@DCW@AOMHrHb_u-v)(MVY?0!$omxpN|4Wsutl>Lzhr#YBMP~%
zdBecN2quwv-#<zS*bFz1S1pP`{i;!w%n-b4Bnu6#Pxa=b56&=#*iKN=_&($)kh-S2
z#n;+=%OCb|o#>#l_8O~+@9k@xE_Sn-Ybt{8o4cCK53GOEKL~wFXV95?p-CN@WCmTQ
z`+mFpl4>=%)pmiZ{QTZr^7eq7bg_f^=nO8#;>W{CnO`OA>+GMhgW#mR;>*|(;i~sr
zUHLiPTyaVW&VvIiV_eM{-XUD+WFDVtxLS|s+H1JNS#^Vc6=_H&K1K8tgcT&;x+@z!
z?!bCNnJ9-c<&}l$gAIypSEeV!`#|5hfBb!%4)coz7=NHwx7wQd!*{+V&)d<U-(K5P
z(sYYQAqbCS2+pO*^7SL9XvL~qYfXp9(e&c|_|b61RA~5Y89Fjc)1Cz;M{v7XkMPiw
zH@@o(6<9Yd4W|*7i4)cnk5zkLw)4!NKc#mcfu$}&>CcX?oY_pbs*)MC7YDN0-%Lxl
zoljjs&QaaLaF$s}uXCttIo}F?EiWoy-_pubgfWlH3<!S<F&uTsh0~3WTJLvdGJAnr
zanhJSulJxO{inE=)#X*O=a(o=H<nbEYb|MCggj7_TV|ED6SHc{6TQF>XDxZq$GqO@
zv1N+gJbEc{W`SSbpUBKWn?wDfdssI6|F6F9(E3mJ_Jif%%612q3=HK?b?9`%m8qq+
zdP;KCj|q;M$uY(D-~3W1H7#fP68Z?!L#V*lNxIL@KzSkZ^9cJ~)+?O67^(gC6yZE`
z2HB6OVg(6P@79NuD*{%9dsSJUFw~2V{hL2ltKV$(md1>L0_j%yfMxf+HFTv6G!L2|
z8k$w)MOZkF59jWkNx&YEtggCuhy~F^V7}_YF^6MxjQ1@NQS>-ZiCra=Jmk|!S3P;E
ziR8j9c)BmZzMCH{Xsv0?QAOVPm4(|8ka1DZP`1YZsnEN}q8IlL*z_yXyJt_9?)2U-
zdGnbQ)Ou{q7Z3$_4JHd2RkAY)eJ~B>gk1}>>8n^oT;b=`aj!JA7KNgiJb|i0wrgGK
z5dJ*oWGiQJqr#)sAA6G-WXjJYu?@Lx5r7p`aJQpxt^ta&a2L0Eq1(cJ`xou~nuqoJ
z6n|gkbM@=Tx|DprNo!^D<rF4V71f=J6k*VquO=KF==hhW^+W$0&_*=vKblne!hAkS
z>=h{dW37aJBYC_*e2q|@P#>+l@>|RYZfG8nO(Ox%c3<XW-CO?toxe$AVh&W#8-(}B
z3+FzkGYnaeT<Vg+`<Reo%q(Bh8x{X`Q#ov|t)%eP6)zUQkIbSn!}w@;XulbaK)rI|
z2-Gs}e+4QF*<4dPd`vRLNXXBfuO>)4)!#vX&rM3ZGJm-qfloevydf4%<MIyZ)w{>f
zsmf^)<38FHLhua;vyJ88+0I_Uu104|cWg=#MjF;+@spra?|P-2zf~uk7%|f4H78Bt
ztoLd7)9xGe?k^QC1yWmxOq8ra`!uaL{sr(7cna*b6CpGr${L*1drdpIyyv1}NhRQG
z-8W5!JKyJed6}7!_P5EHvvkW4s`XpfbxfX?Z3jX&8g2g(uno@6Ad}zPvj-#G24zsi
zd0u~DY~SGs8v=5JP18w%j9!_5^`qWE9>I%3k2#=G9;S0N;Is|yGy?L82OO2P`b#OO
z!+=U#mEJK99HzTVl($=T1WiQO{%Sp1gR4X2d`Y9WS&zH@cP@1qqz-J>70LemQy&G-
z%}!U%uN01lK3S{N=Tmu+H%1Li<tF42USml!Oi1^8;#6q13G>SvBRjT@3m?&pq8}$o
z!I@0{I}XpP=xVc)gU%UwUX1)l5nHgCuXkEgpgTtf7oOKi15B?UGC9mQd0bvP!`}u%
zc~Fm_cljy8e$FMv-Zt0r*Ba^M6F?r3sB`o5`(|X81*$<mVp0~efy1_0bN}>=)!u7j
zs;cEPsj2}X_)%rHe28Txi)lw66(=m~ai$k<10~P)OoILzWXQ+;F=kx-Xp<)?p-a|t
z&C4l7fjZ}?h=ZcMB>wC#-It)2H?f02=d0vi#h&T0R?RJ%mN?R7mU?WSmr#u3e62Fp
z(~y!k?}`o^8VC;o$r)5>Z~aV4>GJ)RmGp|5d6@DWT0W6Z9KV0a>&RIBSEHreaCKgZ
zph*Ino+P@_1d)Ymi(Q${tg}lba<x+0`{Qp8a9(>D5mI(I;r{0l=i1GN#^Gy6L(m}J
z$#Wtv)ivv83r{mPHG&(9dLc=qK_8s%pW++ipc+pgke~JIXUt=agh(J@JyQJBHOD!=
ztXO^&AWFQTuzHa|GM40oaB)Cvw<q*QRM(Ba?(G`rL}KYA96Evx9Y=k}aZ&sBuYgbX
zLNZ~7a^e{C!J6kqn4c&aai9rF1Nf*aFB0TC_}?KrJcFw5j$P5u0ISPIf{KEaC8N+m
zNak{CYh9qf4podBQ~1zHSs265x#ISxk|c>BoSY&<-*a8#^z6*hVH(dv7K}z?epRUy
z3IUn3ITna%B{@ZhND%<Z)q6gGTMR9!Fer9IZ*>1A_{%9eBfLQbGV+Mfi&N<wj4W^c
z@)l&s3x7CH-kZXFc%4VVP`>Pfy>g=yXwGcjh(pcau1nt^Z#BtBzr%mUo{K}tefU^!
zu(R&bx#gX&B)qbRTUhkEEo=ciFZQWGD+tg%^*9$2vC&_!f6VpqhYMD@ekEn==zIuS
zHcRS7P~aZ2yb%3VFJ0)~O_*OZT=VttT>3vQI+I8R?0Kt%nLYlZbxspR;tT><`7gp)
z1m$)2))1$nx~KArh2XZmm!Ir{0+9XJ+NjR9%nebXC95ICrk(G>=gBh>ATW$jrPAxL
z_2{RY;f-hHzrl)@_Y>OYhG7Of?zw)=%b7%rpf8zKV)^8MaDa}N<qW}vl;B98{m~<n
zB=Y6anripU3^Fr#>;E;X5;pldQ2Jrxng;lfOSP=xREV)T|MdeT6|)Reg>F!0hLmrc
z>5R8d=L$GM5?n%2-VCQEx5!z(N9;j>XZ2G$0MMo=%=j^57xHN`?I)M|)GhJ06ro&G
zma3d9zpgndGSHrIF+5|{E7#9<zw*aBYP>hn?U@<IiX_a}fY;`p7qx^o$&IGHzrWHa
zr2oKzHC*Y3dw#+5dNaER0R9&ps80+rBB^K^h*M+)zU{DrosIwU_M|g|3UOzABp&cw
zg)uel3^s=|1N~XPZ8W~l|MR%?>5364NH3xdztDy5sI$5-1(5msRK~o#i0ZLU707<c
zE;{lSv6>cyQ{HUl3r7*<<y$9q!4QFDE3eyS-G<;Ou<mSYGso*9c~5DDE=QFc;}kS7
ziu`eGnFlWNTFiQ0IE@R!S#4#l!6W?-PG4@f#&!*MAaqpWFDF9-cq}5fTgvg2^-hn6
zt<svOcxy9{`eo`~Lba_<@Y4Y+*LJKwVZK51PtgoVY-5z!17L!POWxF$qoINIE}x6_
z^jjC@=lVU@9ds>8@dlr4glGl!1JJ49aHV<v5CAxuzv+g+9q)dUwV|>7nT%@Yws2S)
zx)*oexO>9%Vb_V(WhN;;7*!DNIP%a!w@R2vmEt0gCPl4I`y^NP)7j`Vea1(ABh;Ld
zvKWLUr6Anh9WPmu)hExL6R%8|!S&YW8ltkW!R7PuP*e2in1U6o3Z<?x9Rzy&qQ6}_
zJ8t4iNl}gbQ<5DX;F!p=ydh|qs(Y#1=WCSOvNtr)jhE%yyz{h_<Uo|^Oe_+N{MsZM
z_ImX=K(t5XC=<;UPm*{5)zBSJpGl`dbt;K+TF;iHz0oWoC_uxTnp5s=ZiF082^fiz
zI)XryPx8&*AZpK#L3B|~pijbYyB|8)J~*7;>|bV#fSN{JGicc-xHC_MWR{kPM|}ml
zd^ze~eCsUcQ{qRW4Wv&Lu*P&!*pGkuDHnvmN%NJ{ecRuNCSlL`e&B>Rp8HL|NG7p?
zSFB{H>=-4rnV@@{G@p<!At`bL?g;ah0X@!g7dWLml5sMcahS?bP?nM79v)e*?!$T=
zEft&S_?9`r*LUf@OBGcIFL^^ma>ZOyr2A+OXaNAz*a@5cc`SrBg1<oZd3_hLf{Bp#
zc(?w<S>Vu)n_g)}VIw4_KYCna={idjTNs7Gr#p6@S&!Zg)9j5ULj&?E^ug9yXtODt
zaQbCBD0+E4Z+eU@D5;kDF5S0F^U_Wy=`<PAym|CQl`8OkP={7g&t<lZ!NzS}&iVpb
zQ;GHxIJXsfBkXKXU<+OGJs<Zs5WDJ}-fH4d^vAP0IVpEd{VNX*Qd-zyb9aPN`hL{5
zzF117U6n&%W59S11cxQiZ+cx|6pdl*>d<n*gJ@TM-Ic09ziK11hOUJeq}2%Jpb-02
z_~BUU%~TI>imYO1&TDo1zLr^6tzS$|_x@NQRam+uwD*6PifN72EPx;5fEmAVS85fk
zFN8SZiIDO1>-M8k$L!SCn@fg7ypy<qYF`L^q5KAm)#3uW=HPT+^`uj!H-E4WGy0p~
zbp`@JN_JRDga*)2LGV$<&3XIlN{Wh|Z2=uk_Oj*%3acQz4F1$v((rTK{-sD4E47D@
zTyTK0zYS06e~VcW&RYK;gbEwblWG;xIsa?3RI(b#D*Bgj!7Ym%!sw8DcNckTcQ7v|
zzj9zG`tL-CcYYuk{m<s7$t{pL;-)l*B!|$WRkG1QzJ8IdXZCmR8zNSV+#$AyjZC7v
zMRDe&bYEjm_=lc1i`#Fp<)Qx;u_2nKl&}KYRW?Oo{I5wTJMpVJyA%1N#Dt~>dX}_8
z-sDoP^2C!G-MvSTkvD>x1u+*rs+@`;c&e|*)=qfguh3v!b;6xj8$kaVA&xi&FVF$n
zeAe{qfE1O7$@_4t>k4qRnnlQv2oA$+<kBD(JK}ZTJiH0$d-8AprPa4Z31@UN7&dZY
zN*Emj{)?4Jy05}b$*3&jl_e)@U!hQj6^zzM>Q0gbt>Z?mcTJ)rAm0nIMs{u{W*)lO
z+5r2H1O8SQAoc|&-z|fOP4CM+NMti-x|PxA{^OZZcLBb6I_Ug#)}bO@`rBVp>qm^U
zvz3PZfYPB`!zAn?c4Uf$jBrzg6)=E3B{Ngz3pzM5W0TfaSBQSrF^{E9dlSjsqnZQ<
zOZ^pd;lt*GP|D*^|JjNCpPhFavK?06*=Ex$Kn>Ekr&{2Nc4oR3QzNNy2Ag*>%e}Fs
z+lm3Vdy^0F#sJ5SMPX~YIs~Az&M=gj3yw@QF5l(&Tvb24^-wr8N9UuF9eI;=c~|cO
z_S0cyW#LXT@WRTL?%^KxbK~j*czUvo6|T-4-wG0}94ZmX5HDaT6>!{krc=x+7cNDE
zZp8n`g8>|&ufE^v&kPKTPEz=#UcFRL%04UE;Q-cq*rnDL+Ux$$^;O@Zis(*)Hck>)
zf$XLIx`J*uMfFPtj{sf4Bh18g0-X&*PdSyBXp?-~wD?f!HHTmSW&vCc60i0WfARne
z30CQr?*2iG^jE(*V_~!z0)VQ|QUSGBBc&|<Z`Q~MW)cBbFH%XyXNOB;M{Tr~6lyyo
z5QVDGz{#MzCzH6sfj6&QfH!ew)BCi}>SoF%)cxf%r8D0~{XGLCqlZ<yDMgqfJ1DKc
zprr6-vXWx+3$r<ONNtaT0{mrQ?Hf&tXjb&!3YcG_R^{yD)*T-WL5Ji-X)V9A^9^(i
z^)K||o>Q_v-N|Z<n&rEC7f$4Oomt^WcYn^@)QfTXmZmCS&HYkrKwH_7!3EsL;6{<s
zo$XvibC7rt)3rEO+Rj5YOFng3C_`g_hHmB!qyzx`&u-(&%ksg|{mE=yI1^#5-fj}J
zGts5Ht^V3}$!TA@KzI&(uNK8%#Z|_Af|k*M!^yR)$NMj$RS@yN4k$&`8oEYH(JkYa
z`bJK&?*gzaSXPGs&V4#zF5gJ3huU@TcfZUC^zKHhO$JHpcE{;AAkgQVY~Js)cZ9$<
zIZwej)U|K=8N9KLvQm`3@>Ov&(0Jb&mpY_YjAKIg$UVA2m*wU-Z75pOxd;h>`!5~t
zLIh|0N`;fj7}_O&U;LjFK`w7#b8M%wX+godFym;8VJ4sjOGn1V6XM*dX)azOp#l_q
z>kw_>uNGFtyB;gYL)%p1E#agk?ey+<)h}AGYCR1WxY))}Nm~z2!2bRc&>5kvrna%6
z(lVPt4xR(G?p1F(-|6%5^UzH?0?^+^fG4_{^8kkjoVQ;}ulk^&Y?p^8UBRj1pbff6
zFaa?t4^_@Yjy7~y51o5;zW`~%69O5N*OGxQHt4Dp3BId!M4;&(>1y%!&Vbfv0#?m*
zY1flS&2>p#ni<6D{|=t>@V_S?=ZlRG0^pnAz(CMzofHck7-0sz-d_ND0k?q!r%x{C
z^_hL+X=<C%VRu1;jZN7Q-Jr98JLcVM=gruTbmAuLAQTcup=oZEqKrXqZfHdw;5!U+
zWp*!X%s0b0@mz2kfR*{!_U~QM7ap+(0`LP63uQ@SId`?C0|72Z+#k98_(ou5vS$yC
zAQY$y|2qKIVq8>_Ztg55u)WoSg`^=AOK+)n#`UAs@xOHI3yP-8RIgn9A&*2H!ukkl
ziq<ukIRX$~>OQIg&m1Wjr?5e3PPz8Pa{!HhZF-Iv-kSZ&RRr^Lx!*k8dgfcm!Yciw
zv&5O3dDlRRIU5upy<y9^<V>q%Uv~uvS;<I985-kqHPc7HXg!qXpDCL(OIS}O%8Brq
znCiV;(oV1}^fmt$f5}1%WNpk|6ZG|WXb9xXoJ|D~)=mz4+MZ%bZSDACElmnR*<9ek
z3sq=j(OU;Rc;O7*Y|KCpqpN0g1~Iv6s<K{EzBYGr3|gAlb<1y3HHRbffUdBNZ^Q-%
zPkhu0x^(Y0r4~>nKUK?~alx?@X<xzz$JKX!l|_QNhW?SCG6DaQji?v-a!m;H7qy&j
zGL4DL0X325f2A;}huM=-j@$2+1iD>|DWu(UK0o{0!gov^O`6n;u)EX(qEBOP0|E0w
zNI;frtgS};8^m`d$U3`2pi{u{&k+pn&=kNtNK*IqlH2``a|Tv<XUuT9s4k|Qc3W`o
zXnB_5I3KARntHA9S_idrI6{37Db6iauyKKTdd){SD~Re^PTwaJaaL2XDhqxYsmn#0
zCZDv)p;nmNjxDdA+)vp0bo)ulGqrGbb)HlwBLN5dPKq$Ixr#49d1Kqv(6C5vwXr%P
ztbH%WF|}%{H^5NXeVoy<W*?u-j(tX-Q27t<VrB)`)P8016e#ZK_Wlx$XQwae7`1qt
zjP7iqq}!<#3b6bjI%TDh9jLE5ktFo5OAAejP<`Hkc+#bzr7zg6$}1n`{7FhtkS?6z
zgf_=pD$92n_02e_C13kC6P3z!lICsUV;DBL;i-R&f<hF`0JGTn1zzS)xQP6uEPQK9
zLsf$wkB=m?8UOiSraqqY_Kec`@$`s0v`w(cNigTa_o8M9{Y!#EFt#IE><EhBDV6mf
zm(~kruqT~2DmU4;_u~rYY;VFwP+s>flVyQ5iGQKrPomO53G=83x;+tsLT&;#d?to_
zEMy~<nExnpDwecnh@gA>S8diNV8zKP4OWIq#@~<mi=`Hst=}bTA*SiA-@koXy40;T
zS55xPCSSc;Wvg$k7WTL|*k`YD<BMDVE1`!&y}XUVcs(W-3N&m8`FL3m!x~eSx|(2&
zli{Mz@HYDkQd}_9{2l!L1TEP7U#6wXwBiDtde$Sc99jgnpt)8E{;AY^6KKaN0aG{u
z8JLC7aHK-n44LgT7fhhOT&~m4F<h_ga;ku<KJP*LuiO#q8vWNR)WiB4$}u$~i+^2m
zFJ7@8Oj?~d4i2eP58rVkAw0BS{kXv1KAiTjjtm6Zjm6q}eD7KN2b@+mY19oLa3^V}
z^t!Nep*b!}D8lwMI$f4EV;0Bt`@AUqIq)`y`~_@Q>%_P-x?2blaPd2<K?Ra|7xkbv
z$M#zwXWe3Rscd*<#__B~t^|&ts33Dr&0`?SvWTW7gswt7a!{HvB7hd1vPTjK(6!(g
zrd5y{Q(WvH7PV!HKeHVcf7paowC=_`<^Ne<SGT~GbP>6+buM&2xBd8b^`6tQo@Meh
zH`wULH-|B)`l29)-<A_*u#kVWHZ`a<u<^mUw(TjRa}pyBw12a|AdH7nJU+nN@w=yP
zA_$PxnbhZT9N`ipcyT2H=${fL@VXFsK$rw=aFYzvqR~igcOEl?wpFj}^Nym;;#REw
zt;{g1&tD4u3lD_sxQqlt1e5J^vkK`t*@5k9XQDl|I5uN)yq}Pc58W}7aYfz==lBn=
zN+CLMo$6RH3#6T9ZH*>jV=I}S1=`KFTOQ)jWJYcM-P;Vi{kbV(|2<~%xFU>KxQ`8l
zZSF4VM<DzSFXsP{_ENDQSk1F)b%W*4Xl=rY$Mr49LM4p(F$#G4?Y$$>8)yw);Om>G
zugR=p$j<-r`gI5VNZA<$vZg?L-+sECr2)jEkVtlbKxD>S@a*&vqoP@+jhy^Ga4f%q
z0cg%fzjF!L{$NkB#z{jY@F)W^?E-;(*l0M8RkF^W+9k}FC@%<??Ny}eJpAQvHpvTi
z{uUE>0G7oFbd`l30)FOFq=3BQ`@!3%CFM+uqhOxH=#}WAiIccIy5d@~j+4GwhNJK?
zjgk8Kk?nNp_l_77Z#1y^{;nhEF*82@ilO;5-g)A+e0%bH3U-G0lCujMp#XS)?WBsh
zRqqnZdK?sx9ywNqMTB7sQ_Of=NhJV2Pe|wa1p_>>abgzJOm{-Wwakr$VsuhqekfEq
zX!EvfW@8WGq-XIYO4$FGbNOi{h}>idz>+|-1pyW0e{TZ;M-nFFA=qI{rN5s$TB1X7
zGL6AF78H*p>>x8Ha*mX9v52P1Vbv*2ATpcd0AxzgoW!}-rKXvT+xstsq5mNK15C})
zQE@Jg7uEmoAX#?(9x~!zNX5dECM9fe^2Z_D+oh_R3Ud-xmcIb@xptW}AoF1#Ce<%+
zuHR7a$@4}FOXS&W&?Mm}uCgCfNW3m~OCQmTuq`QNK_cF2VW013YcWeV_7;sZbI@vs
zsjz#Q&Jfd0Y`Ft&(7?4^UXKX>*ELSOV;gdqGqE_Kf8T%UGz&JRL)y@z!x8=krJ!d0
z$}sfzTXeebKz2OE#ZHiVtO%Tgt#_Dt=y$4SX%b@;Ls{*Ykv_=PzUEP3;cHg2x`RGj
z?`Gc<`1zz=0_3VYeKJv?;C#5_J^o3L;$vp886Cyk>r>z?zcva#UN=~i>VsN4Pgd%(
z`tY4b45R!oaMt0+HfpKvn~Z!h^+uLyL9?`<Jy{3swKZg?1Zet7jiX#_YEsLzpJ?=R
zFonM|h(7?T$gjp*=(hi$z6XCH90iGzzJz&^Tx^XUC?*jk0f2_}&I@RyWvEd6|3$u_
zg{S?83Sp{HLVIX*P3e8bAQ84X9a12GvwPn5TQjHx@etr46$4kg!g_^^G$&2!{a3cl
zUn(rI%#SG@$XqoA!~Y`JT16Akw1WUA_3AUSV*}nQQtR8PW>lPTstM2~#G?UJn^^r?
zs4yF9RUDO)v{uM6lj#_1Xf(N!lJWX&OY4~1e%&&Y&U4>}a2RzY-_sKQy+faf@j?OC
zb4q|D4wUbwjokao^IOaQqX!<29%B9zCD)knH`a~)rFa~*b_|;s{Q4-%Sc@6x50O0T
z6k}x&vxTleRFrT!sQ(R~+B}&}jH^cmj4PgW?b$3=^*^ZS#UH@=@!G98%4ed;KnV|r
z&v^r+G#6Xy?{7N!R|<x?JF7cU!Vf;Ce2f&vJ*F8EYX6~D^!Kul$KVZ<udpZa!`CQq
zF52s!-18*<zQns-upO@hIpci}ie-^5vMu+w=ek=3F31^3{|0ONTMp&}f=O%wI`OiD
zApNE^JnjuY&`#AfR4QaqEPI1uvsDMQ#3NXJ4QlhgQ(*aujheMia>(-1VM-UIRaj%4
z1P-O`YlCp5k8wR(8Ji{(dKeu3KaIUrSRBt2Hi`td2)1Yv5?mJu4#C|M++lHdADkdT
z0_@@vf(BjOHQ3@1T!Xv2ef<9CI~V8f?A7cu)m7D1-P1Ka)%7lqy|bU{NA}`%9AH1Z
z6TuaDB{P6hIa=VUd7aJFS$_G4(&FgJ^EdDm<YwX1*0<tKn&>6aC*As}y*Cq)?WXqp
z>c*?>c;#zlhdx{T6zO@i<?#Er{AoFQpv{w*!?EZdb#ATxf040#+e2-i#w$bxEeO*v
zn2IcAThShOwXl$s1gGLtuPx5NJ+WX^Sno888gWG?k0Z5faZ1lGG!A89fY-0z4u~&(
z4-L%(Yc&;=piwV4`u)zLuQjh4_g@E2mA;v1uTI*J@kC$Onp=K^g}9b8_$9jOeBsZI
ztn7EOwtgH<c3vz~<Cl@^b8yF#X*4F0tty+AkrIo1r2g*DwHZWtLGlIMn|C`=%po5o
zoZ7+_@ucGcdycZ>bj2qL`fuFy05?A+e8RO%XXik08LIkW3w5fmiqz?EOZ_Ib)?CXS
z{;%1%LGG(IXrt?UP4rn}1T`8*AN}4u$Llm3H_?i532R>E23MJ}zt4Y#*<6jj5GAxP
z+%yjQmxVnl|KxZC!W;J`e0z<Ap(=EZ1fGQ3uT~~BIuJV#U}x^Tnah~W0*cG~WVg-U
z;#NO8;~C~J!t;H#E4EQDDv*U{)<=HYMV0-;*wVcSmY16@N{D=gu#GA{zuW)x&pOq4
z`Jvdf3pP-DRL!xRG_Bd<O7ftUBJ4xS$y`?4)hMH4K2Z9ODsY`5Yb`ke)7hC2Y3*n1
z=}yDwI!+>=`EuUIgea-R1MspM$B-$2w(X4-uBBc5L|m2DpGsBUZ(8Vl4Glc^@m@Ns
z8edZqNERUoZ!Xc7p*}re(UaJodGn#k*D?_pL;Tn;eFK6EGj~u~pWJPRqEf0?R4zA5
z;|=I>X27kK$9*v0<5gllIM^g==ZWN{L4t^imDYHCTjH2#W9fIPXrOyhh$fO8$T`gm
z{i3Rm#%FFkb-uuXGvT!b?AZ4j7B~s~acM@6APb2LTVJROk8pM-$mPwCxlszHHO#hM
z*+_Ej{Leivk+J{lMwZstIgC)>GdwtZFc)0{MDjzWSNWY|K{i$p{X#HU^;z`<tqhS#
zZmvE0&&SC4KLPLj6Y>{HyVTC>q=W<GhJ2x+@1btRpxfrKcfx_co=nquGV=OkPigK*
zoIoPeLgTWukjB<0)UVltCq{nyV+s6_i5_O=@^ak<_@U0h%SA5bfwy&Gh15S>0>P?E
zhgPO-P08rd{~7%|i;gU;)iBOPY?Rfv6zO?iq>KIz;Ipq}e`0-u9|RsDQ?c7pn~^<~
z=Ra-e&HYc~g#Q`4DetD2Jk{;tLp*rxZL!6i5Y11#@4FV@RJ2g|JzDq=p;tubUMXrP
zb~`i^WPjk6@1O@cT2dA9UTRWwwgt$!pJ$VC7-hR`RC;<{&0>Q8XhG?E!0OKSl!y`r
zKt9WV@?6jIlw4Iqmb2Cp<@*UV-JA#!Qcm<aLHU!_@QL8e@tNaj5LoMvS427y_`g9A
zmB>y1_(u!fN*MbkLzn26@-ec>T*m56-IQuzMu4QSFp@ncmV2RzoXI#1zF~6uQZGVB
z5<k8)B=H57{Ch$|5CokRnMtl-AMyc6A{IuGlvMolfwKw*F*k>3uhZ1PJqk(&t1+f3
zmE3MO4?T`LJ+%X~KW_p9p&3ms7Uo-Z%f2p8K09%D>%siO&>VEu(4($;ygQi`DC{Ft
z63N!Zd7v-m;72&v1M{~#N-71U$vlY0APOQ(-XWOfj=gFMdQK7V;-Td(-T>0GFR)x*
z4=W9`ftIgTrvz_8HbUiUrFa_8z>Tc1D2S!yR;xxp<~{Na+_;BOCS9ukc*CCAOghvm
zu?XtjC`3komd8oOK2~m0fs@XlcUD4fAJ*Q__gHy0RC1$(wm@q1?vrv-8@$$#4Gza|
zR&FI3;FtJo-zBV;zt6!+2bi5Iz}{Suz5yoaP0Rc*69ClF9#!*sjk6wU9p;1?=o;~3
zF0tLbWu26In>n8aG>b!Zn{?}7J^l0tNOQZNZZPR7M56rp<Ri@$mFW%1{WSLI=&)ZY
zlF$um3T!_1fCGM{Vv=LhOV*-cgM8?}Ff6zk$+VbE6t2IbxSDDKJlKD;N?)TEAeg#$
z9ZD_#+WsAx*yKzljyP)Q&*A37Xv=x$DVQzN5iKD><y&B}l3Sqnle-iF#r>N?gqjq8
z6u56gX+VT*UWlIjI&DUCUo6p0gSqMpZDx8_{$emVhgC8qE^ZW11kEqMeEoX7LHW_n
zv!Uc6akVG$Im=%Lm`)%_EZyCpISH>ZHI?fIzkBC*6mC8E#B<d@?fWG>BZ53RcnoXP
z(76$@<Es&dGZL=PH+Ei8-@`De-mMzO)KlmG@o7`|#-3cL{*XuZFQ2*e1~tQ*ch#Zx
zMjn<gn*2ajt`&E=k>zTb7nnPqr*y5p1JVR#wgeFT@97tYKmk3KWUcdya<;0@92%uc
zw=bMLqWyZBF~@(fE|7Lq(48j*KXcVQ2;X25twWd#OFcf->%E+!rLuD5kB3WtxFTUV
zq!AONYthNJojbfNC3T|g>(orpGIyt`mo+(m2N5u`V^HJFp~_hkx`}ZZP`W$pa162@
z@jER#wPPPd0BRITBLiI3(SGx@ZK8Smcm3~~-t02`XD4#UhJC-C<pnzGBhN|KSSLNu
z&YE+EheV#jOWH}7Nrt7^UBi&JFEWMd47qOqHvPoJI$4-MO#XZU5ffWf3Ie~&QA&FB
zJKez`xbUCXRH6Ma>2=j_cs}qr!ODX(5E8;abhnxeTo!9_U}L#4LN5On2v6R*yh1$R
zi3QK^)|rlLv@z1ZdA-}M<xX^eoXq^+b#-&e=2y{_xR*N$%EDfc2bO97YyOiiOH25G
z#a$#MB;-n5rtnVYSzfp7e?kvFJh%Jzf!;>cb=-E`Tylx?TU@l-o{MS-?`~-*5d1BF
z1|r&6z+mw9glL<DHcw}<O`DPUwNOiarwe;VQYuJ^Ut=cdGU|w$4JTSFt_k4%CRTd6
zKCgL&ZU=oNg0K-M>vVTmv-nE?GoODFPuU>nMBb>x3Q~_Oh@Ng_FGJZ!A>L(|2C(_x
zK|p!PzwF>iJzh+kMGn#j&-rcuA;}H*j$vkYiDIVj?wYBt&<|HK2u|yAYGbXu3c58D
zWGQfU%hSe?@cEpAyxQhhuQ?xPdYkD7Qj(T~X@i07M$=WgY=xgHwtR8cEZC+9G8<av
zzqL>u&Z_~5zf)f@Jzr!(dS{FkP14$48+ccb>8#6x$yT)@x}iEasDgh1;v^lM!4Q9#
zwa~77g}cw6<-C+O`47&mr`UDoM1A|qM9XVrs#$QK7;T*Cm+q{E0lFa@M3nL@WFb)s
zr8ruoSmgo4g)Y7t#cnxw9oCBlpGF+pb!G}x&5JCELViEEPz0FXTRP=m9F4#KF1)R}
zY4FDHq=BBaMVGFF3G875m?AxdETu(8k*VVh`r-(pCN}b5>zq<d62^$*EGp;|XS==8
z4X81FniLb@I<tH$Dhcfrm2906p^&i6^9YJ<Szfd6Dw0nzoj-Kngj^oU;VkoHeFG=w
zw_ZB@wBl&yyiRF7gz&*#b2NtTJ!B+sn-h-OIRdE$jHD!!_lbM}b>_R(<7Gc$q8qEf
z#9_0WBeY(j$YKA8z*F|+TJL(7I5CGA$d%#+nJA<E`jS{7ML6r)lK#+JqRp9cBv!$7
z2s$q<%cwz+etT#11H+~Fbs({em-aYZbVjg{j~<>F3$cDW#qP>R$@+(OEdC&T(9Lc}
zR2A4aTS686i5DICtR2th*HKX4H}VO{nM2707Y2Og!TH9q{%^D=|552waIu@%=y{|t
zzYs*-*`O7Miav1-RBL#@c_`v=alqYC9nWJv&8c@+HRp@-I)`Abg=!992QMJNhIWzv
zx-}W*aSGvAL`&Pe1Q191+-?|ssH&r$XM%O$!O?2q*C?Q-$q@9Jj!5Fo*W>PN5*%it
z)g^{tL)*Mn*<+-GfAqm5pP3WS8PuCb)S<{{Ste(+2Vo|e;5<MJXYl@?UnOFNx^xUD
z$r`!jizhx`I|tIh<?2mNr5;WyL7aj)5t?_{)Z$RmjG|jentE91vDoF3cb<Yxp2sYH
zMc!wEAFKdmM1=S+>%cxy=+V~emubOrpDW<?#^Dz_NYn`2irN4nMiwZyxw($hC#JSe
zIm{)tH)*I;1LjT3BcC-!qNXM%CkI1yYMN#z8)}5#WU}CP&i&Oir;ulvobrpgUEYIT
z=n}7Ifa&W-43jNFZ%dlI2Cx-I#kI4*^BlYvCy9Wzma9{J8i{(FG8>!eLs#Ypj`sJ+
zHM4V%m_86uLMMfwcar$k92J@&Ok2xCyDK`QwgFmQM7)lc4fc#ID-2y?dU*f!2;#E(
z!id~Z?jbCT3};P-;0F~1>Q!FrK?f{W4(8tF;G?#03b7%Wu!9;x5<9qVC`H1)f+BGg
z07X-62DPwCg}c6W6VOy(4GpBkH(tP((U(fddu7HYU*iwhL(3KwQeEF(@&@rX2U{X0
z5rcjzPaz6I0r|vjy~Sb^;=GR3o6QH&!@1T%|2Sj<%C3Cf^nBuxNuuND5)))b?no|J
zrnb#ukyx`gLHUOPmi5yG7~zq(`g#5i_;Dp9b!jXYg%iB+o9faF+aX6PB_|8C85!Ll
zoJ)0Ds_-5Bk!_Doa_KC_Y@1|;i|u#7+S|O?88*`c8PEZV;%RjQirdhzynX<VutM^r
z<a6Pwj)s5Je?0SWn*u_a&1d#-%q~kn7CFxNkR_8dqnLR{R|ZsLgQ;{dDTYWYa0xCE
z=;E9~nosKaN9vg9#A2CRf2iKyowLyx!O}ig30NEJL=$2aOy%NYsM>rqcB`ugW+&ia
zeP!WA_acSQ89pt823cMs%eKz5WTgXCYeIm}xVcX*k$d}$M(%0c7oBUK9&o<Q71dg!
z+Ke2|<g97PkX(s=-hKcavU3ASgrzjbb~E|}BZu(f*I$hjuezty_F_UnB_EyUWWlQ+
znLhl&Wol<F42?g-fl-=r&T1HkW~ANNw%Jp&)OM~3*La1YrXa|4e=5wHp64+_=%dCl
zq+4EZ2PN*!G&$k&4#-pyJEDdpu8yAm->F>V7(ebB1Ky{`e_soz@ug9};-77%7PD*1
z{<c#P7xTP+2Eh+MifR+YKd%a}6S1(|*xaxo`vl0DTJawL+<~5I%Zz`k!Fjc$K$dJK
zOrHO9L6-6RZh4n&FN*|Wi@smnsWZzXPYQ<bCnxe;vV7;EO9ZV}yS@X=>KkheP3B*=
zN#96o_fd+5#H0aY1dB$^YvNe3W1&+`**|Kgqkrf~YHz#syh_XcCLOz#prJ-=%@u<=
zS}@2yX*sA9+-3x!!mFRPTzDhZ96?T(bNF9m;W}^X5BlA<uQ#A#m%E>LAMCb6^F%%X
z2UzZQ!x;3mnM_kg{xRwN+W$!ze)w&Te!{<j>8kpGBM5*eD0^R-E+}qSFCOGGH#r=q
zr_&5Ov2Tg%52xdW;5#_dveKDtB}kL`kC}&|!aQ|t5KqOc3{mBB_5N5YP`t6d*GXWM
zNRd5T|FAJ8n*@hQ4Th)%sNSP2b`E($LS1`8ly$<^##!Rn{KR+I?W=#l#hbbhf}h3g
z*~=RFKLfeUd;KBChXKyR`egArS9I*#@8-<#kO#s#XjY608<#!pRswQJ5-_iqb1|i~
z^>4p@{5<<t{&iXSwj{x#$sjf-uD<+sv`c^c-s@<XR6@fQ`S;xN?<SCGLor;B@{bBL
z&4Cu^@o|qU%habQZde{E3q*0(utOgEPphP(U2VVxua8gN<Vj*+6bH}2qPu@viCI0<
zR$U_B@cngvo6KX`#&~)<p^hf)knG?`fi|VeFPknMy&8A?i@YP^vuhm!rys#?$w&Wk
zf<JaGlEhPXC^AGnXI@g}T6aRhjl!C_C0Vw>rZ&UQuw^O~rDiMP+w`~mauX^7^f>!y
z&OxC5^ZUmgl4Q#i<37Tyd1+62awpmQdes*a77DAU;JGVS7k3mD^xiLvA19>CFY@Mx
zK2!sPUb!D)z)wt5q~^G1XUZa)W#pUkw+p0FRI$hdNK2gSfmeBnpK;isV+S1{?qWpG
zn@jEqe{w(T8``?g-LH?V?QN{kKcrb?`Vw%S(XJ7_e|OKyM){mw(Bjj^@Ft%+Sw}eq
zb@`mnFs<)a!iAfe0xzlKg;`*DMwYDWsUHPI)+1HcOwo0&2o?x^QEXG-VhjckP_b7L
znFhV$3MKl22fc9-KI3mQS-AK`s5_zjK6=$*ra8j}G|{^&PaS&WcU$op>0QiU{3?L&
zp8)}cL@K^vtBPb~=yZ*q^K<u43uMUgzX+aNe5Cc1?eR+Oz>r*9))GNwjDa#t8q~0u
zZUU@OxJ~wp$t_<H1FGm-a#=Zp#!xg4+stsVJFW&&mcXHAF=NVJ&_<BTG;}!9X`a!A
z>^9T~*{nrJ5%d-<5cdpT-Mx^Z1Ot9|fD~@))0OB{Y9lC=wU-nbnAvg*I4*unCM|wF
zw0ijKR<RJIRlB)j!>2x@W3BLZ)hsJ>IT0Q`KY&QE7&s4}TUQPkEuH^Sml*GvcS%?S
z^9wW%2%+bDBk^9ugp_)ei!E({9&t!5#XEqG6n)q{_~IKn-isi@&|ij<o&!KxgM4Q*
zE^h<<V@VXI%y$TCCWQe1D(-jqKDkuvC&u65l;To1MrVSFm46DGP%JyPD@}Vp4fVZE
zUz84MtUZ6+M96Fi-+a?9uPq^U!pi*vmcl!(4b;+OQGvQ92tf72DslOcenC5{&5bgQ
z)ik5eRfvsNxuIhPBJurYDJ8&3?T%ca>6(JJ2=pD~-7#A?I1@g~Ijio=9GuhNp*3X~
z1v%l=$e4}p3w-%uOROCWQpp?>1&M=+_E^xW$AjIk$%qjP1$>4Yrn7%V&3=l3n8p|_
z&ApeAaa`k5-TWwz`s)UDA@HGj=2dmOe#CPQ($p#3x9Eca+E22S2|$gxN)bWVwCPkC
zeGYn~1vaaYfTismY#|sSE_c1x*Dnp4-fHDhx=u^bubW-&aBf7ae-K!)=!fp~gl}wC
z;=ayehhD#hFtZ7CpMtxFazA{B$NiSRy_VTG!nD#cxZI}i7oj|ZcQCU>nS0H0oBmu8
zDS7)CyJFCl^cf^|2Y8>ti;`jbu4MZha7*jt4*ETup#bq<8TU?^TU-BufA4Z9H#<%f
zSzA=|aGBo9mw%sPWo&nhEGf~I=}l-m)UQ|eKX04qho$aw@c&lZR+_FaUNg~FvvYR7
zX$Jq`D&34CJG6}hzAw5^u0wd}D*U*A4LU+!xw5WGWGQQ00;K&v*H3+4>*}8M{7P}Q
zErtKJT!yCKOM+IrKb;uC-$Uwq`<yd7s0M~Po|9<bG9odkUy`;ha!ywDYN;h5`9!}N
ziAFWgmwB#d&;U1<ky&I}_7{#`HtpVnM3vvUkGw9-$na4hZ1LOA{JJI$*Cb=Zn@rX8
zc*^`piugQ71(@>xHYn7U<t=&Mr6LemMJw)jZ;gpHF!#GRS5{wR5{u39Id?qd1?1q1
zKEH0HSCa8J4*Ba>q5r*P>VKt_o!$6#7iG`PMqn@berqW>@9R|)Tf!O|&zJDzTX6>t
zxRCPg_V!R`#mrFNTEDVS`BiYaoc??0YNnY0A^&FR2_U-PN!%o{pJo5$(`MM(`lWSE
zjly(JG^g0qqS5{3>t=STRCTG8Sw=!u#${Y8>E%x;w_X{j+*WiTc471M`K|3jLRd~y
zO3c-|Kz=FM!Os!``v#$uoFy_o376`2BZa(sFkHIVVeha(zD*4E)w`UFK5M9|4hNUa
zi_hLt0GzLOHO9~UL-xE*1td<?M>sXND4f0{Aker~)-Bq9hC2#_bY`cxuswp5(FQyv
z`WJNTt6|+MyXY0>LhpmWzo2+%lPMOu&bO=K*?5hx8>p$kI{VR<EX<btpz|2A@EQ`_
zo>r}q1>QXQsVwq)<|+r}7xSf3)j#=J9UNko7yulwj;6)Y$k2sjzf#2L^+R7soq^`<
zF-L9BWVxyvYmQ&X@4s7}#g!h<Gxt+92;zE5+dr<!E8Li%^Ap#9`&g^?+OeztK&^%|
z8EiIA_87M#D!pMbz7sKKb~<g2S`sovXRm_2#S4@F$Uus~O}dF(oMBlhj@A}rcEi}|
z0jM{yM9e2QkGxU*2Wu1!4`tI&eOP$7bE2mfNy8GI^8M$ljU3f_;`m_l@iiS3koX$J
z&inR?h9Z&m$gIUg#n9VTlWLP)3|^ESbanLsXeL|Tz~qSfr+?x_AEw1;wIlUCZ%PgS
z(3^Exj!ne!8uH78G)8z1BoeKa>QbMI0onpYP9IcLNdMX=(8vbQip{ji{p7gejQAq*
zuZwCtJC5(25R=r3y`NH~)T}}UqwwHdFK_a3g`dt~NVpvmX7ULOpx+ZRt2WWR-@}~@
zI|<^YRlHVZ$R24PRz-)(guSx}`49)wr|!ADYLieGNq?b4%v_0<^Xiw%r<2`)Dd2fH
z4P+-`F`!-%X;rJ}s8D+<qQ3CUv40l>UU=}#`?+SC(q^;Zo}!vhdO0-k6mpVZ!R7Ua
z>8AC^>~+e~EBtYK_4|h`Sqcbr%Lb89N1vFa=*gQ~+nk^8o9~<YzB@;j+0<}LY`}Bl
z!IO!ZRCfQ9<C#n1Vq;}9dNJN+G!XZa@u|dw9=DBMj`Je;W(R%E{2PRXLQt!hUiW_T
zV=3pwS)!<w`hX>VeI1u%{eozQIkhY^jo6|Xe_|lTMA`0jbF_IExZwGt8K+C7EnPP_
z0Iyqhv(T54IGO_T<}HtWqk3L+>D_0jdfFdwMQs%ee>Sd$m(y*cXvr=H4j^@RKAmS?
z$xH(N#Mz@D4DC>QG%1|OfQ6n<D?CK?8kUrj(uDcX$f=897NNW#8Zf&$A~7<k{gHhv
zj;-X1&)gci@w;i8`f^zCQzT=|8Ug3vK3u*jPZImr0TfCuA(sZtOr)~3Q`dPXmIuij
zmk~>?4x9RI^de0o7(I!$2M`k%kJtK4Nb`bFkXNlR&hlSip{K+#5=}&EHgN4z)s}J~
z+yl|&clWrvqivkZ@N+S1z4JLbuxK55O9z;*6(V|R)4;vk`vF_V3+=*<3e+$b>vSyh
zd9~TT;mbZMScANlE`yf3Bgl30SC=;nx!1K=MD_KjO??Cf<c7u{ki}kO4JM<Zi)Bz1
z^{GU7qVPvA<X6>EdYC>$KQT~-InBPwaJKb~P(c{nt?iGMMR06`J$<9O&7y<HG8ga%
z*&3PMrEDVFR76}3N7iG%6VGRm-_@7ocxGPRe~eKDMU@q(YP0tT^gkJ$ZJRD`R#My9
z8&T`$4|BM8Qmj=07kRwB_1nseko4WHr;Tm2IxN;S|I>{(TNgUCXAHep_+U|eq&^SM
zLvZ~W9o<@-?5Mcpf@XQM@4uh7GGxd1K4v?uigd$gd{R6MN2M<`skru1Oc#KiH>9wy
zlRBI_L_?5O4VADV>cT9JL&3+*;-DHoJ&n4lR4SVK{qNMkWN;3c_{;6`EW!Tix6ZPk
z@s8}TRm3h$H>H#<hPetw-VIDt#;x$j2Px4yM$^`p9E2kUNB@!vBk<c1{Xi@FFdmby
zF~Z8Me=eZ?l9AL}`}`~lr6dvbE!$rW>F;x8ocf4lUsIv#2<3<ch4Bm0bTe&xajCJ`
zeP2IwphEl{AlE50LjQGJ&wTsS$9e4ZYaja!>f&?2p0;5@S6}iwW+2&<xqd&?Rp1^o
z!w)r;dcG)m6C6+pzUb;=kR|Iy3%ISfy$$ss@N*mNSrX15V^nBuV2#S~+4QDCiYHC6
z+y>KdEi=PbA5V@g<-m>~&PgpCh6B@mjYM?BgZig|AL8>j*sS2NL7UMg7y^mZ-b}Fn
zRq*8)ge<k~+-xi=1HJ;5{<3>P%0jBOb>Q{lZ!yIX1SM6Zia+)#a1Y57Lk}w=u3iSo
zfq2`Pg;k`i{i`mwGR|CVQ)1n3eg=d2uv{7g*0pzjcG8&LWF`@;J{!pg!;2P=w`-Ls
z9|@BIUsq5!6=MSi1UF?m+hZXNM|QOYHA82#wGQz!Yc6*Xc>h;mh<fnrv2rvy2dI~8
ziK-dah3Yd3`L>C(2G+x97mzS?1hKw;%UvKzyT4rYXQTJy>~%2_X*D$rI+N%konMO=
zE2Meeh3a(cVlM5#iS+B&yM%4{PE%VbFx(7yV#7nl1N*1-HIl~Z%&2laL`V1cFBnLx
z+3NT0v&r&!VtvN%$81&JpQTiHrRK9|)L|6MuL?;eJ@gTe`5uD(D3Oj{`gK6+Wddld
z!TZLPu?Xqf_&0x^<<|{zjG;eu+8OPA2MOGv%zY=*x~Cg^FaB{1GRBsvHj~kG<qX(r
zVWH7d>-U&k)qK7g7vDto=FyY<UKcHTx5VmG_$~Lw0Sl|oH^xa>9iJBirP*!CVAuKN
zCWhn(8PI5F`sul%0m6|Isr;qi*s}_sO{geb5zG{KRtv)D2fd1pO1M$FVe`lEh7SAV
ziC<N@S2oVGFJ)b|&m;7)gD((ETfiDT&gitJ#7pJRt|yDx67S2;{Ksp7O1N{<upLWb
z>eRq<E+7_xA1FTQlrlUYyqjuaB_t(rh6Y>ZN6P$9YQM|5B#L9$3!1Po2x)aJYnPp)
zUOtx3Yy;kT_tr}}SM8|?2dY2{?{rgKEL!TS&Ad!P#}qvAO-b0!!E^opte+SMoc7!j
zXNzm#oePW*9pbxln(3nkfPx>%%ZK!UA#SU*0e8%|`)~|0$B(r1?v}Kg;17>*omWh)
zOu*!VHLO+n!X1QuVDl_FWO=V*ez7)03L?(z3L0iZ`SCRP<-UbkBH5SQ#vtPoVd34^
zT#10%2Zz;U`?Z=Yi~$bVKX0DRJ0Zz2M17mHD>gloCcd<^6I6Z2J_O-U9l`s?+U17S
z%_pU>8_ts^JS&Vrg5_BLsYOt_@c(vKkTtjDce@(9V>EK#x=$6g+8KB=d=$bl$n~f5
z9-joFj|TnL-}ce(TR+8L|6E0;NFsA6nGW`?7H|C2Hh@crCeDI`OiPWH@Tj;L&9xw*
zARO_5K3ks{tRO5kw=4z~#-HnEe#k+(bec}A6Zrx*KeYh3Ze6vECc&Im#>=c-=;(2$
zEGT!aRp&z(&qSVYNn@k?W6srUB2I_a*&a{scPiQoA)@=>7ngS#c@9v-vEerSb7noB
z=wctR?qW`=NBml$HP`m;<F%;Vt)%`QC}BT7)uT{Brn9@k9d?^vooi%=U?l%{Iuj7f
z?AI3Hx2%2pfN_~uS$@bSQp|bNchHo@M~d|!_cj3}(uQK$=k%00*Q~L?y?Ky}a3qs!
z6!W?>M<n-8>k(rsw5}~dN=!_WCpja&aw!5mAAEjoF7oJihh;I5FcPV%w?jaS`z4f4
zUbgMh(AKv0)+1)Wr$9ef?H??4UMS!jvbxw23NG8Yn@snx^5M$%G{;LT_o7!G-@Wcg
zFUmg((CRTPddJ;<Hy`-F0)+`oB)nf~i7UXRt*KhiZICPtF}jF`se=&hy8Hm^kq5NZ
z!D!h7>i;a|H~}{<H%LL)`R;m9wDVN@P-~hBGLJle`~?yt>+Sw5!vt=+h;xy2-j(Y0
z><KqmC`v-P*pc#KXv|mR9B%T2eQs-EK9-yDlug^uc6<;V*5D@H-i?Qm@hiOjyTS1z
zkqQE&+|=CI|76T@VgPr`ZePC=&fk3E9_4osU4AxMRDX;$AI)oo1c>uy`@h-wb=z;_
zi-k5%IV9PvT=gsharNM74G2zwT~opkcUgHy<TF<(dCP`I)Hbt`eAwV;f2?t0jQ?1U
zJ@m0B?$QZk?b46(y1VDF?ir??)*K%l)O%)ayUtXgt_8@j-^BQkFtQIiN&FG_V0i98
zd}_FE`L&AMt)P`4HEbgr!KJtK#O$~H|9WCUimb_7ZXY+JU)tUf*vz)-h=eK-w|Zk!
z!*URg)160duU+x4FY;7VcAhPCv{{ZONB=AY?3StI-F%;*&s4?HZ;-p0cT#oznh0PQ
zEf}82LA-=riO4VWR2&J&F9Sx@b@4sKUvzOct^OZZAkQE)Ce|OT7}2hxn~3!{3k$ej
zwh{i??|tfBU-SD!D0Ycjb&i2E)9VCkx@h*Ba{o@ne^wOWroF;(BfkAS?m7PBx=aC!
z@VZkYNdjkK>%q{d5{|xM13ydjUtBuug4IT!7wXrtDk=@OC-1$rU=m=H(>z3)w8+^G
z%ZC5ka#2>A_~?}TlD5|C<5Ny`hYkoYlT6KXQ9K7)YL(wusP5W-W@P!L-SS+=M3vZ~
zDEElYJ$d<MwUzKKri&y7iQt5=SlkrfT345=c0G;oZhVZ^z(znCu>T!B=>6y|i<em2
z9SqO)$6LAlPO<ZrES<~NtZ&^SV<8W>8=}2U@uD?QbzMmM?Pi?f1{>qln7bnF|7g0`
z<qccN_~Q7a3XN3yMiqH`F(nVak1ySN)7x-suI4DX&dP(wR(=`Npwb$8lN+rfHz@cp
zz6^9|fl4;ATmvzKi{Y#^|FLqVNg$w=4}}HR<4T(+EVT2)AmregfPDOG@|%g_7=s~*
z$QwK*vQLfi{jp0zWUfp$t$mojR~e>y<9+XU_m1j|8=@(s$b`uxE`tLcUiaI=UL7mG
zf5oEk{)WaG6hY|v9u_at15WA($dm7v57xo2`?+X=tdCqV-hO8W&lF~i32mNNDF0o(
zPXLy;#|#JlBa)yMxf<P&cIktB_~E4>EW__5NBhi)mgDC6E*69;f%@U9DOY$tU0F!#
zb-y6()BAga%RI4Xe|>v+8y_U`OeOoWAzA{PKOQ0TL9}BPOvS%a0JUfG)-a|85Mcj6
zC#2JdgxT+MA)(oKMP*Rj*6@0+6p+<R_wlwVQ~fCufy^x62O}p)HZDS;H8P;tnr9hO
zh~aPupGhi`zf4_$FDT`HJE>Ae+}AdlqMT_~)(>o4+1zVkun$(fF>NtMi-_r*szlaz
z(>*KsYOF9BX%*?|-!rbPzAFoGUo+ARSA#R-2aQfs&S!GFx*)=nY%spu`xcV153StJ
zf0AE6k;U~87sTu~RQ{>gM&(s>PKj4AJ>3F4sPsRzr{A;GyK<Fa<Lz{)IhQxOo;Ubu
z`jmUkwVYM{R3srBf|T1bc{{~+{_3jWcg1e~#`=36)(Z<Yf3>5FNP`2~(=DKRp*lUu
znxr;R<-ph0g`V^>K^tT)quDu=(#Xnak*{`kN!o|;#CxhuTj)%KD?-H~)_bg4=0g9A
z^<uR%2iGzhNOVf3#I89zARAEvSF?y>U$@oGAofk>d4t8o5$80V(ojc1?iy>fAGMpM
zbHa+7!}7_NrcjU}(0KZ31}F+8=%n_l*fCg$m|HkyIR03H_=^%=z~qIS{xOrfHKNo8
zPe#9HijmwUu_4N`DXOk4L+riCC7%1qy?G!5%u17h?PVq{Fl2U1gs-fli_;s*Y9k=V
z!5TcWZqWksqi(bf;JeY4n_d_8#Y0)+-(X4l__CJ9A&_(ua1lwA5cd_9^F9-cAT;>d
zV2cns-M34aS=>|q_2=VU6m8L2{E<3nj~?G9D-G*Pwr9(E5j6(Bzp(j>T^W$GGEBib
zI!NTtXBaM!-%Xt`vyx6y+>NM?E{~KA`I;m!uD&<L`lbg01?@=*XCiwT{}?)5H=qDp
z8J=?T__JyP0h%f?aNnDKGOdq7y(pyu#zGfi|Gcyi)$!^QYSwoPt*r8Ol?2k43MNE~
z36+Fa0#5dWFe>G6wm>Uv=3C}^+!2E1?x`B|*J+J;y#)-f4$okW|650o2`iuxTj8u-
zzr#r|mEZJ2=7{h|Na?=o_i62?zRGa_RFzxT5Dtj-qo?#g*~0W)X3O;+M*5dg5TDyB
z>3@{pnFN-ot8g`cCM-U$pqgztkB}9MRpoa~9|CUNuLWCWfW<#a<yUv$)yEXtE<cK{
z?78U5Do59pDL|+9{mNmf)BJmrnAc6t#h0>(g$yVi=ijHj19(WGC;J4i|C=s-`uY;&
zM_?XIqf32MFVak-7^dILySird!2woeGv0II26FGu4)|{ceG_Qf*U|MxFKjE^!Mh1p
z@a@@#FZm=BvLrW#Ef`rDS)VqOx)HaW%HS;<Y)6w<kg|B5V%Ei&sf_hD;WZaVLVwny
z6aIwWClrNGKePC5cv*vA9=YVj#TxQQ0Fv!L<c^&WDW`fNXVV4eZnO6ZX`hHFEygY8
zBUvS^*=VGk7<BerNZRpuG4ww53?E(NIWLP2*pxxV_Lwh(q`&y?x1M%3b3|6zV#gp>
zCe%%Nh@mhT$$X1G6w^*i<DWi#qiraWuxfp~PpUGv+K@FlnXKdNYfGAjas+Wb(3Bb)
zsxbyJ`Y?0xMNqH<;^}{#zXJbwub|Ou$|C;_-f_a(rT$WnHl5l)sQGgc;ut%;ix#gK
zp)wq^RNfB>?q45S79?|J#<lxog2Z>@&62SAstZHgwPgjI16{m%j@qZeqvr+Hu%{A;
zDPbkFBdzp#W9L3CmHe%}_y-$+G`L2=CB>aCs%9|XZ<>4(T;khPgZte~4{;c~B}v4T
zQ&loA&BEyl%FL=nWLvwe*5lfwtbkAMiHnuGpNCdo1r&=FVOOu?y-p`Zra+s8;AAD|
zh)cf*W(BU|@_+KiWSx4K*HTS+Yvn|QYCt7F65IUswk(xk%H6jsRRH3$(O9ZUYTifv
zs76;~lx6c4`PE59X!hkBNK6$-R5||@k`4UGUT(NM*c@iovd=rbLZ07`X*#UW^5422
zzFM+6gWGy7b`lm^?+2(L91YwVLQnWedC`XP*cy5_-4dk@oP;oIqVd|jK<RY5Aom`^
z5tgXNKIJ&O1fx_`C;-Y&^~Td!`9d)JS3J;58zl?2nxa$76__!+>7zro`pMzO?6|sQ
zS}Wzqr?~mh_(f}fxtBI>7L3~lRG1qT>;;-cE3zfh=|M2*@<LF6*J~VX6B5|{Da@v^
zuVR&}aXpaG7{mT1GO0DsnI_(zKEqOVfLE`vE!<+}R^bYr00Ab!8x|M8&O<2QxRu7{
znS!v+)Q`J$o*j;SD!F(n-3bD#n!(&&%Mvb<j}={izFw$w_awC&=C1^Yw{MB#)e*rU
zI<XwK_9DU`bnnVo!>8dJIiQG9S07DHY&P}jgzAYISJSe&s08o8j8UJpmJVSt0{bre
z?SGKMG;xa}K&WTb>eX(VQLIx{RWD{|qb@C_Vvzr$5y&UHLQP+@_%E2jL1{4pa*SI`
zn=!EzQpapM!-lBEgVtPt1%(&3=L8M|u1I;d)=u0)yEdMf|Lfg*7Yzl(m^H}%vuE-<
z@tdW)Rk~>C*vm0ln<4*uPIQO_yT|Bfwg^4qE?=PEB(D3aH@rG%jRxfpcqf%AT!5u|
z-xWn5lOy#8TfO!*pKJ3`O!=RclUD|!&>|V%ieIKIb^F(Ptz7$@@594r7d7r(e7{}w
zO6Lvg_JM;e)vw3lDJ6e{8{FL__l|xm<SkMSHPcEA5fbQN{q~kH8XNRGb@{si*HA@N
z(Cq*pO-8>C*5pD$bGEKikC#6{y6T=f;C~V@RtM?|AiiUB58SY&QJ77gk09rb+F=BK
zhx1+`M7R!D6j96gv;ejoY-$MuvV_K#P6pTI8z#(vA%p_?zl)fUh9x^Gg0I0nson{p
zM89ao1GU=_)(>UuMHOTkbDpnHUEG|ja{;oS+89LN;y?d<`aPJg1wVwDLEy)FRSio$
z%#xe82a;p=ns?fgi7%!n*-2xIyb@d_e>?v96JeCH9*vM5GHCeeDkq%3h=!Ol%WDUF
zqZ#l8Kt*K~M$FB2W}ELee>!Vd47&RBa8Wvj4uPV>v!y^I?ORXqXpAxX^QFU+L16vt
z)mpH_V<ALZShGMRk+Nv)wI*%pZPR-{cH-v5rjK+L7Y^s|&;;JW+!Pl*-`40Ez=CR+
zf4$ZHPEG@TpXQn;hA;9cT5PMrjCUpnw;Ac27<@O600lWY57wBn8}xY1hN<#rrn2qF
z^F-@zKI_a9DN8NDPJ=RWSdaSRC&2I$OTL>4;{~F#F6(PghZIRct_>VwUMv%!*K3)n
zQWg<*VNMm@tCQb?9)QUmx->5|WOk~#ynAgj!jUx%uV?GGCL3PG<%4rMMil{M&o~T@
zwS4&KXW?8kn6iDeG*ZWh<3AGfVSZDGFc8vKJ>;oRF#`G^e%){ABYfWrxDHF?|AbI-
zkjBb(d1i90L-F<jKW*lojojP47}jsQMP1bbE~vZ7wpJ>~1IQh4kS=%??P@$IA2sil
zLhuotnQm*QF}r`Qd5To>U#Xm{OXahxejehjyJl#p_FqAY`d^Omlhi$a>iRlBg2Rng
zyXn2)m8Xg*!chzDX$uE{h8nU|rX3YRXPNJ|l4hc!4t$c7?8l@BYkqfla&Qws{i54&
zphZqz6(xHC{!T(}$?g`1B~~+@4JhRlFIjX=S;Oke;`p8T`&c;~N)+Y5+cSF`>Kf@C
zM$MKfx5g^lEe{j-ey88lGOzs&XjwvR>q=!=CU9!@PdbeD(J%oh2<NaPK(%7<M-rB-
zDGcJPfDwZ6kPlTHWp+WIL)1g$>Y`^WNnsw^=;VtX?@W&rBj%D&Qz0qrrW3)IxH4&n
z8i*AWrIH9WgV<rgL()3&>gd>DhI1+;PxI674t5z!M-dnDu^h#?W#LZfmt$Z%1E_5j
z>MTBUWo2qO954sL2poR}r3OK2e29wFenUEvUY;oL^(|O~Hs{a~T36by_QHyxlJ=b4
zCVEQ{Q2t^HuUUz;N;MM(X%-3i@%NXVbeJp&I8+B6<|ONdp7Es&b}B{xF0NtD=q1Y<
zsws902GY<1T^lwOk$HQhi$XcvF07M(Ns7qnt)ET>B88-qbFv}U#aSXa!6%89vvGHS
zzOq$6Z$3lUGHeu{KQJ!9^NViy+z?+8S}E^{*G^dxdR88R+~$Ww27K+;lBgCCtJap?
z(5ha`a^>Ut03#)RhCzgLeqm`(Aj$IIh%<1_w>QY`A|w7QUnJH?5K>N4+fSJ`SEw-P
zHs}?Abj{Vf&4Vyvoet9T`EAF9+t~B%b2iH){5WWRn|NamtCLBm3i3iU4)S7^=&?z;
z_1xAFCe2TP2qF*0e1B|-dnry!4UZlX7@G9alKSx~P&StueAA=okw(dZ{z}5=U%0|!
zpStK>WAk{zGmGx46yIyF#ANr~vR=Q_Wpv<`#M|-h|Iy$iC;F(pI!sVbeP&{cA!2s=
z{g1~O_-hsZWIr8d)-v7GZf?jsUXvF?mMzh5b*w>H5-$&*)f(G;i@+2MV5a%@2}4`z
zC5jnHw3@FEp0&S_gZBa%6G`-K$0#7pb?N1`^$IqK$v+c-Qug%=hGPMfJ4!vkN#0<Z
z){&Gs^hCchU@q-S=lE;B6tj*C9qIq`^MzgUJ6aUbc+&+kRZ-G_>$%<}wdZ{WQ7|QS
zBn+&$d_iYp3?5MhH5cwrw>~s~!V7nz0MzE)T!mY3xiG2Hwc_7l!ihU>*WS`g9;=-<
zraht&pHQ>M0qV-8|72~Af|rV6ubwMmNzm4d_YxMJy`4};lO_;9zR6E%EY~3!6Pwv`
z>#JkZCDmO<BygdAOV`)bFfB{O&9y`jBRv{J)(mYoElPiIEY&;=K&c<fz*X~N0Z2ee
z$?cT+-f4Q}L2B0sd=_G_{CGp}-gz>Pj&xqid=cpb7+?r}hn2cr+*2|=<8MAnLb%O6
zhS*B&uk*c%*2_g9^A;9G!(iQmd8y!8bpq<dNs=h}WDxT0R}I(X`HxQlo13`tI@RVu
zK3DQB1BV_D0~)hKts4XDQ88VI1y@Y>bL=thb{>0BWF$YvH1o<ygze(Xf&n$%_HyPK
zVoElE&h&!_Y$Oi<Z?Ba?G*VMxNsp+xgPF}Yr{1*duMn_IPwEh!KghMP_8AM-7V;1S
z;Xc%U#{FES>v;<{AQ(pAqi;cUAO}|iq>QMTz{Q#>8<?lS&~%bksw;c|sq+i1#^0oS
z9_HQUB{m<bX2aUE-SA?@zxPFOEL5b<7ABzd@S(os@S#v4#$jejXUAWY{|ikn(@0~=
zNsMWY{Z(xk-yfI-9Ujum2BpTWUo^LQt+}@@O-<vz$h?v2k=;fV<)t$^(u#;7((SWa
z@y|Bode+og1r&guge-y2h&uWVW7_u&X-f7CV#`;E`}vvT{WIdJ<uk_ZdNvpccv57f
zLAuWR|I^@OYAEVUrJR(vn8xLrQ2Opv2FwBw)7vWcOK-?fa(qBVw0VNj)$^ZFLQ&5m
z6j4l{W;iOT-=ow&BvH&lTovu=Y<LTD(tkj!Ns>5>^W?2x0pg>Q8C#*`x1_i&@Hd|{
zl@heg%k~RYBJr2A+dL))LVoX?(53l_$o?&1Gj)IX19-E@-eNB^agxkf*uBOLF-VDo
zWb~zDlc&7cy|0B|n@9pVS(UL9v1NAO!-^G7Q+>uWlr=2mi@vzIj}MmpF<Hd@YT>Ya
zWRL$)-RTZF0VFrwhYR^_DL;^6w!qc#eXItSTe!IT20E3ayA~B?IR#GseTr}3mZF2x
zNJQkU9+Jvx=ibUy=+6@AZYHi0eEGuEVYHp(=YDv>h-ib~EMnpzO#X&{XuFJjL0^mj
z|BY313%$`-I`uBrDB3^6ZsHuDEFQ8CBRpd{t5W0+0CcuI_AP=_eBSX;%xd5A>w+4G
zJH2HJh-YrAMf`i^ulS%J`^uvwEQDqM>>G<**v71%gw*~cT-Sx12r!H2_j)I1sfoD!
zXFVV8tK?#4gDRb`TWY6~e<NFdOt)&+>cocM=fElVX0A>>=0mM-)W*|KH;Xq09^b??
zxo7VK8TnxkMHMUGuiI2TN-N$1d?ZXC=PI)#y8Qi<f->x8wk&nNZVpkh`0U*4JCvDD
z7xA~XDMSAKT+eIWC{4#!TCZh7>@bO{Xg{*z{7zz?8{EP~mna`(X7fuNP~vm@Y2{&7
z&-bCXNIWqMJHIbgK<tFa&6|TUw#8L^$bZ5PDB2GzIgD>6BT$y(ef;H_s#N>qvAf<y
zq@$aD4{gAM^YL%2jf*(<UiP}D-tBooihokdkH0S>imUULCQLGBxftj5A#U8_*>L1>
zrra=Ynxu?sZLO5+FmDxtU6q{|$a=cQ=4>p+@#DV-2a@rbEM&QCuR63G4T$%pe>1lk
ziR))l<CSs9r5D?3G*9?3y(yxjbn7J2w6JtN3HCjj<@iz?6`|TZZ^3D5LXkbA*<WIZ
VLf}&0^xwl-R#Hi#Ow1_g{{W3m`DFkA

diff --git a/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss b/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss
index 15ee60cdb8427..bfa6c6ce78aad 100644
--- a/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss
+++ b/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss
@@ -47,4 +47,25 @@ $fa-font-path: '~font-awesome/fonts';
   @extend .fa-question-circle-o;
 
   cursor: default;
-}
\ No newline at end of file
+}
+
+.icon-note {
+  @extend .fa;
+  @extend .fa-info-circle;
+
+  cursor: default;
+}
+
+.icon-danger {
+  @extend .fa;
+  @extend .fa-exclamation-triangle;
+
+  cursor: default;
+}
+
+.icon-success {
+  @extend .fa;
+  @extend .fa-check-circle-o;
+
+  cursor: default;
+}
diff --git a/modules/web-console/frontend/public/stylesheets/form-field.scss b/modules/web-console/frontend/public/stylesheets/form-field.scss
index f126786f1bcf3..ae33d7509a579 100644
--- a/modules/web-console/frontend/public/stylesheets/form-field.scss
+++ b/modules/web-console/frontend/public/stylesheets/form-field.scss
@@ -106,3 +106,40 @@
         @include make-lg-column(8);
     }
 }
+
+.ignite-form-field {
+    &__btn {
+        overflow: hidden;
+
+        border-top-left-radius: 0;
+        border-bottom-left-radius: 0;
+
+        &.btn {
+            float: right;
+            margin-right: 0;
+
+            line-height: 20px;
+        }
+
+        input {
+            position: absolute;
+            left: 100px;
+        }
+
+        input:checked + span {
+            color: $brand-info;
+        }
+    }
+
+    &__btn ~ &__btn {
+        border-right: 0;
+        border-top-right-radius: 0;
+        border-bottom-right-radius: 0;  
+    }
+
+    &__btn ~ .input-tip input {
+        border-right: 0;
+        border-top-right-radius: 0;
+        border-bottom-right-radius: 0;  
+    }
+}
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index 172abf478f755..4318fc2abc5eb 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -227,7 +227,7 @@ ul.navbar-nav, .sidebar-nav {
         overflow: hidden;
         white-space: nowrap;
         text-overflow: ellipsis;
-        
+
         &:hover,
         &:focus {
             text-decoration: none;
@@ -601,6 +601,10 @@ button.form-control {
 .theme-line .notebook-header {
     border-color: $gray-lighter;
 
+    button:last-child {
+        margin-right: 0;
+    }
+
     h1 {
         padding: 0;
         margin: 0;
@@ -611,7 +615,7 @@ button.form-control {
             overflow: hidden;
             text-overflow: ellipsis;
             white-space: nowrap;
-            margin-top: 5px;
+            height: 24px;
         }
 
         .btn-group {
@@ -637,7 +641,7 @@ button.form-control {
 }
 
 .theme-line .paragraphs {
-    .panel-group .panel + .panel {
+    .panel-group .panel-paragraph + .panel-paragraph {
         margin-top: 30px;
     }
 
@@ -679,8 +683,14 @@ button.form-control {
         line-height: 55px;
     }
 
-    .sql-controls {
+    .panel-collapse {
         border-top: 1px solid $ignite-border-color;
+    }
+
+    .sql-controls {
+        position: relative;
+        top: -1px;
+        border-top: 1px solid #ddd;
 
         padding: 10px 10px;
 
@@ -690,9 +700,50 @@ button.form-control {
         }
 
         label {
-            line-height: 20px !important;
+            line-height: 28px;
             vertical-align: middle;
         }
+
+        .btn {
+            line-height: 20px;
+        }
+
+        .ignite-form-field {
+            margin-right: 10px;
+
+            .ignite-form-field__label {
+                float: left;
+                width: auto;
+                margin-right: 5px;
+                line-height: 28px;
+            }
+
+            .ignite-form-field__label + div {
+                display: block;
+                float: none;
+                width: auto;
+            }
+        }
+
+        .tipLabel .btn {
+            float: right;
+        }
+
+        .pull-right {
+            margin-left: 10px;
+
+            .ignite-form-field {
+                margin-right: -24px;
+
+                label {
+                    margin-left: 5px;
+                }
+            }
+        }
+
+        .col-sm-3 + .tipLabel {
+            margin-left: 0;
+        }
     }
 
     .sql-result {
@@ -1243,6 +1294,17 @@ button.form-control {
     .clickable { cursor: pointer; }
 }
 
+
+.theme-line .summary {
+    .actions-note {
+        i {
+            margin-right: 5px;
+        }
+
+        margin: 15px 0;
+    }
+}
+
 .theme-line .popover.summary-project-structure {
     @extend .popover.settings;
 
@@ -1693,6 +1755,7 @@ th[st-sort] {
 }
 
 .chart-settings-link {
+    margin-top: -2px;
     padding-left: 10px;
     line-height: $input-height;
 
@@ -1991,6 +2054,10 @@ treecontrol.tree-classic {
         margin-right: 0
     }
 
+    .ui-grid-cell-actions {
+        line-height: 28px;
+    }
+
     .no-rows {
         .center-container {
             background: white;
@@ -2226,3 +2293,37 @@ html,body,.splash-screen {
         animation: none 0s;
     }
 }
+
+.admin-page {
+    .panel-heading {
+        border-bottom: 0;
+        padding-bottom: 0;
+
+        cursor: default;
+
+        i {
+            margin-right: 10px;
+        }
+
+        label {
+            cursor: default;
+            line-height: 24px;
+        }
+
+        sub {
+            bottom: 0;
+        }
+    }
+
+    .ui-grid-header-cell input {
+      font-weight: normal;
+    }
+
+    .ui-grid-header-cell input {
+      font-weight: normal;
+    }
+
+    .ui-grid-filter-select {
+        width: calc(100% - 10px);
+    }
+}
diff --git a/modules/web-console/frontend/test/unit/JavaTypes.test.js b/modules/web-console/frontend/test/unit/JavaTypes.test.js
index 2df8c6ac2426e..49e78cc75efd4 100644
--- a/modules/web-console/frontend/test/unit/JavaTypes.test.js
+++ b/modules/web-console/frontend/test/unit/JavaTypes.test.js
@@ -17,11 +17,11 @@
 
 import JavaTypes from '../../app/services/JavaTypes.service.js';
 
-import ClusterDflts from '../../app/modules/configuration/generator/defaults/cluster.provider';
-import CacheDflts from '../../app/modules/configuration/generator/defaults/cache.provider';
-import IgfsDflts from '../../app/modules/configuration/generator/defaults/igfs.provider';
+import ClusterDflts from '../../app/modules/configuration/generator/defaults/Cluster.service';
+import CacheDflts from '../../app/modules/configuration/generator/defaults/Cache.service';
+import IgfsDflts from '../../app/modules/configuration/generator/defaults/IGFS.service';
 
-const INSTANCE = new JavaTypes((new ClusterDflts()).$get[0](), (new CacheDflts()).$get[0](), (new IgfsDflts()).$get[0]());
+const INSTANCE = new JavaTypes(new ClusterDflts(), new CacheDflts(), new IgfsDflts());
 
 import { assert } from 'chai';
 
@@ -58,9 +58,14 @@ suite('JavaTypesTestsSuite', () => {
 
     test('shortClassName', () => {
         assert.equal(INSTANCE.shortClassName('java.math.BigDecimal'), 'BigDecimal');
+        assert.equal(INSTANCE.shortClassName('BigDecimal'), 'BigDecimal');
         assert.equal(INSTANCE.shortClassName('int'), 'int');
         assert.equal(INSTANCE.shortClassName('java.lang.Integer'), 'Integer');
+        assert.equal(INSTANCE.shortClassName('Integer'), 'Integer');
         assert.equal(INSTANCE.shortClassName('java.util.UUID'), 'UUID');
+        assert.equal(INSTANCE.shortClassName('java.sql.Date'), 'Date');
+        assert.equal(INSTANCE.shortClassName('Date'), 'Date');
+        assert.equal(INSTANCE.shortClassName('com.my.Abstract'), 'Abstract');
         assert.equal(INSTANCE.shortClassName('Abstract'), 'Abstract');
     });
 
@@ -113,8 +118,8 @@ suite('JavaTypesTestsSuite', () => {
         assert.equal(INSTANCE.isKeyword(' '), false);
     });
 
-    test('isJavaPrimitive', () => {
-        assert.equal(INSTANCE.isJavaPrimitive('boolean'), true);
+    test('isPrimitive', () => {
+        assert.equal(INSTANCE.isPrimitive('boolean'), true);
     });
 
     test('validUUID', () => {
diff --git a/modules/web-console/frontend/test/unit/Version.test.js b/modules/web-console/frontend/test/unit/Version.test.js
index a67fde8b2e13d..2d75ab583a2c6 100644
--- a/modules/web-console/frontend/test/unit/Version.test.js
+++ b/modules/web-console/frontend/test/unit/Version.test.js
@@ -39,7 +39,13 @@ suite('VersionServiceTestsSuite', () => {
     });
 
     test('Version a = b', () => {
-        assert.equal(INSTANCE.compare('1.7.0', '1.7.0'), 0);
+        assert.equal(INSTANCE.compare('1.0.0', '1.0.0'), 0);
+        assert.equal(INSTANCE.compare('1.2.0', '1.2.0'), 0);
+        assert.equal(INSTANCE.compare('1.2.3', '1.2.3'), 0);
+
+        assert.equal(INSTANCE.compare('1.0.0-1', '1.0.0-1'), 0);
+        assert.equal(INSTANCE.compare('1.2.0-1', '1.2.0-1'), 0);
+        assert.equal(INSTANCE.compare('1.2.3-1', '1.2.3-1'), 0);
     });
 
     test('Version a < b', () => {
diff --git a/modules/web-console/frontend/views/configuration/domains-import.jade b/modules/web-console/frontend/views/configuration/domains-import.jade
index e4f95bcaa6e69..bbcb391cbedef 100644
--- a/modules/web-console/frontend/views/configuration/domains-import.jade
+++ b/modules/web-console/frontend/views/configuration/domains-import.jade
@@ -62,7 +62,10 @@ mixin td-ellipses-lbl(w, lbl)
                             +ignite-form-field-dropdown('Driver JAR:', 'ui.selectedJdbcDriverJar', '"jdbcDriverJar"', false, true, false,
                                 'Choose JDBC driver', '', 'jdbcDriverJars',
                                 'Select appropriate JAR with JDBC driver<br> To add another driver you need to place it into "/jdbc-drivers" folder of Ignite Web Agent<br> Refer to Ignite Web Agent README.txt for for more information'
-                            )(data-container='.modal-domain-import')
+                            )(
+                                data-container='.modal-domain-import'
+                                data-ignite-form-field-input-autofocus='true'
+                            )
                         .settings-row.settings-row_small-label
                             +java-class('JDBC driver:', 'selectedPreset.jdbcDriverClass', '"jdbcDriverClass"', true, true, 'Fully qualified class name of JDBC driver that will be used to connect to database')
                         .settings-row.settings-row_small-label
diff --git a/modules/web-console/frontend/views/configuration/summary.jade b/modules/web-console/frontend/views/configuration/summary.jade
index 9a6e553f43103..a04f0dbbf074b 100644
--- a/modules/web-console/frontend/views/configuration/summary.jade
+++ b/modules/web-console/frontend/views/configuration/summary.jade
@@ -21,7 +21,7 @@ mixin hard-link(ref, txt)
 
 .docs-header
     h1 Configurations Summary
-.docs-body
+.docs-body.summary
     ignite-information
         ul
             li Preview XML configurations for #[a(href='https://apacheignite.readme.io/docs/clients-vs-servers' target='_blank') server and client] nodes
@@ -29,7 +29,6 @@ mixin hard-link(ref, txt)
             li Preview #[a(href='https://apacheignite.readme.io/docs/docker-deployment' target='_blank') Docker file]
             li Preview POM dependencies
             li Download ready-to-use Maven project
-
     hr
     .padding-dflt(ng-if='ui.ready && (!clusters || clusters.length == 0)')
         | You have no clusters configured. Please configure them #[a(ui-sref='base.configuration.clusters') here].
@@ -37,13 +36,21 @@ mixin hard-link(ref, txt)
     div(ng-show='clusters && clusters.length > 0' ignite-loading='summaryPage' ignite-loading-text='Loading summary screen...' ignite-loading-position='top')
         +main-table('clusters', 'clustersView', 'clusterName', 'selectItem(row)', '{{$index + 1}}) {{row.name}}', 'name')
         div(ng-show='selectedItem && contentVisible(displayedRows, selectedItem)')
-            .padding-top-dflt(bs-affix)
-                button.btn.btn-primary(id='download' ng-click='downloadConfiguration()' bs-tooltip='' data-title='Download project' data-placement='bottom') Download project
-                .btn.btn-primary(bs-tooltip='' data-title='Preview generated project structure' data-placement='bottom')
-                    div(bs-popover data-template-url='/configuration/summary-project-structure.html', data-placement='bottom', data-trigger='click' data-auto-close='true')
-                        i.fa.fa-sitemap
-                        label.tipLabel Project structure
-                button.btn.btn-primary(id='proprietary-jdbc-drivers' ng-if='downloadJdbcDriversVisible()' ng-click='downloadJdbcDrivers()' bs-tooltip='' data-title='Open proprietary JDBC drivers download pages' data-placement='bottom') Download JDBC drivers
+            .actions.padding-top-dflt(bs-affix)
+                div
+                    button.btn.btn-primary(id='download' ng-click='downloadConfiguration()' bs-tooltip='' data-title='Download project' data-placement='bottom' ng-disabled='isPrepareDownloading')
+                        div
+                            i.fa.fa-fw.fa-download(ng-hide='isPrepareDownloading')
+                            i.fa.fa-fw.fa-refresh.fa-spin(ng-show='isPrepareDownloading')
+                            span.tipLabel Download project
+                    button.btn.btn-primary(bs-tooltip='' data-title='Preview generated project structure' data-placement='bottom')
+                        div(bs-popover data-template-url='/configuration/summary-project-structure.html', data-placement='bottom', data-trigger='click' data-auto-close='true')
+                            i.fa.fa-sitemap
+                            label.tipLabel Project structure
+                    button.btn.btn-primary(id='proprietary-jdbc-drivers' ng-if='downloadJdbcDriversVisible()' ng-click='downloadJdbcDrivers()' bs-tooltip='' data-title='Open proprietary JDBC drivers download pages' data-placement='bottom') Download JDBC drivers
+                .actions-note(ng-show='ui.isSafari')
+                    i.icon-note
+                    label "Download project" is not fully supported in Safari. Please rename downloaded file from "Unknown" to "&lt;project-name&gt;.zip"
                 hr
             .bs-affix-fix
             .panel-group(bs-collapse ng-init='ui.activePanels=[0,1]' ng-model='ui.activePanels' data-allow-multiple='true')
diff --git a/modules/web-console/frontend/views/settings/admin.jade b/modules/web-console/frontend/views/settings/admin.jade
index 862d959e1fb81..c9858269eeaf3 100644
--- a/modules/web-console/frontend/views/settings/admin.jade
+++ b/modules/web-console/frontend/views/settings/admin.jade
@@ -14,63 +14,38 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-.row(ng-controller='adminController')
+mixin grid-settings()
+    i.fa.fa-bars(data-animation='am-flip-x' bs-dropdown='' aria-haspopup='true' aria-expanded='expanded' data-auto-close='1' data-trigger='click')
+    ul.select.dropdown-menu(role='menu')
+        li(ng-repeat='item in ctrl.gridOptions.categories|filter:{selectable:true}')
+            a(ng-click='ctrl.toggleColumns(item, !item.visible)')
+                i.fa.fa-check-square-o.pull-left(ng-if='item.visible')
+                i.fa.fa-square-o.pull-left(ng-if='!item.visible')
+                span {{::item.name}}
+        li.divider
+        li
+            a(ng-click='ctrl.selectAllColumns()') Select all
+        li
+            a(ng-click='ctrl.clearAllColumns()') Clear all
+        li.divider
+        li
+            a(ng-click='$hide()') Close
+
+.admin-page.row(ng-controller='adminController')
     .docs-content.greedy
         .docs-header
             h1 List of registered users
             hr
         .docs-body
-            .col-xs-12
-                table.table.table-striped.table-vertical-middle.admin(st-table='displayedUsers' st-safe-src='users')
-                    thead
-                        tr
-                            th.header(colspan='10')
-                                .col-xs-3
-                                    input.form-control(type='text' st-search='label' placeholder='Filter users...')
-                                .col-xs-9.admin-summary.text-right(colspan='10')
-                                    strong Total users: {{ users.length }}
-                                .col-xs-offset-6.col-xs-6.text-right
-                                    div(st-pagination st-items-by-page='15' st-displayed-pages='5' st-template='../templates/pagination.html')
-                        tr
-                            th(st-sort='userName') User
-                            th(st-sort='email') Email
-                            th(st-sort='company') Company
-                            th(st-sort='country') Country
-                            th.col-xs-2(st-sort='lastLogin' st-sort-default='reverse') Last login
-                            th.text-nowrap(st-sort='counters.clusters' st-descending-first bs-tooltip='"Clusters count"' data-placement='top')
-                                i.fa.fa-sitemap()
-                            th.text-nowrap(st-sort='counters.models' st-descending-first bs-tooltip='"Models count"' data-placement='top')
-                                i.fa.fa-object-group()
-                            th.text-nowrap(st-sort='counters.caches' st-descending-first bs-tooltip='"Caches count"' data-placement='top')
-                                i.fa.fa-database()
-                            th.text-nowrap(st-sort='counters.igfs' st-descending-first bs-tooltip='"IGFS count"' data-placement='top')
-                                i.fa.fa-folder-o()
-                            th(width='1%') Actions
-                    tbody
-                        tr(ng-repeat='row in displayedUsers track by row._id')
-                            td {{::row.userName}}
-                            td
-                                a(ng-href='mailto:{{::row.email}}') {{::row.email}}
-                            td {{::row.company}}
-                            td {{::row.countryCode}}
-                            td {{::row.lastLogin | date:'medium'}}
-                            td {{::row.counters.clusters}}
-                            td {{::row.counters.models}}
-                            td {{::row.counters.caches}}
-                            td {{::row.counters.igfs}}
-                            td.text-center
-                                a.btn.btn-default.dropdown-toggle(bs-dropdown='' ng-show='row._id != user._id' data-placement='bottom-right')
-                                    i.fa.fa-gear &nbsp;
-                                    span.caret
-                                ul.dropdown-menu(role='menu')
-                                    li
-                                        a(ng-click='becomeUser(row)') Become this user
-                                    li
-                                        a(ng-click='toggleAdmin(row)' ng-if='row.admin && row._id !== user._id') Revoke admin
-                                        a(ng-click='toggleAdmin(row)' ng-if='!row.admin && row._id !== user._id')  Grant admin
-                                    li
-                                        a(ng-click='removeUser(row)') Remove user
-                    tfoot
-                        tr
-                            td.text-right(colspan='10')
-                                div(st-pagination st-items-by-page='15' st-displayed-pages='5' st-template='../templates/pagination.html')
+            .row
+                .col-xs-12
+                    .panel.panel-default
+                        .panel-heading.ui-grid-settings
+                            +grid-settings
+                            label Total users: 
+                                strong {{ users.length }}&nbsp;&nbsp;&nbsp;
+                            label Showing users: 
+                                strong {{ ctrl.gridApi.grid.getVisibleRows().length }}
+                                sub(ng-show='users.length === ctrl.gridApi.grid.getVisibleRows().length') all
+                        .panel-collapse
+                            .grid(ui-grid='ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-pinning)
diff --git a/modules/web-console/frontend/views/sql/notebook-new.jade b/modules/web-console/frontend/views/sql/notebook-new.jade
index 8d9e8c4d9aaa9..9585e92598423 100644
--- a/modules/web-console/frontend/views/sql/notebook-new.jade
+++ b/modules/web-console/frontend/views/sql/notebook-new.jade
@@ -21,7 +21,7 @@
                 button.close(ng-click='$hide()') &times;
                 h4.modal-title
                     i.fa.fa-file-o
-                    | New SQL notebook
+                    | New query notebook
             form.form-horizontal.modal-body.row(name='ui.inputForm' novalidate)
                 div
                     .col-sm-2
diff --git a/modules/web-console/frontend/views/sql/sql.jade b/modules/web-console/frontend/views/sql/sql.jade
index e3f64617adb1f..03015e8aad612 100644
--- a/modules/web-console/frontend/views/sql/sql.jade
+++ b/modules/web-console/frontend/views/sql/sql.jade
@@ -14,6 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
+include /app/helpers/jade/mixins.jade
 include /app/directives/ui-grid-settings/ui-grid-settings.jade
 
 mixin btn-toolbar(btn, click, tip, focusId)
@@ -56,10 +57,15 @@ mixin notebook-rename
             .input-tip
                 input.form-control(ng-model='notebook.editName' required ignite-on-enter='renameNotebook(notebook.editName)' ignite-on-escape='notebook.edit = false;')
         h1.pull-right
-            a.dropdown-toggle(data-toggle='dropdown' bs-dropdown='scrollParagraphs' data-placement='bottom-right') Scroll to query
+            a.dropdown-toggle(style='margin-right: 20px' data-toggle='dropdown' bs-dropdown='scrollParagraphs' data-placement='bottom-right') Scroll to query
                 span.caret
-            .btn-group(style='margin-top: 2px')
-                +btn-toolbar('fa-plus', 'addParagraph()', 'Add new query')
+            button.btn.btn-default(style='margin-top: 2px' ng-click='addQuery()' ignite-on-click-focus=focusId)
+                i.fa.fa-fw.fa-plus
+                | Add query
+
+            button.btn.btn-default(style='margin-top: 2px' ng-click='addScan()' ignite-on-click-focus=focusId)
+                i.fa.fa-fw.fa-plus
+                | Add scan
 
 mixin notebook-error
     h2 Failed to load notebook
@@ -68,7 +74,7 @@ mixin notebook-error
 
 mixin paragraph-rename
     .col-sm-6(ng-hide='paragraph.edit')
-        i.tipLabel.fa(ng-class='paragraphExpanded(paragraph) ? "fa-chevron-circle-down" : "fa-chevron-circle-right"')
+        i.fa(ng-class='paragraphExpanded(paragraph) ? "fa-chevron-circle-down" : "fa-chevron-circle-right"')
         label {{paragraph.name}}
 
         .btn-group(ng-hide='notebook.paragraphs.length > 1')
@@ -85,51 +91,45 @@ mixin paragraph-rename
             input.form-control(id='paragraph-name-{{paragraph.id}}' ng-model='paragraph.editName' required ng-click='$event.stopPropagation();' ignite-on-enter='renameParagraph(paragraph, paragraph.editName)' ignite-on-escape='paragraph.edit = false')
 
 mixin query-settings
-    label.tipLabel Refresh rate:
-        button.btn.btn-default.fa.fa-clock-o.tipLabel(title='Click to show refresh rate dialog' ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='/sql/paragraph-rate.html' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
-    label.tipLabel Page size:
-        button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-options='item for item in pageSizes' bs-select bs-tooltip data-placement='bottom-right' data-title='Max number of rows to show in query result as one page')
-    label.margin-left-dflt(title='Fetch first page of results only')
-        input(type='checkbox' ng-model='paragraph.firstPageOnly')
-        span Fetch first page only
-    label.margin-left-dflt(title='Execute query locally on selected node.\nNode selection dialog will be shown before query execution.')
-        input(type='checkbox' ng-model='paragraph.localQry')
-        span Local query
+    label.tipLabel(bs-tooltip data-placement='bottom' data-title='Configure periodical execution of last successfully executed query') Refresh rate:
+        button.btn.btn-default.fa.fa-clock-o.tipLabel(ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='/sql/paragraph-rate.html' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
+
+    label.tipLabel(bs-tooltip data-placement='bottom' data-title='Max number of rows to show in query result as one page') Page size:
+        button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-select bs-options='item for item in pageSizes')
+
+    label.tipLabel(bs-tooltip data-placement='bottom' data-title='Limit query max results to specified number of pages') Max pages:
+        button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.maxPages' bs-select bs-options='item.value as item.label for item in maxPages')
+
+    label.tipLabel(ng-if='nonCollocatedJoinsAvailable(paragraph)' bs-tooltip data-placement='bottom' data-title='Non-collocated joins is a special mode that allow to join data across cluster without collocation.<br/>\
+        Nested joins are not supported for now.<br/>\
+        <b>NOTE</b>: In some cases it may consume more heap memory or may take a long time than collocated joins.' data-trigger='hover')
+        input(type='checkbox' ng-model='paragraph.nonCollocatedJoins')
+        span Allow non-collocated joins
 
 mixin query-actions
-    .btn-group(bs-tooltip='' data-title='{{actionTooltip(paragraph, "execute", true)}}' data-placement='bottom')
-        button.btn.btn-primary(ng-disabled='!actionAvailable(paragraph, true)' ng-click='execute(paragraph)') Execute
-        button.btn.btn-primary.dropdown-toggle(
-            ng-disabled='!actionAvailable(paragraph, true)'
-            bs-dropdown=''
-            data-container='body'
-            data-placement='bottom-right'
-        )
-            span.caret
-        ul.dropdown-menu(role='menu')
-            li #[a(href='javascript:void(0)' ng-click='execute(paragraph)') Execute]
-            li #[a(href='javascript:void(0)' ng-if='nonCollocatedJoinsAvailable(paragraph)' ng-click='execute(paragraph, true)') Execute non collocated joins]
-    .btn-group(bs-tooltip='' data-title='{{actionTooltip(paragraph, "execute scan", false)}}' data-placement='bottom')
-        button.btn.btn-primary(ng-disabled='!actionAvailable(paragraph, false)' ng-click='scan(paragraph)') Scan
-        button.btn.btn-primary.dropdown-toggle(
-            ng-disabled='!actionAvailable(paragraph, false)'
-            bs-dropdown=''
-            data-container='body'
-            data-placement='bottom-right'
-        )
-            span.caret
-        ul.dropdown-menu(role='menu')
-            li #[a(href='javascript:void(0)' ng-click='scan(paragraph)') Scan]
-            li #[a(href='javascript:void(0)' ng-click='actionAvailable(paragraph, false) && scanWithFilter(paragraph)') Scan with filter]
+    button.btn.btn-primary(ng-disabled='!actionAvailable(paragraph, true)' ng-click='execute(paragraph)') Execute
+    button.btn.btn-primary(ng-disabled='!actionAvailable(paragraph, true)' ng-click='execute(paragraph, true)') Execute on selected node
+
     a.btn.btn-default(ng-disabled='!actionAvailable(paragraph, true)' ng-click='explain(paragraph)' data-placement='bottom' bs-tooltip='' data-title='{{actionTooltip(paragraph, "explain", true)}}') Explain
 
-mixin query-controls
-    .sql-controls
-        +query-actions()
-        .pull-right
-            +query-settings()
+mixin table-result-heading-query
+    .total.row
+        .col-xs-4
+            +ui-grid-settings
+            label Page: #[b {{paragraph.page}}]
+            label.margin-left-dflt Results so far: #[b {{paragraph.rows.length + paragraph.total}}]
+            label.margin-left-dflt Duration: #[b {{paragraph.duration | duration}}]
+        .col-xs-4
+            div(ng-if='paragraph.qryType === "query"')
+                +result-toolbar
+        .col-xs-4
+            .pull-right
+                .btn-group(ng-disabled='paragraph.loading')
+                    button.btn.btn-primary(ng-click='exportCsv(paragraph)' bs-tooltip data-title='{{actionTooltip(paragraph, "export", false)}}') Export
+                    button.btn.btn-primary.dropdown-toggle(id='export-item-dropdown' data-toggle='dropdown' data-container='body' bs-dropdown='exportDropdown' data-placement='bottom-right')
+                        span.caret
 
-mixin table-result
+mixin table-result-heading-scan
     .total.row
         .col-xs-4
             +ui-grid-settings
@@ -137,17 +137,16 @@ mixin table-result
             label.margin-left-dflt Results so far: #[b {{paragraph.rows.length + paragraph.total}}]
             label.margin-left-dflt Duration: #[b {{paragraph.duration | duration}}]
         .col-xs-4
-            +result-toolbar
+            div(ng-if='paragraph.qryType === "query"')
+                +result-toolbar
         .col-xs-4
             .pull-right
-                label(style='margin-right: 10px;')
-                    input(type='checkbox' ng-model='paragraph.systemColumns' ng-change='toggleSystemColumns(paragraph)' ng-disabled='paragraph.disabledSystemColumns')
-                    span Show _KEY, _VAL columns
                 .btn-group(ng-disabled='paragraph.loading')
                     button.btn.btn-primary(ng-click='exportCsv(paragraph)' bs-tooltip data-title='{{actionTooltip(paragraph, "export", false)}}') Export
                     button.btn.btn-primary.dropdown-toggle(id='export-item-dropdown' data-toggle='dropdown' data-container='body' bs-dropdown='exportDropdown' data-placement='bottom-right')
                         span.caret
-    
+
+mixin table-result-body
     .grid(ui-grid='paragraph.gridOptions' ui-grid-resize-columns ui-grid-exporter)
 
 mixin chart-result
@@ -166,12 +165,99 @@ mixin chart-result
                 +result-toolbar
         label.margin-top-dflt Charts do not support #[b Explain] and #[b Scan] query
 
+mixin paragraph-scan
+    .panel-heading(bs-collapse-toggle)
+        .row
+            +paragraph-rename
+    .panel-collapse(role='tabpanel' bs-collapse-target)
+        .col-sm-12.sql-controls
+            .col-sm-3
+                +dropdown-required('Cache:', 'paragraph.cacheName', '"cache"', 'true', 'false', 'Choose cache', 'caches')
+            .col-sm-3
+                +text-enabled('Filter:', 'paragraph.filter', '"filter"', true, false, 'Enter filter')
+                    label.btn.btn-default.ignite-form-field__btn(ng-click='paragraph.caseSensitive = !paragraph.caseSensitive')
+                        input(type='checkbox' ng-model='paragraph.caseSensitive')
+                        span(bs-tooltip data-title='Select this checkbox for case sensitive search') Cs
+            label.tipLabel(bs-tooltip data-placement='bottom' data-title='Max number of rows to show in query result as one page') Page size:
+                button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-select bs-options='item for item in pageSizes')
+
+        .col-sm-12.sql-controls
+            button.btn.btn-primary(ng-disabled='!actionAvailable(paragraph, false)' ng-click='scan(paragraph)')
+                | Scan
+            button.btn.btn-primary(ng-disabled='!actionAvailable(paragraph, false)' ng-click='scan(paragraph, true)')
+                | Scan on selected node
+
+        .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
+            .error(ng-switch-when='error') Error: {{paragraph.errMsg}}
+            .empty(ng-switch-when='empty') Result set is empty
+            .table(ng-switch-when='table')
+                +table-result-heading-scan
+                +table-result-body
+            .footer.clearfix()
+                .pull-left
+                    | Showing results for scan of #[b{{ paragraph.queryArgs.cacheName | defaultName }}]
+                    span(ng-if='paragraph.queryArgs.filter') &nbsp; with filter: #[b {{ paragraph.queryArgs.filter }}]
+                    span(ng-if='paragraph.queryArgs.localNid') &nbsp; on node: #[b {{ paragraph.queryArgs.localNid | limitTo:8 }}]
+
+                -var nextVisibleCondition = 'paragraph.resultType() != "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
+
+                .pull-right(ng-show='#{nextVisibleCondition}' ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
+                    i.fa.fa-chevron-circle-right
+                    a Next
+
+mixin paragraph-query
+    .row.panel-heading(bs-collapse-toggle)
+        +paragraph-rename
+    .panel-collapse(role='tabpanel' bs-collapse-target)
+        .col-sm-12
+            .col-xs-8.col-sm-9(style='border-right: 1px solid #eee')
+                .sql-editor(ignite-ace='{onLoad: aceInit(paragraph), theme: "chrome", mode: "sql", require: ["ace/ext/language_tools"],' +
+                'advanced: {enableSnippets: false, enableBasicAutocompletion: true, enableLiveAutocompletion: true}}'
+                ng-model='paragraph.query')
+            .col-xs-4.col-sm-3
+                div(ng-show='caches.length > 0' style='padding: 5px 10px' st-table='displayedCaches' st-safe-src='caches')
+                    lable.labelField.labelFormField Caches:
+                    i.fa.fa-database.tipField(title='Click to show cache types metadata dialog' bs-popover data-template-url='/sql/cache-metadata.html' data-placement='bottom' data-trigger='click' data-container='#{{ paragraph.id }}')
+                    .input-tip
+                        input.form-control(type='text' st-search='label' placeholder='Filter caches...')
+                    table.links
+                        tbody.scrollable-y(style='max-height: 15em; display: block;')
+                            tr(ng-repeat='cache in displayedCaches track by cache.name')
+                                td(style='width: 100%')
+                                    input.labelField(id='cache_{{ [paragraph.id, $index].join("_") }}' type='radio' value='{{cache.name}}' ng-model='paragraph.cacheName')
+                                    label(for='cache_{{ [paragraph.id, $index].join("_") }} ' ng-bind-html='cache.label')
+                .empty-caches(ng-show='displayedCaches.length == 0 && caches.length != 0')
+                    label Wrong caches filter
+                .empty-caches(ng-show='caches.length == 0')
+                    label No caches
+        .col-sm-12.sql-controls
+            +query-actions
+
+            .pull-right
+                +query-settings
+        .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
+            .error(ng-switch-when='error') Error: {{paragraph.errMsg}}
+            .empty(ng-switch-when='empty') Result set is empty
+            .table(ng-switch-when='table')
+                +table-result-heading-query
+                +table-result-body
+            .chart(ng-switch-when='chart')
+                +chart-result
+            .footer.clearfix
+                a.pull-left(ng-click='showResultQuery(paragraph)') Show query
+
+                -var nextVisibleCondition = 'paragraph.resultType() != "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
+
+                .pull-right(ng-show='#{nextVisibleCondition}' ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
+                    i.fa.fa-chevron-circle-right
+                    a Next
+
 .row(ng-controller='sqlController')
     .docs-content
         .row(ng-if='notebook' bs-affix style='margin-bottom: 20px;')
             +notebook-rename
 
-        ignite-information(data-title='With SQL notebook you can' style='margin-top: 0; margin-bottom: 30px')
+        ignite-information(data-title='With query notebook you can' style='margin-top: 0; margin-bottom: 30px')
             ul
                 li Create any number of queries
                 li Execute and explain SQL queries
@@ -184,46 +270,9 @@ mixin chart-result
         div(ng-if='notebook' ignite-loading='sqlLoading' ignite-loading-text='{{ loadingText }}' ignite-loading-position='top')
             .docs-body.paragraphs
                 .panel-group(bs-collapse ng-model='notebook.expandedParagraphs' data-allow-multiple='true' data-start-collapsed='false')
-                    .panel.panel-default(ng-repeat='paragraph in notebook.paragraphs' id='{{paragraph.id}}')
-                        .panel-heading(bs-collapse-toggle)
-                            .row
-                                +paragraph-rename
-                        .panel-collapse(role='tabpanel' bs-collapse-target)
-                            .col-sm-12
-                                .col-xs-8.col-sm-9(style='border-right: 1px solid #eee')
-                                    .sql-editor(ignite-ace='{onLoad: aceInit(paragraph), theme: "chrome", mode: "sql", require: ["ace/ext/language_tools"],' +
-                                    'advanced: {enableSnippets: false, enableBasicAutocompletion: true, enableLiveAutocompletion: true}}'
-                                    ng-model='paragraph.query')
-                                .col-xs-4.col-sm-3
-                                    div(ng-show='caches.length > 0' style='padding: 5px 10px' st-table='displayedCaches' st-safe-src='caches')
-                                        lable.labelField.labelFormField Caches:
-                                        i.fa.fa-database.tipField(title='Click to show cache types metadata dialog' bs-popover data-template-url='/sql/cache-metadata.html' data-placement='bottom' data-trigger='click' data-container='#{{ paragraph.id }}')
-                                        .input-tip
-                                            input.form-control(type='text' st-search='label' placeholder='Filter caches...')
-                                        table.links
-                                            tbody.scrollable-y(style='max-height: 15em; display: block;')
-                                                tr(ng-repeat='cache in displayedCaches track by cache.name')
-                                                    td(style='width: 100%')
-                                                        input.labelField(id='cache_{{ [paragraph.id, $index].join("_") }}' type='radio' value='{{cache.name}}' ng-model='paragraph.cacheName')
-                                                        label(for='cache_{{ [paragraph.id, $index].join("_") }} ' ng-bind='cache.label')
-                                    .empty-caches(ng-show='displayedCaches.length == 0 && caches.length != 0')
-                                        label Wrong caches filter
-                                    .empty-caches(ng-show='caches.length == 0')
-                                        label No caches
-                            .col-sm-12
-                                +query-controls
-                            .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
-                                .error(ng-switch-when='error') Error: {{paragraph.errMsg}}
-                                .empty(ng-switch-when='empty') Result set is empty
-                                .table(ng-switch-when='table')
-                                    +table-result
-                                .chart(ng-switch-when='chart')
-                                    +chart-result
-                                .footer.clearfix
-                                    a.pull-left(ng-click='showResultQuery(paragraph)') Show query
-
-                                    -var nextVisibleCondition = 'paragraph.resultType() != "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
-
-                                    .pull-right(ng-show=nextVisibleCondition ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
-                                        i.fa.fa-chevron-circle-right
-                                        a Next
+
+                    .panel-paragraph(ng-repeat='paragraph in notebook.paragraphs' id='{{paragraph.id}}' ng-form='form_{{paragraph.id}}')
+                        .panel.panel-default(ng-if='paragraph.qryType === "scan"')
+                            +paragraph-scan
+                        .panel.panel-default(ng-if='paragraph.qryType === "query"')
+                            +paragraph-query
diff --git a/modules/web-console/frontend/views/templates/alert.jade b/modules/web-console/frontend/views/templates/alert.jade
index 182ba997d06b3..d30d2fd447e5e 100644
--- a/modules/web-console/frontend/views/templates/alert.jade
+++ b/modules/web-console/frontend/views/templates/alert.jade
@@ -16,6 +16,6 @@
 
 .alert(ng-show='type' ng-class='[type ? "alert-" + type : null]')
     button.close(type='button', ng-if='dismissable', ng-click='$hide()') &times;
-    i.alert-icon.fa(ng-if='icon' ng-class='[icon]')
+    i.alert-icon(ng-if='icon' ng-class='[icon]')
     span.alert-title(ng-bind-html='title')
     span.alert-content(ng-bind-html='content')
diff --git a/modules/web-console/frontend/views/templates/select.jade b/modules/web-console/frontend/views/templates/select.jade
index 5b6cc0170f5f0..aa6a2ef9e8a82 100644
--- a/modules/web-console/frontend/views/templates/select.jade
+++ b/modules/web-console/frontend/views/templates/select.jade
@@ -23,4 +23,4 @@ ul.select.dropdown-menu(tabindex='-1' ng-show='$isVisible()' role='select')
         hr(ng-if='match.value == undefined' style='margin: 5px 0')
         a(id='li-dropdown-item-{{$index}}'  role='menuitem' tabindex='-1' ng-class='{active: $isActive($index)}' ng-click='$select($index, $event)' bs-tooltip='widthIsSufficient && !widthIsSufficient("li-dropdown-item-{{$index}}", $index, match.label) ? match.label : ""' data-placement='right auto')
             i(class='{{$iconCheckmark}}' ng-if='$isActive($index)' ng-class='{active: $isActive($index)}')
-            span(ng-bind='match.label')
+            span(ng-bind-html='match.label')

From b252b441a9ada31c7200b385d75e0b3e7c0362dd Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 23 Dec 2016 18:20:44 +0700
Subject: [PATCH 073/446] Implemented Visor tasks for Services.

(cherry picked from commit fdf1f4b)
---
 .../visor/service/VisorCancelServiceTask.java |  70 ++++++++++
 .../visor/service/VisorServiceDescriptor.java | 132 ++++++++++++++++++
 .../visor/service/VisorServiceTask.java       |  75 ++++++++++
 .../internal/visor/util/VisorTaskUtils.java   |  15 +-
 .../resources/META-INF/classnames.properties  |  65 +++++++--
 5 files changed, 342 insertions(+), 15 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorCancelServiceTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorCancelServiceTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorCancelServiceTask.java
new file mode 100644
index 0000000000000..64987e92fc066
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorCancelServiceTask.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.service;
+
+import org.apache.ignite.IgniteServices;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+
+/**
+ * Task for cancel services with specified name.
+ */
+@GridInternal
+public class VisorCancelServiceTask extends VisorOneNodeTask<String, Void> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCancelServiceJob job(String arg) {
+        return new VisorCancelServiceJob(arg, debug);
+    }
+
+    /**
+     * Job for cancel services with specified name.
+     */
+    private static class VisorCancelServiceJob extends VisorJob<String, Void> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with specified argument.
+         *
+         * @param arg Job argument.
+         * @param debug Debug flag.
+         */
+        protected VisorCancelServiceJob(String arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Void run(final String arg) {
+            IgniteServices services = ignite.services();
+
+            services.cancel(arg);
+
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorCancelServiceJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java
new file mode 100644
index 0000000000000..83ec77d5c22e0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.service;
+
+import java.io.Serializable;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.util.VisorTaskUtils;
+import org.apache.ignite.services.ServiceDescriptor;
+
+/**
+ * Data transfer object for {@link ServiceDescriptor} object.
+ */
+public class VisorServiceDescriptor implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Service name. */
+    private String name;
+
+    /** Service class. */
+    private String srvcCls;
+
+    /** Maximum allowed total number of deployed services in the grid, {@code 0} for unlimited. */
+    private int totalCnt;
+
+    /** Maximum allowed number of deployed services on each node. */
+    private int maxPerNodeCnt;
+
+    /** Cache name used for key-to-node affinity calculation. */
+    private String cacheName;
+
+    /** ID of grid node that initiated the service deployment. */
+    private UUID originNodeId;
+
+    /**
+     * Service deployment topology snapshot.
+     * Number of service instances deployed on a node mapped to node ID.
+     */
+    private Map<UUID, Integer> topSnapshot;
+
+    /**
+     * Default constructor.
+     */
+    public VisorServiceDescriptor() {
+        // No-op.
+    }
+
+    /**
+     * Create task result with given parameters
+     *
+     */
+    public VisorServiceDescriptor(ServiceDescriptor srvc) {
+        name = srvc.name();
+        srvcCls = VisorTaskUtils.compactClass(srvc.serviceClass());
+        totalCnt = srvc.totalCount();
+        maxPerNodeCnt = srvc.maxPerNodeCount();
+        cacheName = srvc.cacheName();
+        originNodeId = srvc.originNodeId();
+        topSnapshot = srvc.topologySnapshot();
+    }
+
+    /**
+     * @return Service name.
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * @return Service class.
+     */
+    public String getServiceClass() {
+        return srvcCls;
+    }
+
+    /**
+     * @return Maximum allowed total number of deployed services in the grid, 0 for unlimited.
+     */
+    public int getTotalCnt() {
+        return totalCnt;
+    }
+
+    /**
+     * @return Maximum allowed number of deployed services on each node.
+     */
+    public int getMaxPerNodeCnt() {
+        return maxPerNodeCnt;
+    }
+
+    /**
+     * @return Cache name used for key-to-node affinity calculation.
+     */
+    public String getCacheName() {
+        return cacheName;
+    }
+
+    /**
+     * @return ID of grid node that initiated the service deployment.
+     */
+    public UUID getOriginNodeId() {
+        return originNodeId;
+    }
+
+    /**
+     * @return Service deployment topology snapshot. Number of service instances deployed on a node mapped to node ID.
+     */
+    public Map<UUID, Integer> getTopologySnapshot() {
+        return topSnapshot;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorServiceDescriptor.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java
new file mode 100644
index 0000000000000..1b3495c5e917d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.service;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.apache.ignite.services.ServiceDescriptor;
+
+/**
+ * Task for collect topology service configuration.
+ */
+@GridInternal
+public class VisorServiceTask extends VisorOneNodeTask<Void, Collection<VisorServiceDescriptor>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorServiceJob job(Void arg) {
+        return new VisorServiceJob(arg, debug);
+    }
+
+    /**
+     * Job for collect topology service configuration.
+     */
+    private static class VisorServiceJob extends VisorJob<Void, Collection<VisorServiceDescriptor>> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with specified argument.
+         *
+         * @param arg Job argument.
+         * @param debug Debug flag.
+         */
+        protected VisorServiceJob(Void arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Collection<VisorServiceDescriptor> run(final Void arg) {
+            Collection<ServiceDescriptor> services = ignite.services().serviceDescriptors();
+
+            Collection<VisorServiceDescriptor> res = new ArrayList<>(services.size());
+
+            for (ServiceDescriptor srvc: services)
+                res.add(new VisorServiceDescriptor(srvc));
+
+            return res;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorServiceJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
index 1e9346ced5654..3f5003a03d401 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
@@ -267,6 +267,19 @@ public static int[] concat(int[]... arrays) {
         return U.compact(obj.getClass().getName());
     }
 
+    /**
+     * Compact class names.
+     *
+     * @param cls Class object for compact.
+     * @return Compacted string.
+     */
+    @Nullable public static String compactClass(Class cls) {
+        if (cls == null)
+            return null;
+
+        return U.compact(cls.getName());
+    }
+
     /**
      * Compact class names.
      *
@@ -277,7 +290,7 @@ public static int[] concat(int[]... arrays) {
         if (obj == null)
             return null;
 
-        return U.compact(obj.getClass().getName());
+        return compactClass(obj.getClass());
     }
 
     /**
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 4c9596c854ea0..4d0b931ffc607 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -294,12 +294,17 @@ org.apache.ignite.internal.jdbc2.JdbcDatabaseMetadata$UpdateMetadataTask
 org.apache.ignite.internal.jdbc2.JdbcQueryTask
 org.apache.ignite.internal.jdbc2.JdbcQueryTask$1
 org.apache.ignite.internal.jdbc2.JdbcQueryTask$QueryResult
+org.apache.ignite.internal.jdbc2.JdbcQueryTaskV2
+org.apache.ignite.internal.jdbc2.JdbcQueryTaskV2$1
+org.apache.ignite.internal.jdbc2.JdbcQueryTaskV2$QueryResult
+org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery
 org.apache.ignite.internal.managers.GridManagerAdapter$1$1
 org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager$CheckpointSet
 org.apache.ignite.internal.managers.checkpoint.GridCheckpointRequest
 org.apache.ignite.internal.managers.communication.GridIoManager$ConcurrentHashMap0
 org.apache.ignite.internal.managers.communication.GridIoMessage
 org.apache.ignite.internal.managers.communication.GridIoUserMessage
+org.apache.ignite.internal.managers.communication.IgniteIoTestMessage
 org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean
 org.apache.ignite.internal.managers.deployment.GridDeploymentPerVersionStore$2
 org.apache.ignite.internal.managers.deployment.GridDeploymentRequest
@@ -387,20 +392,20 @@ org.apache.ignite.internal.processors.cache.GridCacheAdapter$3
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$30
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$32
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$4
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$48
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$49
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$50
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$51
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$52
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$53
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$54
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$55
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$57
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$58
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$58$1
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$59
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$6
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$60
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$61
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$62
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$63
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$64
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$65
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$66
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$67
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$69
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$70
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$70$1
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$71
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$72
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$9
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$AsyncOp$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$AsyncOp$1$1
@@ -719,8 +724,11 @@ org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtFor
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysResponse
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$1
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$1$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$2
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$3
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$4$1
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$5$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$DemandWorker$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$DemandWorker$2
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId
@@ -1110,6 +1118,12 @@ org.apache.ignite.internal.processors.hadoop.HadoopJobStatus
 org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan
 org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo
 org.apache.ignite.internal.processors.hadoop.HadoopTaskType
+org.apache.ignite.internal.processors.hadoop.message.HadoopMessage
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopDirectShuffleMessage
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleAck
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishRequest
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishResponse
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleMessage
 org.apache.ignite.internal.processors.igfs.IgfsAckMessage
 org.apache.ignite.internal.processors.igfs.IgfsAttributes
 org.apache.ignite.internal.processors.igfs.IgfsBlockKey
@@ -1207,6 +1221,7 @@ org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryProcessor
 org.apache.ignite.internal.processors.platform.cache.PlatformCachePartialUpdateException
 org.apache.ignite.internal.processors.platform.cache.affinity.PlatformAffinity$1
 org.apache.ignite.internal.processors.platform.cache.affinity.PlatformAffinityFunction
+org.apache.ignite.internal.processors.platform.cache.expiry.PlatformExpiryPolicyFactory
 org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQuery
 org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQueryFilter
 org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQueryImpl
@@ -1243,6 +1258,9 @@ org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$9
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetConfigurationClosure
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetService
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetServiceImpl
+org.apache.ignite.internal.processors.platform.entityframework.PlatformDotNetEntityFrameworkCacheExtension$CleanupCompletionListener
+org.apache.ignite.internal.processors.platform.entityframework.PlatformDotNetEntityFrameworkCacheExtension$RemoveOldEntriesRunnable
+org.apache.ignite.internal.processors.platform.entityframework.PlatformDotNetEntityFrameworkIncreaseVersionProcessor
 org.apache.ignite.internal.processors.platform.events.PlatformEventFilterListenerImpl
 org.apache.ignite.internal.processors.platform.message.PlatformMessageFilter
 org.apache.ignite.internal.processors.platform.messaging.PlatformMessageFilterImpl
@@ -1265,6 +1283,7 @@ org.apache.ignite.internal.processors.query.GridQueryProcessor$6
 org.apache.ignite.internal.processors.query.GridQueryProcessor$7
 org.apache.ignite.internal.processors.query.GridQueryProcessor$8
 org.apache.ignite.internal.processors.query.GridQueryProcessor$IndexType
+org.apache.ignite.internal.processors.query.IgniteSQLException
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageRequest
@@ -1330,6 +1349,9 @@ org.apache.ignite.internal.processors.rest.handlers.datastructures.DataStructure
 org.apache.ignite.internal.processors.rest.handlers.query.CacheQueryFieldsMetaResult
 org.apache.ignite.internal.processors.rest.handlers.query.CacheQueryResult
 org.apache.ignite.internal.processors.rest.handlers.query.QueryCommandHandler$QueryCursorIterator
+org.apache.ignite.internal.processors.rest.handlers.redis.GridRedisRestCommandHandler$1
+org.apache.ignite.internal.processors.rest.handlers.redis.exception.GridRedisGenericException
+org.apache.ignite.internal.processors.rest.handlers.redis.exception.GridRedisTypeException
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler$2
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler$ExeCallable
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultRequest
@@ -1341,6 +1363,9 @@ org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpMemcachedNioList
 org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpMemcachedNioListener$2
 org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpRestNioListener$1
 org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpRestNioListener$1$1
+org.apache.ignite.internal.processors.rest.protocols.tcp.redis.GridRedisCommand
+org.apache.ignite.internal.processors.rest.protocols.tcp.redis.GridRedisMessage
+org.apache.ignite.internal.processors.rest.protocols.tcp.redis.GridRedisNioListener$1
 org.apache.ignite.internal.processors.rest.request.RestQueryRequest$QueryType
 org.apache.ignite.internal.processors.service.GridServiceAssignments
 org.apache.ignite.internal.processors.service.GridServiceAssignmentsKey
@@ -1585,10 +1610,13 @@ org.apache.ignite.internal.util.lang.IgniteReducer2X
 org.apache.ignite.internal.util.lang.IgniteReducer3
 org.apache.ignite.internal.util.lang.IgniteReducer3X
 org.apache.ignite.internal.util.lang.IgniteReducerX
+org.apache.ignite.internal.util.lang.IgniteSingletonIterator
 org.apache.ignite.internal.util.nio.GridNioEmbeddedFuture$1
 org.apache.ignite.internal.util.nio.GridNioException
 org.apache.ignite.internal.util.nio.GridNioMessageTracker
 org.apache.ignite.internal.util.nio.GridNioServer$NioOperation
+org.apache.ignite.internal.util.nio.GridNioServer$RandomBalancer
+org.apache.ignite.internal.util.nio.GridNioServer$SizeBasedBalancer
 org.apache.ignite.internal.util.nio.GridNioSessionMetaKey
 org.apache.ignite.internal.util.nio.ssl.GridNioSslHandler
 org.apache.ignite.internal.util.offheap.GridOffHeapEvent
@@ -1801,6 +1829,11 @@ org.apache.ignite.internal.visor.query.VisorQueryResult
 org.apache.ignite.internal.visor.query.VisorQueryResultEx
 org.apache.ignite.internal.visor.query.VisorQueryScanSubstringFilter
 org.apache.ignite.internal.visor.query.VisorQueryTask
+org.apache.ignite.internal.visor.service.VisorCancelServiceTask
+org.apache.ignite.internal.visor.service.VisorCancelServiceTask$VisorCancelServiceJob
+org.apache.ignite.internal.visor.service.VisorServiceDescriptor
+org.apache.ignite.internal.visor.service.VisorServiceTask
+org.apache.ignite.internal.visor.service.VisorServiceTask$VisorServiceJob
 org.apache.ignite.internal.visor.util.VisorClusterGroupEmptyException
 org.apache.ignite.internal.visor.util.VisorEventMapper
 org.apache.ignite.internal.visor.util.VisorExceptionWrapper
@@ -1858,12 +1891,15 @@ org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointData
 org.apache.ignite.spi.collision.jobstealing.JobStealingRequest
 org.apache.ignite.spi.collision.priorityqueue.PriorityQueueCollisionSpi$PriorityGridCollisionJobContextComparator
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$1
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$10
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$11
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure$1
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$8
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$9
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosureNew
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosureNew$1
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeClosure
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeMessage
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeMessage2
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeTimeoutException
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$NodeIdMessage
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$RecoveryLastReceivedMessage
@@ -1923,3 +1959,4 @@ org.apache.ignite.transactions.TransactionOptimisticException
 org.apache.ignite.transactions.TransactionRollbackException
 org.apache.ignite.transactions.TransactionState
 org.apache.ignite.transactions.TransactionTimeoutException
+org.apache.ignite.util.AttributeNodeFilter

From fb8191028eb19b0683b8239b7024f7fa6ccabd4e Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 23 Dec 2016 18:40:51 +0700
Subject: [PATCH 074/446] Updated classnames for ignite-1.7.5.

---
 .../resources/META-INF/classnames.properties  | 60 +++++--------------
 1 file changed, 15 insertions(+), 45 deletions(-)

diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 4d0b931ffc607..8a6dc6603c4f1 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -294,17 +294,12 @@ org.apache.ignite.internal.jdbc2.JdbcDatabaseMetadata$UpdateMetadataTask
 org.apache.ignite.internal.jdbc2.JdbcQueryTask
 org.apache.ignite.internal.jdbc2.JdbcQueryTask$1
 org.apache.ignite.internal.jdbc2.JdbcQueryTask$QueryResult
-org.apache.ignite.internal.jdbc2.JdbcQueryTaskV2
-org.apache.ignite.internal.jdbc2.JdbcQueryTaskV2$1
-org.apache.ignite.internal.jdbc2.JdbcQueryTaskV2$QueryResult
-org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery
 org.apache.ignite.internal.managers.GridManagerAdapter$1$1
 org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager$CheckpointSet
 org.apache.ignite.internal.managers.checkpoint.GridCheckpointRequest
 org.apache.ignite.internal.managers.communication.GridIoManager$ConcurrentHashMap0
 org.apache.ignite.internal.managers.communication.GridIoMessage
 org.apache.ignite.internal.managers.communication.GridIoUserMessage
-org.apache.ignite.internal.managers.communication.IgniteIoTestMessage
 org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean
 org.apache.ignite.internal.managers.deployment.GridDeploymentPerVersionStore$2
 org.apache.ignite.internal.managers.deployment.GridDeploymentRequest
@@ -392,20 +387,20 @@ org.apache.ignite.internal.processors.cache.GridCacheAdapter$3
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$30
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$32
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$4
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$48
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$49
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$50
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$51
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$52
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$53
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$54
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$55
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$57
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$58
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$58$1
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$59
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$6
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$60
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$61
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$62
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$63
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$64
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$65
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$66
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$67
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$69
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$70
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$70$1
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$71
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$72
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$9
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$AsyncOp$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$AsyncOp$1$1
@@ -724,11 +719,9 @@ org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtFor
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysResponse
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$1
-org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$1$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$2
-org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$3
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$3$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$4$1
-org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$5$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$DemandWorker$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$DemandWorker$2
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId
@@ -1118,12 +1111,6 @@ org.apache.ignite.internal.processors.hadoop.HadoopJobStatus
 org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan
 org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo
 org.apache.ignite.internal.processors.hadoop.HadoopTaskType
-org.apache.ignite.internal.processors.hadoop.message.HadoopMessage
-org.apache.ignite.internal.processors.hadoop.shuffle.HadoopDirectShuffleMessage
-org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleAck
-org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishRequest
-org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishResponse
-org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleMessage
 org.apache.ignite.internal.processors.igfs.IgfsAckMessage
 org.apache.ignite.internal.processors.igfs.IgfsAttributes
 org.apache.ignite.internal.processors.igfs.IgfsBlockKey
@@ -1221,7 +1208,6 @@ org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryProcessor
 org.apache.ignite.internal.processors.platform.cache.PlatformCachePartialUpdateException
 org.apache.ignite.internal.processors.platform.cache.affinity.PlatformAffinity$1
 org.apache.ignite.internal.processors.platform.cache.affinity.PlatformAffinityFunction
-org.apache.ignite.internal.processors.platform.cache.expiry.PlatformExpiryPolicyFactory
 org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQuery
 org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQueryFilter
 org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQueryImpl
@@ -1258,9 +1244,6 @@ org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$9
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetConfigurationClosure
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetService
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetServiceImpl
-org.apache.ignite.internal.processors.platform.entityframework.PlatformDotNetEntityFrameworkCacheExtension$CleanupCompletionListener
-org.apache.ignite.internal.processors.platform.entityframework.PlatformDotNetEntityFrameworkCacheExtension$RemoveOldEntriesRunnable
-org.apache.ignite.internal.processors.platform.entityframework.PlatformDotNetEntityFrameworkIncreaseVersionProcessor
 org.apache.ignite.internal.processors.platform.events.PlatformEventFilterListenerImpl
 org.apache.ignite.internal.processors.platform.message.PlatformMessageFilter
 org.apache.ignite.internal.processors.platform.messaging.PlatformMessageFilterImpl
@@ -1283,7 +1266,6 @@ org.apache.ignite.internal.processors.query.GridQueryProcessor$6
 org.apache.ignite.internal.processors.query.GridQueryProcessor$7
 org.apache.ignite.internal.processors.query.GridQueryProcessor$8
 org.apache.ignite.internal.processors.query.GridQueryProcessor$IndexType
-org.apache.ignite.internal.processors.query.IgniteSQLException
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageRequest
@@ -1349,9 +1331,6 @@ org.apache.ignite.internal.processors.rest.handlers.datastructures.DataStructure
 org.apache.ignite.internal.processors.rest.handlers.query.CacheQueryFieldsMetaResult
 org.apache.ignite.internal.processors.rest.handlers.query.CacheQueryResult
 org.apache.ignite.internal.processors.rest.handlers.query.QueryCommandHandler$QueryCursorIterator
-org.apache.ignite.internal.processors.rest.handlers.redis.GridRedisRestCommandHandler$1
-org.apache.ignite.internal.processors.rest.handlers.redis.exception.GridRedisGenericException
-org.apache.ignite.internal.processors.rest.handlers.redis.exception.GridRedisTypeException
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler$2
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler$ExeCallable
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultRequest
@@ -1363,9 +1342,6 @@ org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpMemcachedNioList
 org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpMemcachedNioListener$2
 org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpRestNioListener$1
 org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpRestNioListener$1$1
-org.apache.ignite.internal.processors.rest.protocols.tcp.redis.GridRedisCommand
-org.apache.ignite.internal.processors.rest.protocols.tcp.redis.GridRedisMessage
-org.apache.ignite.internal.processors.rest.protocols.tcp.redis.GridRedisNioListener$1
 org.apache.ignite.internal.processors.rest.request.RestQueryRequest$QueryType
 org.apache.ignite.internal.processors.service.GridServiceAssignments
 org.apache.ignite.internal.processors.service.GridServiceAssignmentsKey
@@ -1610,13 +1586,10 @@ org.apache.ignite.internal.util.lang.IgniteReducer2X
 org.apache.ignite.internal.util.lang.IgniteReducer3
 org.apache.ignite.internal.util.lang.IgniteReducer3X
 org.apache.ignite.internal.util.lang.IgniteReducerX
-org.apache.ignite.internal.util.lang.IgniteSingletonIterator
 org.apache.ignite.internal.util.nio.GridNioEmbeddedFuture$1
 org.apache.ignite.internal.util.nio.GridNioException
 org.apache.ignite.internal.util.nio.GridNioMessageTracker
 org.apache.ignite.internal.util.nio.GridNioServer$NioOperation
-org.apache.ignite.internal.util.nio.GridNioServer$RandomBalancer
-org.apache.ignite.internal.util.nio.GridNioServer$SizeBasedBalancer
 org.apache.ignite.internal.util.nio.GridNioSessionMetaKey
 org.apache.ignite.internal.util.nio.ssl.GridNioSslHandler
 org.apache.ignite.internal.util.offheap.GridOffHeapEvent
@@ -1891,15 +1864,12 @@ org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointData
 org.apache.ignite.spi.collision.jobstealing.JobStealingRequest
 org.apache.ignite.spi.collision.priorityqueue.PriorityQueueCollisionSpi$PriorityGridCollisionJobContextComparator
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$1
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$10
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$11
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure$1
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosureNew
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosureNew$1
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$8
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$9
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeClosure
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeMessage
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeMessage2
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeTimeoutException
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$NodeIdMessage
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$RecoveryLastReceivedMessage

From 2da2816fd74b17590f45781268337da5205c44fa Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Fri, 23 Dec 2016 18:58:47 +0700
Subject: [PATCH 075/446] Fixed broken links.

(cherry picked from commit 6ca8670)
---
 .../core/src/main/java/org/apache/ignite/IgniteLogger.java  | 6 +++---
 .../main/java/org/apache/ignite/logger/java/JavaLogger.java | 4 ++--
 .../testframework/junits/logger/GridTestLog4jLogger.java    | 4 ++--
 .../java/org/apache/ignite/logger/log4j/Log4JLogger.java    | 4 ++--
 .../apache/ignite/spi/deployment/uri/UriDeploymentSpi.java  | 2 +-
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java b/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
index 8d814fd78b39e..b1433a8ac230a 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
@@ -23,8 +23,8 @@
 /**
  * This interface defines basic logging functionality used throughout the system. We had to
  * abstract it out so that we can use whatever logging is used by the hosting environment.
- * Currently, <a target=_new href="http://logging.apache.org/log4j/docs/">log4j</a>,
- * <a target=_new href="http://www.jboss.org/developers/guides/logging">JBoss</a>,
+ * Currently, <a target=_new href="http://logging.apache.org/log4j/1.2/">log4j</a>,
+ * <a target=_new href="http://docs.jboss.org/hibernate/orm/4.3/topical/html/logging/Logging">JBoss</a>,
  * <a target=_new href="http://jakarta.apache.org/commons/logging/">JCL</a> and
  * console logging are provided as supported implementations.
  * <p>
@@ -158,4 +158,4 @@ public interface IgniteLogger {
      * @return Name of the file being logged to if one is configured or {@code null} otherwise.
      */
     public String fileName();
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/logger/java/JavaLogger.java b/modules/core/src/main/java/org/apache/ignite/logger/java/JavaLogger.java
index d5ff5e388d130..6aa7d3879698a 100644
--- a/modules/core/src/main/java/org/apache/ignite/logger/java/JavaLogger.java
+++ b/modules/core/src/main/java/org/apache/ignite/logger/java/JavaLogger.java
@@ -86,7 +86,7 @@
  *      ...
  *      cfg.setGridLogger(log);
  * </pre>
- * Please take a look at <a target=_new href="http://java.sun.com/j2se/1.4.2/docs/api20/java/util/logging/Logger.html">Logger javadoc</a>
+ * Please take a look at <a target=_new href="http://docs.oracle.com/javase/7/docs/api/java/util/logging/Logger.html">Logger javadoc</a>
  * for additional information.
  * <p>
  * It's recommended to use Ignite logger injection instead of using/instantiating
@@ -406,4 +406,4 @@ private static <T> T findHandler(Logger log, Class<T> cls) {
 
         return null;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLogger.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLogger.java
index 74f5160bb045e..6a46c7d7cc7da 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLogger.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLogger.java
@@ -50,7 +50,7 @@
 
 /**
  * Log4j-based implementation for logging. This logger should be used
- * by loaders that have prefer <a target=_new href="http://logging.apache.org/log4j/docs/">log4j</a>-based logging.
+ * by loaders that have prefer <a target=_new href="http://logging.apache.org/log4j/1.2/">log4j</a>-based logging.
  * <p>
  * Here is a typical example of configuring log4j logger in Ignite configuration file:
  * <pre name="code" class="xml">
@@ -521,4 +521,4 @@ public static Collection<String> logFiles() {
     @Override public String toString() {
         return S.toString(GridTestLog4jLogger.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java b/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
index eaae2d49f5eea..d5b0f0227bd85 100644
--- a/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
+++ b/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
@@ -50,7 +50,7 @@
 
 /**
  * Log4j-based implementation for logging. This logger should be used
- * by loaders that have prefer <a target=_new href="http://logging.apache.org/log4j/docs/">log4j</a>-based logging.
+ * by loaders that have prefer <a target=_new href="http://logging.apache.org/log4j/1.2/">log4j</a>-based logging.
  * <p>
  * Here is a typical example of configuring log4j logger in Ignite configuration file:
  * <pre name="code" class="xml">
@@ -532,4 +532,4 @@ public static Collection<String> logFiles() {
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java
index 22d42db4ac04f..30940e4640956 100644
--- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java
+++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java
@@ -111,7 +111,7 @@
  * {@code META-INF/} entry may contain {@code ignite.xml} file which is a
  * task descriptor file. The purpose of task descriptor XML file is to specify
  * all tasks to be deployed. This file is a regular
- * <a href="http://www.springframework.org/documentation">Spring</a> XML
+ * <a href="https://spring.io/docs">Spring</a> XML
  * definition file.  {@code META-INF/} entry may also contain any other file
  * specified by JAR format.
  * </li>

From 2ccae40e2a21398d15c3762b72575216c56a7fb0 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 23 Dec 2016 17:51:49 +0300
Subject: [PATCH 076/446] IGNITE-4109 - BinaryType.isEnum() throws an exception
 if typeId==0

---
 .../ignite/internal/binary/BinaryContext.java  |  4 ++--
 .../internal/binary/BinaryTypeProxy.java       | 15 ++++++++++++---
 .../ignite/internal/binary/BinaryUtils.java    |  4 +++-
 .../internal/binary/BinaryEnumsSelfTest.java   | 18 ++++++++++++++++++
 4 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index cc18318b04e61..4030ef02c799c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -928,7 +928,7 @@ public int fieldId(int typeId, String fieldName) {
      * @param typeId Type ID.
      * @return Instance of ID mapper.
      */
-    public BinaryInternalMapper userTypeMapper(int typeId) {
+    BinaryInternalMapper userTypeMapper(int typeId) {
         BinaryInternalMapper mapper = typeId2Mapper.get(typeId);
 
         return mapper != null ? mapper : SIMPLE_NAME_LOWER_CASE_MAPPER;
@@ -938,7 +938,7 @@ public BinaryInternalMapper userTypeMapper(int typeId) {
      * @param clsName Type name.
      * @return Instance of ID mapper.
      */
-    private BinaryInternalMapper userTypeMapper(String clsName) {
+    BinaryInternalMapper userTypeMapper(String clsName) {
         BinaryInternalMapper mapper = cls2Mappers.get(clsName);
 
         if (mapper != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
index 17b0bc6e50628..df9901e88d351 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
@@ -24,6 +24,7 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 import java.util.Collection;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Binary type proxy. Is used to delay or completely avoid metadata lookup.
@@ -34,21 +35,26 @@ public class BinaryTypeProxy implements BinaryType {
     private final BinaryContext ctx;
 
     /** Type ID. */
-    private final int typeId;
+    private int typeId;
+
+    /** Raw data. */
+    private final String clsName;
 
     /** Target type. */
     @GridToStringExclude
     private volatile BinaryType target;
 
     /**
-     * Constrcutor.
+     * Constructor.
      *
      * @param ctx Context.
      * @param typeId Type ID.
+     * @param clsName Class name.
      */
-    public BinaryTypeProxy(BinaryContext ctx, int typeId) {
+    public BinaryTypeProxy(BinaryContext ctx, int typeId, @Nullable String clsName) {
         this.ctx = ctx;
         this.typeId = typeId;
+        this.clsName = clsName;
     }
 
     /** {@inheritDoc} */
@@ -93,6 +99,9 @@ private BinaryType target() {
         if (target == null) {
             synchronized (this) {
                 if (target == null) {
+                    if (typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID && clsName != null)
+                        typeId = ctx.typeId(clsName);
+
                     target = ctx.metadata(typeId);
 
                     if (target == null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index bbf50213fdd6d..fdc54c7871711 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -2228,7 +2228,9 @@ public static BinaryType typeProxy(BinaryContext ctx, BinaryObjectEx obj) {
         if (ctx == null)
             throw new BinaryObjectException("BinaryContext is not set for the object.");
 
-        return new BinaryTypeProxy(ctx, obj.typeId());
+        String clsName = obj instanceof BinaryEnumObjectImpl ? ((BinaryEnumObjectImpl)obj).className() : null;
+
+        return new BinaryTypeProxy(ctx, obj.typeId(), clsName);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryEnumsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryEnumsSelfTest.java
index fb7e618899afb..91add0de06d39 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryEnumsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryEnumsSelfTest.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.configuration.BinaryConfiguration;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
 import org.apache.ignite.marshaller.Marshaller;
@@ -388,6 +389,23 @@ public void checkSimpleBuilderArray(boolean registered) throws Exception {
         validateSimpleArray(registered);
     }
 
+    /**
+     * Check ability to resolve typeId from class name.
+     *
+     * @throws Exception If failed.
+     */
+    public void testZeroTypeId() throws Exception {
+        startUp(true);
+
+        final BinaryContext ctx =
+            ((CacheObjectBinaryProcessorImpl)((IgniteEx)node1).context().cacheObjects()).binaryContext();
+
+        final BinaryObject enumObj =
+            new BinaryEnumObjectImpl(ctx, 0, EnumType.class.getName(), EnumType.ONE.ordinal());
+
+        assert enumObj.type().isEnum();
+    }
+
     /**
      * Validate simple array.
      *

From 9273e51cd039049a4aae73f9dcafc02915bc6153 Mon Sep 17 00:00:00 2001
From: Alexandr Kuramshin <akuramshin@gridgain.com>
Date: Mon, 26 Dec 2016 13:23:28 +0300
Subject: [PATCH 077/446] ignite-4167 Do not log cache key/values

---
 .../apache/ignite/IgniteSystemProperties.java |   5 +
 .../ignite/cache/affinity/AffinityKey.java    |   4 +-
 .../org/apache/ignite/events/CacheEvent.java  |   6 +-
 .../ignite/events/CacheQueryReadEvent.java    |   8 +-
 .../internal/binary/BinaryEnumObjectImpl.java |  10 +-
 .../internal/binary/BinaryMetadata.java       |   5 +-
 .../internal/binary/BinaryObjectExImpl.java   |   8 +-
 .../cache/CacheInvokeDirectResult.java        |   2 +-
 .../processors/cache/CacheInvokeResult.java   |   2 +-
 .../processors/cache/CacheLazyEntry.java      |   4 +-
 .../processors/cache/CacheObjectAdapter.java  |   7 +-
 .../processors/cache/GridCacheAdapter.java    |   5 +-
 .../cache/GridCacheMvccCandidate.java         |   9 +-
 .../processors/cache/GridCacheReturn.java     |   2 +-
 .../distributed/dht/GridDhtCacheAdapter.java  |   2 +-
 .../distributed/near/GridNearLockFuture.java  |   2 +-
 .../cache/query/GridCacheQueryAdapter.java    |   2 +
 .../cache/query/GridCacheQueryManager.java    |  13 +-
 .../cache/query/GridCacheQueryRequest.java    |   2 +
 .../cache/query/GridCacheSqlQuery.java        |   4 +-
 .../continuous/CacheContinuousQueryEvent.java |   8 +-
 .../CacheContinuousQueryManager.java          |   4 +-
 .../store/GridCacheStoreManagerAdapter.java   |  30 +-
 .../store/GridCacheWriteBehindStore.java      |   2 +-
 .../transactions/IgniteTxLocalAdapter.java    |  11 +-
 .../GridCacheVersionConflictContext.java      |   2 +-
 .../closure/GridClosureProcessor.java         |   4 +-
 .../continuous/GridContinuousMessage.java     |   2 +-
 .../datastructures/CollocatedSetItemKey.java  |   2 +-
 .../GridCacheAtomicLongValue.java             |   2 +
 .../GridCacheAtomicSequenceImpl.java          |   2 +
 .../GridCacheAtomicSequenceValue.java         |   2 +
 .../GridCacheCountDownLatchValue.java         |   3 +
 .../datastructures/GridCacheSetItemKey.java   |   2 +-
 .../processors/job/GridJobWorker.java         |   7 +-
 .../odbc/OdbcQueryExecuteRequest.java         |   6 +-
 .../platform/PlatformNativeException.java     |   3 +-
 .../processors/rest/GridRestResponse.java     |   2 +-
 .../util/future/GridFutureAdapter.java        |   2 +-
 .../util/lang/GridMetadataAwareAdapter.java   |   2 +-
 .../util/tostring/GridToStringBuilder.java    | 642 ++++++++++++++++--
 .../util/tostring/GridToStringInclude.java    |  12 +-
 .../tostring/GridToStringThreadLocal.java     |  12 +-
 ...ridCacheBinaryObjectsAbstractSelfTest.java |   7 +-
 .../tostring/GridToStringBuilderSelfTest.java |  33 +-
 45 files changed, 776 insertions(+), 130 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index fe78d889b178f..0da0f4990912a 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -132,6 +132,11 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_QUIET = "IGNITE_QUIET";
 
+    /**
+     * Setting to {@code true} enables writing sensitive information in {@code toString()} output.
+     */
+    public static final String IGNITE_TO_STRING_INCLUDE_SENSITIVE = "IGNITE_TO_STRING_INCLUDE_SENSITIVE";
+
     /**
      * If this property is set to {@code true} (default) and Ignite is launched
      * in verbose mode (see {@link #IGNITE_QUIET}) and no console appenders can be found
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/AffinityKey.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/AffinityKey.java
index c745ed8e5450a..4215b0517f3a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/AffinityKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/AffinityKey.java
@@ -60,12 +60,12 @@ public class AffinityKey<K> implements Externalizable {
     private static final long serialVersionUID = 0L;
 
     /** Key. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private K key;
 
     /** Affinity key. */
     @AffinityKeyMapped
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private Object affKey;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/events/CacheEvent.java b/modules/core/src/main/java/org/apache/ignite/events/CacheEvent.java
index 29aeb3dc683ad..30f4b37b5a704 100644
--- a/modules/core/src/main/java/org/apache/ignite/events/CacheEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/events/CacheEvent.java
@@ -90,7 +90,7 @@ public class CacheEvent extends EventAdapter {
     private int part;
 
     /** Cache entry. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private Object key;
 
     /** Event ID. */
@@ -102,11 +102,11 @@ public class CacheEvent extends EventAdapter {
     private final Object lockId;
 
     /** New value. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private final Object newVal;
 
     /** Old value. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private final Object oldVal;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/events/CacheQueryReadEvent.java b/modules/core/src/main/java/org/apache/ignite/events/CacheQueryReadEvent.java
index 40c5dae3d63d4..f63ed0c29ae42 100644
--- a/modules/core/src/main/java/org/apache/ignite/events/CacheQueryReadEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/events/CacheQueryReadEvent.java
@@ -96,19 +96,19 @@ public class CacheQueryReadEvent<K, V> extends EventAdapter {
     private final String taskName;
 
     /** Key. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private final K key;
 
     /** Value. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private final V val;
 
     /** Old value. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private final V oldVal;
 
     /** Result row. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private final Object row;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
index 69de3f2cdc8f1..9fd6bc18e8e22 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheObjectContext;
 import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
@@ -195,6 +196,9 @@ public BinaryEnumObjectImpl(BinaryContext ctx, byte[] arr) {
 
     /** {@inheritDoc} */
     @Override public String toString() {
+        if (!S.INCLUDE_SENSITIVE)
+            return ord >= 0 ? "BinaryEnum" : "null";
+
         // 1. Try deserializing the object.
         try {
             Object val = deserialize();
@@ -216,12 +220,12 @@ public BinaryEnumObjectImpl(BinaryContext ctx, byte[] arr) {
         }
 
         if (type != null)
-            return type.typeName() + "[ordinal=" + ord  + ']';
+            return S.toString(type.typeName(), "ordinal", ord, true);
         else {
             if (typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID)
-                return "BinaryEnum[clsName=" + clsName + ", ordinal=" + ord + ']';
+                return S.toString("BinaryEnum", "clsName", clsName, true, "ordinal", ord, true);
             else
-                return "BinaryEnum[typeId=" + typeId + ", ordinal=" + ord + ']';
+                return S.toString("BinaryEnum", "typeId", typeId, true, "ordinal", ord, true);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadata.java
index 0911d468dff89..ec92b081904c3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadata.java
@@ -42,16 +42,19 @@ public class BinaryMetadata implements Externalizable {
     private static final long serialVersionUID = 0L;
 
     /** Type ID. */
+    @GridToStringInclude(sensitive = true)
     private int typeId;
 
     /** Type name. */
+    @GridToStringInclude(sensitive = true)
     private String typeName;
 
     /** Recorded object fields. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private Map<String, Integer> fields;
 
     /** Affinity key field name. */
+    @GridToStringInclude(sensitive = true)
     private String affKeyFieldName;
 
     /** Schemas associated with type. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
index e15e7704eff50..5f1e3e9c894bc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.binary.BinaryType;
 import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl;
 import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
@@ -220,8 +221,11 @@ private String toString(BinaryReaderHandles ctx, IdentityHashMap<BinaryObject, I
             meta = null;
         }
 
-        if (meta == null)
-            return BinaryObject.class.getSimpleName() +  " [idHash=" + idHash + ", hash=" + hash + ", typeId=" + typeId() + ']';
+        if (meta == null || !S.INCLUDE_SENSITIVE)
+            return S.toString(S.INCLUDE_SENSITIVE ? BinaryObject.class.getSimpleName() : "BinaryObject",
+                "idHash", idHash, false,
+                "hash", hash, false,
+                "typeId", typeId(), true);
 
         handles.put(this, idHash);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
index 0d519f712d34b..cc453f919a403 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
@@ -45,7 +45,7 @@ public class CacheInvokeDirectResult implements Message {
     private CacheObject res;
 
     /** */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     @GridDirectTransient
     private Exception err;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeResult.java
index 48dabb9ae5c2d..b51c136fa41b7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeResult.java
@@ -36,7 +36,7 @@ public class CacheInvokeResult<T> implements EntryProcessorResult<T>, Externaliz
     private static final long serialVersionUID = 0L;
 
     /** */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private T res;
 
     /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLazyEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLazyEntry.java
index 02cccc742cc99..be6019efc14c3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLazyEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLazyEntry.java
@@ -36,11 +36,11 @@ public class CacheLazyEntry<K, V> extends CacheInterceptorEntry<K, V> {
     protected CacheObject valObj;
 
     /** Key. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     protected K key;
 
     /** Value. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     protected V val;
 
     /** Keep binary flag. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java
index 70f5ea625b6cf..09a55242d442e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java
@@ -24,6 +24,7 @@
 import java.nio.ByteBuffer;
 import org.apache.ignite.internal.GridDirectTransient;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
 import org.apache.ignite.plugin.extensions.communication.MessageWriter;
@@ -36,7 +37,7 @@ public abstract class CacheObjectAdapter implements CacheObject, Externalizable
     private static final long serialVersionUID = 2006765505127197251L;
 
     /** */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     @GridDirectTransient
     protected Object val;
 
@@ -119,6 +120,8 @@ protected boolean needCopy(CacheObjectContext ctx) {
 
     /** {@inheritDoc} */
     public String toString() {
-        return getClass().getSimpleName() + " [val=" + val + ", hasValBytes=" + (valBytes != null) + ']';
+        return S.toString(S.INCLUDE_SENSITIVE ? getClass().getSimpleName() : "CacheObject",
+            "val", val, true,
+            "hasValBytes", valBytes != null, false);
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 4d59d50ace4d3..965c6d12916aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2571,7 +2571,10 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
             }
 
             @Override public String toString() {
-                return "putxAsync [key=" + key + ", val=" + val + ", filter=" + filter + ']';
+                return S.toString("putxAsync",
+                    "key", key, true,
+                    "val", val, true,
+                    "filter", filter, false);
             }
         });
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java
index e9dd455901e44..c60d6c8bd70b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java
@@ -36,7 +36,6 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.SB;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate.Mask.DHT_LOCAL;
@@ -665,10 +664,10 @@ public IgniteTxKey key() {
         GridCacheMvccCandidate next = next();
 
         return S.toString(GridCacheMvccCandidate.class, this,
-            "key", parent == null ? null : parent.key(),
-            "masks", Mask.toString(flags()),
-            "prevVer", (prev == null ? null : prev.version()),
-            "nextVer", (next == null ? null : next.version()));
+            "key", parent == null ? null : parent.key(), true,
+            "masks", Mask.toString(flags()), false,
+            "prevVer", prev == null ? null : prev.version(), false,
+            "nextVer", next == null ? null : next.version(), false);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
index 29e74db5ad31d..02c882c30aa80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
@@ -49,7 +49,7 @@ public class GridCacheReturn implements Externalizable, Message {
     private static final long serialVersionUID = 0L;
 
     /** Value. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     @GridDirectTransient
     private volatile Object v;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 35e62670ea2a9..b2fb7b4b392a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -1166,7 +1166,7 @@ private PartitionEntrySet(int partId) {
 
         /** {@inheritDoc} */
         @Override public String toString() {
-            return S.toString(PartitionEntrySet.class, this, "super", super.toString());
+            return S.toString(PartitionEntrySet.class, this, "super", super.toString(), true);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index 24313791da10b..7c9860290eea2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -1487,7 +1487,7 @@ private class MiniFuture extends GridFutureAdapter<Boolean> {
         private ClusterNode node;
 
         /** Keys. */
-        @GridToStringInclude
+        @GridToStringInclude(sensitive = true)
         private Collection<KeyCacheObject> keys;
 
         /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
index b29e5e7a62ebd..1fe263d607bbd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
@@ -45,6 +45,7 @@
 import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
 import org.apache.ignite.internal.util.GridEmptyCloseableIterator;
 import org.apache.ignite.internal.util.lang.GridCloseableIterator;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
 import org.apache.ignite.internal.util.typedef.T2;
@@ -82,6 +83,7 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
     private final String clsName;
 
     /** */
+    @GridToStringInclude(sensitive = true)
     private final String clause;
 
     /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 1165157784b5e..85c01d9937151 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -1539,16 +1539,17 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                     if (log.isDebugEnabled()) {
                         ClusterNode primaryNode = CU.primaryNode(cctx, key);
 
-                        log.debug("Record [key=" + key +
-                            ", val=" + val +
-                            ", incBackups=" + incBackups +
-                            ", priNode=" + (primaryNode != null ? U.id8(primaryNode.id()) : null) +
-                            ", node=" + U.id8(cctx.localNode().id()) + ']');
+                        log.debug(S.toString("Record",
+                            "key", key, true,
+                            "val", val, true,
+                            "incBackups", incBackups, false,
+                            "priNode", primaryNode != null ? U.id8(primaryNode.id()) : null, false,
+                            "node", U.id8(cctx.localNode().id()), false));
                     }
 
                     if (val == null) {
                         if (log.isDebugEnabled())
-                            log.debug("Unsuitable record value: " + val);
+                            log.debug(S.toString("Unsuitable record value", "val", val, true));
 
                         continue;
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
index 60c466226fcf7..ed876a27ff9dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheDeployable;
 import org.apache.ignite.internal.processors.cache.GridCacheMessage;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -63,6 +64,7 @@ public class GridCacheQueryRequest extends GridCacheMessage implements GridCache
     private boolean fields;
 
     /** */
+    @GridToStringInclude(sensitive = true)
     private String clause;
 
     /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
index bb769c962a2e4..82562702d62d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
@@ -47,11 +47,11 @@ public class GridCacheSqlQuery implements Message, GridCacheQueryMarshallable {
     public static final Object[] EMPTY_PARAMS = {};
 
     /** */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private String qry;
 
     /** */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     @GridDirectTransient
     private Object[] params;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEvent.java
index db70e2ea9d42b..eddf30211ffe0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEvent.java
@@ -99,9 +99,9 @@ public int partitionId() {
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(CacheContinuousQueryEvent.class, this,
-            "evtType", getEventType(),
-            "key", getKey(),
-            "newVal", getValue(),
-            "oldVal", getOldValue());
+            "evtType", getEventType(), false,
+            "key", getKey(), true,
+            "newVal", getValue(), true,
+            "oldVal", getOldValue(), true);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index e2fbf52814161..91c199184a2de 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -1209,11 +1209,11 @@ public static class CacheEntryEventImpl extends CacheQueryEntryEvent {
         private static final long serialVersionUID = 0L;
 
         /** */
-        @GridToStringInclude
+        @GridToStringInclude(sensitive = true)
         private Object key;
 
         /** */
-        @GridToStringInclude
+        @GridToStringInclude(sensitive = true)
         private Object val;
 
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 024375e75c477..11d9816ff1e6d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -314,7 +314,8 @@ private CacheStore cacheStoreWrapper(GridKernalContext ctx,
             Object storeKey = cctx.unwrapBinaryIfNeeded(key, !convertBinary());
 
             if (log.isDebugEnabled())
-                log.debug("Loading value from store for key: " + storeKey);
+                log.debug(S.toString("Loading value from store for key",
+                    "key", storeKey, true));
 
             sessionInit0(tx);
 
@@ -564,8 +565,11 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
             key = cctx.unwrapBinaryIfNeeded(key, !convertBinary());
             val = cctx.unwrapBinaryIfNeeded(val, !convertBinary());
 
-            if (log.isDebugEnabled())
-                log.debug("Storing value in cache store [key=" + key + ", val=" + val + ']');
+            if (log.isDebugEnabled()) {
+                log.debug(S.toString("Storing value in cache store",
+                    "key", key, true,
+                    "val", val, true));
+            }
 
             sessionInit0(tx);
 
@@ -589,8 +593,11 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
                 sessionEnd0(tx, threwEx);
             }
 
-            if (log.isDebugEnabled())
-                log.debug("Stored value in cache store [key=" + key + ", val=" + val + ']');
+            if (log.isDebugEnabled()) {
+                log.debug(S.toString("Stored value in cache store",
+                    "key", key, true,
+                    "val", val, true));
+            }
 
             return true;
         }
@@ -667,7 +674,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
             key = cctx.unwrapBinaryIfNeeded(key, !convertBinary());
 
             if (log.isDebugEnabled())
-                log.debug("Removing value from cache store [key=" + key + ']');
+                log.debug(S.toString("Removing value from cache store", "key", key, true));
 
             sessionInit0(tx);
 
@@ -692,7 +699,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
             }
 
             if (log.isDebugEnabled())
-                log.debug("Removed value from cache store [key=" + key + ']');
+                log.debug(S.toString("Removed value from cache store", "key", key, true));
 
             return true;
         }
@@ -715,7 +722,8 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
             Collection<Object> keys0 = cctx.unwrapBinariesIfNeeded(keys, !convertBinary());
 
             if (log.isDebugEnabled())
-                log.debug("Removing values from cache store [keys=" + keys0 + ']');
+                log.debug(S.toString("Removing values from cache store",
+                    "keys", keys0, true));
 
             sessionInit0(tx);
 
@@ -743,7 +751,8 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
             }
 
             if (log.isDebugEnabled())
-                log.debug("Removed values from cache store [keys=" + keys0 + ']');
+                log.debug(S.toString("Removed values from cache store",
+                    "keys", keys0, true));
 
             return true;
         }
@@ -1261,6 +1270,9 @@ private boolean mapContains(Cache.Entry<?, ?> e) {
 
         /** {@inheritDoc} */
         public String toString() {
+            if (!S.INCLUDE_SENSITIVE)
+                return "[size=" + size() + "]";
+
             Iterator<Cache.Entry<?, ?>> it = iterator();
 
             if (!it.hasNext())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
index 858d9a7498510..7e98793045f93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
@@ -908,7 +908,7 @@ private static class StatefulValue<K, V> extends ReentrantReadWriteLock {
         private static final long serialVersionUID = 0L;
 
         /** Value. */
-        @GridToStringInclude
+        @GridToStringInclude(sensitive = true)
         private Entry<? extends K, ? extends V> val;
 
         /** Store operation. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 6d21dcfb716fd..e2f8438215005 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -83,6 +83,7 @@
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiClosure;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -3331,10 +3332,12 @@ private <K, V> IgniteInternalFuture<GridCacheReturn> removeAllAsync0(
 
         assert keys0 != null;
 
-        if (log.isDebugEnabled()) {
-            log.debug("Called removeAllAsync(...) [tx=" + this + ", keys=" + keys0 + ", implicit=" + implicit +
-                ", retval=" + retval + "]");
-        }
+        if (log.isDebugEnabled())
+            log.debug(S.toString("Called removeAllAsync(...)",
+                "tx", this, false,
+                "keys", keys0, true,
+                "implicit", implicit, false,
+                "retval", retval, false));
 
         try {
             checkValid();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionConflictContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionConflictContext.java
index 3849bf541cef8..fa4020668e81e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionConflictContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionConflictContext.java
@@ -178,7 +178,7 @@ public long expireTime() {
     /** {@inheritDoc} */
     @Override public String toString() {
         return state == State.MERGE ?
-            S.toString(GridCacheVersionConflictContext.class, this, "mergeValue", mergeVal) :
+            S.toString(GridCacheVersionConflictContext.class, this, "mergeValue", mergeVal, true) :
             S.toString(GridCacheVersionConflictContext.class, this);
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index a07dbf8e897c7..20fb6a0d0f485 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -1778,7 +1778,7 @@ private static class C1<T, R> implements ComputeJob, Externalizable, GridNoImpli
         protected IgniteClosure<T, R> job;
 
         /** */
-        @GridToStringInclude
+        @GridToStringInclude(sensitive = true)
         private T arg;
 
         /**
@@ -1843,7 +1843,7 @@ public static class C1V2<T, R> implements ComputeJob, Binarylizable, GridNoImpli
         protected IgniteClosure<T, R> job;
 
         /** */
-        @GridToStringInclude
+        @GridToStringInclude(sensitive = true)
         protected T arg;
 
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java
index 0b629ddb8392a..91918c34e0fb1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java
@@ -48,7 +48,7 @@ public class GridContinuousMessage implements Message {
     private UUID routineId;
 
     /** Optional message data. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     @GridDirectTransient
     private Object data;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/CollocatedSetItemKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/CollocatedSetItemKey.java
index 94cffd45b7add..5f3811464b84c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/CollocatedSetItemKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/CollocatedSetItemKey.java
@@ -30,7 +30,7 @@ public class CollocatedSetItemKey implements SetItemKey {
     private IgniteUuid setId;
 
     /** */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private Object item;
 
     /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicLongValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicLongValue.java
index 42e43b691bd0c..5042672bd12c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicLongValue.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicLongValue.java
@@ -22,6 +22,7 @@
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import org.apache.ignite.internal.processors.cache.GridCacheInternal;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
@@ -32,6 +33,7 @@ public final class GridCacheAtomicLongValue implements GridCacheInternal, Extern
     private static final long serialVersionUID = 0L;
 
     /** Value. */
+    @GridToStringInclude(sensitive = true)
     private long val;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
index 7474f467d0800..4f660b6115525 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -85,6 +86,7 @@ public final class GridCacheAtomicSequenceImpl implements GridCacheAtomicSequenc
     private volatile GridCacheContext ctx;
 
     /** Local value of sequence. */
+    @GridToStringInclude(sensitive = true)
     private long locVal;
 
     /**  Upper bound of local counter. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceValue.java
index dd1a1d5342a93..ee540d602d6e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceValue.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceValue.java
@@ -22,6 +22,7 @@
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import org.apache.ignite.internal.processors.cache.GridCacheInternal;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
@@ -32,6 +33,7 @@ public final class GridCacheAtomicSequenceValue implements GridCacheInternal, Ex
     private static final long serialVersionUID = 0L;
 
     /** Counter. */
+    @GridToStringInclude(sensitive = true)
     private long val;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheCountDownLatchValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheCountDownLatchValue.java
index 17a11af72965c..ec996ffc526b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheCountDownLatchValue.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheCountDownLatchValue.java
@@ -22,6 +22,7 @@
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import org.apache.ignite.internal.processors.cache.GridCacheInternal;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
@@ -32,9 +33,11 @@ public final class GridCacheCountDownLatchValue implements GridCacheInternal, Ex
     private static final long serialVersionUID = 0L;
 
     /** Count. */
+    @GridToStringInclude(sensitive = true)
     private int cnt;
 
     /** Initial count. */
+    @GridToStringInclude(sensitive = true)
     private int initCnt;
 
     /** Auto delete flag. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetItemKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetItemKey.java
index 8b47b3debafc9..428089125b273 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetItemKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetItemKey.java
@@ -37,7 +37,7 @@ public class GridCacheSetItemKey implements SetItemKey, Externalizable {
     private IgniteUuid setId;
 
     /** */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private Object item;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index 9bee8490fca4f..6a00d96a0abc2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -566,8 +566,11 @@ private void execute0(boolean skipNtf) {
                         }
                     });
 
-                    if (log.isDebugEnabled())
-                        log.debug("Job execution has successfully finished [job=" + job + ", res=" + res + ']');
+                    if (log.isDebugEnabled()) {
+                        log.debug(S.toString("Job execution has successfully finished",
+                            "job", job, false,
+                            "res", res, true));
+                    }
                 }
             }
             catch (IgniteException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
index 1bcd41f66cc29..c0d1c601c92b7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
@@ -18,11 +18,10 @@
 package org.apache.ignite.internal.processors.odbc;
 
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.Arrays;
-
 /**
  * ODBC query execute request.
  */
@@ -31,6 +30,7 @@ public class OdbcQueryExecuteRequest extends OdbcRequest {
     private final String cacheName;
 
     /** Sql query. */
+    @GridToStringInclude(sensitive = true)
     private final String sqlQry;
 
     /** Sql query arguments. */
@@ -73,6 +73,6 @@ public Object[] arguments() {
 
     /** {@inheritDoc} */
     @Override public String toString() {
-        return S.toString(OdbcQueryExecuteRequest.class, this, "args", Arrays.toString(args));
+        return S.toString(OdbcQueryExecuteRequest.class, this, "args", args, true);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformNativeException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformNativeException.java
index a99664a93764f..5c77cf2a4afbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformNativeException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformNativeException.java
@@ -72,6 +72,7 @@ public Object cause() {
 
     /** {@inheritDoc} */
     @Override public String toString() {
-        return S.toString(PlatformNativeException.class, this, "cause", cause);
+        return S.toString(PlatformNativeException.class, this,
+            "cause", S.INCLUDE_SENSITIVE ? cause : (cause == null ? "null" : cause.getClass().getSimpleName()));
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java
index ecbc6c8c6d4a3..18d1ddf6fc0d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java
@@ -59,7 +59,7 @@ public class GridRestResponse implements Externalizable {
     private String err;
 
     /** Response object. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private Object obj;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
index 2cd534e0905a7..08c39c04bf993 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
@@ -61,7 +61,7 @@ public class GridFutureAdapter<R> extends AbstractQueuedSynchronizer implements
     private byte resFlag;
 
     /** Result. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private Object res;
 
     /** Future start time. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridMetadataAwareAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridMetadataAwareAdapter.java
index decc24438fe1e..1983ea31a6ee9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridMetadataAwareAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridMetadataAwareAdapter.java
@@ -67,7 +67,7 @@ public int key() {
     }
 
     /** Attributes. */
-    @GridToStringInclude
+    @GridToStringInclude(sensitive = true)
     private Object[] data = null;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
index b29d7cddc7366..333f95e64ae58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
@@ -35,6 +35,7 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -85,6 +86,9 @@ public class GridToStringBuilder {
     /** Maximum number of collection (map) entries to print. */
     public static final int MAX_COL_SIZE = 100;
 
+    /** {@link IgniteSystemProperties#IGNITE_TO_STRING_INCLUDE_SENSITIVE} */
+    public static final boolean INCLUDE_SENSITIVE = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE, false);
+
     /** */
     private static ThreadLocal<Queue<GridToStringThreadLocal>> threadCache = new ThreadLocal<Queue<GridToStringThreadLocal>>() {
         @Override protected Queue<GridToStringThreadLocal> initialValue() {
@@ -114,8 +118,45 @@ public class GridToStringBuilder {
      * @param val4 Additional parameter value.
      * @return String presentation of the given object.
      */
-    public static <T> String toString(Class<T> cls, T obj, String name0, Object val0, String name1, Object val1,
-        String name2, Object val2, String name3, Object val3, String name4, Object val4) {
+    public static <T> String toString(Class<T> cls, T obj,
+        String name0, Object val0,
+        String name1, Object val1,
+        String name2, Object val2,
+        String name3, Object val3,
+        String name4, Object val4) {
+        return toString(cls,
+            obj,
+            name0, val0, false,
+            name1, val1, false,
+            name2, val2, false,
+            name3, val3, false,
+            name4, val4, false);
+    }
+
+    /**
+     * Produces auto-generated output of string presentation for given object and its declaration class.
+     *
+     * @param <T> Type of the object.
+     * @param cls Declaration class of the object. Note that this should not be a runtime class.
+     * @param obj Object to get a string presentation for.
+     * @param name0 Additional parameter name.
+     * @param val0 Additional parameter value.
+     * @param name1 Additional parameter name.
+     * @param val1 Additional parameter value.
+     * @param name2 Additional parameter name.
+     * @param val2 Additional parameter value.
+     * @param name3 Additional parameter name.
+     * @param val3 Additional parameter value.
+     * @param name4 Additional parameter name.
+     * @param val4 Additional parameter value.
+     * @return String presentation of the given object.
+     */
+    public static <T> String toString(Class<T> cls, T obj,
+        String name0, Object val0, boolean sens0,
+        String name1, Object val1, boolean sens1,
+        String name2, Object val2, boolean sens2,
+        String name3, Object val3, boolean sens3,
+        String name4, Object val4, boolean sens4) {
         assert cls != null;
         assert obj != null;
         assert name0 != null;
@@ -135,20 +176,26 @@ public static <T> String toString(Class<T> cls, T obj, String name0, Object val0
 
         Object[] addNames = tmp.getAdditionalNames();
         Object[] addVals = tmp.getAdditionalValues();
+        boolean[] addSens = tmp.getAdditionalSensitives();
 
         addNames[0] = name0;
         addVals[0] = val0;
+        addSens[0] = sens0;
         addNames[1] = name1;
         addVals[1] = val1;
+        addSens[1] = sens1;
         addNames[2] = name2;
         addVals[2] = val2;
+        addSens[2] = sens2;
         addNames[3] = name3;
         addVals[3] = val3;
+        addSens[3] = sens3;
         addNames[4] = name4;
         addVals[4] = val4;
+        addSens[4] = sens4;
 
         try {
-            return toStringImpl(cls, tmp.getStringBuilder(), obj, addNames, addVals, 5);
+            return toStringImpl(cls, tmp.getStringBuilder(), obj, addNames, addVals, addSens, 5);
         }
         finally {
             queue.offer(tmp);
@@ -171,8 +218,43 @@ public static <T> String toString(Class<T> cls, T obj, String name0, Object val0
      * @param val3 Additional parameter value.
      * @return String presentation of the given object.
      */
-    public static <T> String toString(Class<T> cls, T obj, String name0, Object val0, String name1, Object val1,
-        String name2, Object val2, String name3, Object val3) {
+    public static <T> String toString(Class<T> cls, T obj,
+        String name0, Object val0,
+        String name1, Object val1,
+        String name2, Object val2,
+        String name3, Object val3) {
+        return toString(cls, obj,
+            name0, val0, false,
+            name1, val1, false,
+            name2, val2, false,
+            name3, val3, false);
+    }
+
+    /**
+     * Produces auto-generated output of string presentation for given object and its declaration class.
+     *
+     * @param <T> Type of the object.
+     * @param cls Declaration class of the object. Note that this should not be a runtime class.
+     * @param obj Object to get a string presentation for.
+     * @param name0 Additional parameter name.
+     * @param val0 Additional parameter value.
+     * @param sens0 Property sensitive flag.
+     * @param name1 Additional parameter name.
+     * @param val1 Additional parameter value.
+     * @param sens1 Property sensitive flag.
+     * @param name2 Additional parameter name.
+     * @param val2 Additional parameter value.
+     * @param sens2 Property sensitive flag.
+     * @param name3 Additional parameter name.
+     * @param val3 Additional parameter value.
+     * @param sens3 Property sensitive flag.
+     * @return String presentation of the given object.
+     */
+    public static <T> String toString(Class<T> cls, T obj,
+        String name0, Object val0, boolean sens0,
+        String name1, Object val1, boolean sens1,
+        String name2, Object val2, boolean sens2,
+        String name3, Object val3, boolean sens3) {
         assert cls != null;
         assert obj != null;
         assert name0 != null;
@@ -191,18 +273,23 @@ public static <T> String toString(Class<T> cls, T obj, String name0, Object val0
 
         Object[] addNames = tmp.getAdditionalNames();
         Object[] addVals = tmp.getAdditionalValues();
+        boolean[] addSens = tmp.getAdditionalSensitives();
 
         addNames[0] = name0;
         addVals[0] = val0;
+        addSens[0] = sens0;
         addNames[1] = name1;
         addVals[1] = val1;
+        addSens[1] = sens1;
         addNames[2] = name2;
         addVals[2] = val2;
+        addSens[2] = sens2;
         addNames[3] = name3;
         addVals[3] = val3;
+        addSens[3] = sens3;
 
         try {
-            return toStringImpl(cls, tmp.getStringBuilder(), obj, addNames, addVals, 4);
+            return toStringImpl(cls, tmp.getStringBuilder(), obj, addNames, addVals, addSens, 4);
         }
         finally {
             queue.offer(tmp);
@@ -223,8 +310,38 @@ public static <T> String toString(Class<T> cls, T obj, String name0, Object val0
      * @param val2 Additional parameter value.
      * @return String presentation of the given object.
      */
-    public static <T> String toString(Class<T> cls, T obj, String name0, Object val0, String name1, Object val1,
+    public static <T> String toString(Class<T> cls, T obj,
+        String name0, Object val0,
+        String name1, Object val1,
         String name2, Object val2) {
+        return toString(cls,
+            obj,
+            name0, val0, false,
+            name1, val1, false,
+            name2, val2, false);
+    }
+
+    /**
+     * Produces auto-generated output of string presentation for given object and its declaration class.
+     *
+     * @param <T> Type of the object.
+     * @param cls Declaration class of the object. Note that this should not be a runtime class.
+     * @param obj Object to get a string presentation for.
+     * @param name0 Additional parameter name.
+     * @param val0 Additional parameter value.
+     * @param sens0 Property sensitive flag.
+     * @param name1 Additional parameter name.
+     * @param val1 Additional parameter value.
+     * @param sens1 Property sensitive flag.
+     * @param name2 Additional parameter name.
+     * @param val2 Additional parameter value.
+     * @param sens2 Property sensitive flag.
+     * @return String presentation of the given object.
+     */
+    public static <T> String toString(Class<T> cls, T obj,
+        String name0, Object val0, boolean sens0,
+        String name1, Object val1, boolean sens1,
+        String name2, Object val2, boolean sens2) {
         assert cls != null;
         assert obj != null;
         assert name0 != null;
@@ -242,16 +359,20 @@ public static <T> String toString(Class<T> cls, T obj, String name0, Object val0
 
         Object[] addNames = tmp.getAdditionalNames();
         Object[] addVals = tmp.getAdditionalValues();
+        boolean[] addSens = tmp.getAdditionalSensitives();
 
         addNames[0] = name0;
         addVals[0] = val0;
+        addSens[0] = sens0;
         addNames[1] = name1;
         addVals[1] = val1;
+        addSens[1] = sens1;
         addNames[2] = name2;
         addVals[2] = val2;
+        addSens[2] = sens2;
 
         try {
-            return toStringImpl(cls, tmp.getStringBuilder(), obj, addNames, addVals, 3);
+            return toStringImpl(cls, tmp.getStringBuilder(), obj, addNames, addVals, addSens, 3);
         }
         finally {
             queue.offer(tmp);
@@ -270,7 +391,29 @@ public static <T> String toString(Class<T> cls, T obj, String name0, Object val0
      * @param val1 Additional parameter value.
      * @return String presentation of the given object.
      */
-    public static <T> String toString(Class<T> cls, T obj, String name0, Object val0, String name1, Object val1) {
+    public static <T> String toString(Class<T> cls, T obj,
+        String name0, Object val0,
+        String name1, Object val1) {
+        return toString(cls, obj, name0, val0, false, name1, val1, false);
+    }
+
+    /**
+     * Produces auto-generated output of string presentation for given object and its declaration class.
+     *
+     * @param <T> Type of the object.
+     * @param cls Declaration class of the object. Note that this should not be a runtime class.
+     * @param obj Object to get a string presentation for.
+     * @param name0 Additional parameter name.
+     * @param val0 Additional parameter value.
+     * @param sens0 Property sensitive flag.
+     * @param name1 Additional parameter name.
+     * @param val1 Additional parameter value.
+     * @param sens1 Property sensitive flag.
+     * @return String presentation of the given object.
+     */
+    public static <T> String toString(Class<T> cls, T obj,
+        String name0, Object val0, boolean sens0,
+        String name1, Object val1, boolean sens1) {
         assert cls != null;
         assert obj != null;
         assert name0 != null;
@@ -287,14 +430,17 @@ public static <T> String toString(Class<T> cls, T obj, String name0, Object val0
 
         Object[] addNames = tmp.getAdditionalNames();
         Object[] addVals = tmp.getAdditionalValues();
+        boolean[] addSens = tmp.getAdditionalSensitives();
 
         addNames[0] = name0;
         addVals[0] = val0;
+        addSens[0] = sens0;
         addNames[1] = name1;
         addVals[1] = val1;
+        addSens[1] = sens1;
 
         try {
-            return toStringImpl(cls, tmp.getStringBuilder(), obj, addNames, addVals, 2);
+            return toStringImpl(cls, tmp.getStringBuilder(), obj, addNames, addVals, addSens, 2);
         }
         finally {
             queue.offer(tmp);
@@ -312,6 +458,21 @@ public static <T> String toString(Class<T> cls, T obj, String name0, Object val0
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj, String name, @Nullable Object val) {
+        return toString(cls, obj, name, val, false);
+    }
+
+    /**
+     * Produces auto-generated output of string presentation for given object and its declaration class.
+     *
+     * @param <T> Type of the object.
+     * @param cls Declaration class of the object. Note that this should not be a runtime class.
+     * @param obj Object to get a string presentation for.
+     * @param name Additional parameter name.
+     * @param val Additional parameter value.
+     * @param sens Property sensitive flag.
+     * @return String presentation of the given object.
+     */
+    public static <T> String toString(Class<T> cls, T obj, String name, @Nullable Object val, boolean sens) {
         assert cls != null;
         assert obj != null;
         assert name != null;
@@ -327,12 +488,14 @@ public static <T> String toString(Class<T> cls, T obj, String name, @Nullable Ob
 
         Object[] addNames = tmp.getAdditionalNames();
         Object[] addVals = tmp.getAdditionalValues();
+        boolean[] addSens = tmp.getAdditionalSensitives();
 
         addNames[0] = name;
         addVals[0] = val;
+        addSens[0] = sens;
 
         try {
-            return toStringImpl(cls, tmp.getStringBuilder(), obj, addNames, addVals, 1);
+            return toStringImpl(cls, tmp.getStringBuilder(), obj, addNames, addVals, addSens, 1);
         }
         finally {
             queue.offer(tmp);
@@ -362,7 +525,7 @@ public static <T> String toString(Class<T> cls, T obj) {
 
         try {
             return toStringImpl(cls, tmp.getStringBuilder(), obj, tmp.getAdditionalNames(),
-                tmp.getAdditionalValues(), 0);
+                tmp.getAdditionalValues(), null, 0);
         }
         finally {
             queue.offer(tmp);
@@ -390,12 +553,16 @@ public static <T> String toString(Class<T> cls, T obj, String parent) {
      * @param obj Object for which to get string presentation.
      * @param addNames Names of additional values to be included.
      * @param addVals Additional values to be included.
+     * @param addSens Sensitive flag of values or {@code null} if all values are not sensitive.
      * @param addLen How many additional values will be included.
      * @return String presentation of the given object.
      * @param <T> Type of object.
      */
     @SuppressWarnings({"unchecked"})
-    private static <T> String toStringImpl(Class<T> cls, SB buf, T obj, Object[] addNames, Object[] addVals,
+    private static <T> String toStringImpl(Class<T> cls, SB buf, T obj,
+        Object[] addNames,
+        Object[] addVals,
+        @Nullable boolean[] addSens,
         int addLen) {
         assert cls != null;
         assert buf != null;
@@ -430,26 +597,10 @@ private static <T> String toStringImpl(Class<T> cls, SB buf, T obj, Object[] add
 
                 buf.a(name).a('=');
 
-                if (field.getType().isArray()) {
-                    if (field.getType().equals(byte[].class))
-                        buf.a(Arrays.toString((byte[])field.get(obj)));
-                    else if (field.getType().equals(boolean[].class))
-                        buf.a(Arrays.toString((boolean[])field.get(obj)));
-                    else if (field.getType().equals(short[].class))
-                        buf.a(Arrays.toString((short[])field.get(obj)));
-                    else if (field.getType().equals(int[].class))
-                        buf.a(Arrays.toString((int[])field.get(obj)));
-                    else if (field.getType().equals(long[].class))
-                        buf.a(Arrays.toString((long[])field.get(obj)));
-                    else if (field.getType().equals(float[].class))
-                        buf.a(Arrays.toString((float[])field.get(obj)));
-                    else if (field.getType().equals(double[].class))
-                        buf.a(Arrays.toString((double[])field.get(obj)));
-                    else if (field.getType().equals(char[].class))
-                        buf.a(Arrays.toString((char[])field.get(obj)));
-                    else
-                        buf.a(Arrays.toString((Object[])field.get(obj)));
-                }
+                Class<?> fieldType = field.getType();
+
+                if (fieldType.isArray())
+                    buf.a(arrayToString(fieldType, field.get(obj)));
                 else {
                     Object val = field.get(obj);
 
@@ -475,15 +626,7 @@ else if (val instanceof Map && ((Map)val).size() > MAX_COL_SIZE) {
                 }
             }
 
-            if (addLen > 0)
-                for (int i = 0; i < addLen; i++) {
-                    if (!first)
-                       buf.a(", ");
-                    else
-                        first = false;
-
-                    buf.a(addNames[i]).a('=').a(addVals[i]);
-                }
+            appendVals(buf, first, addNames, addVals, addSens, addLen);
 
             buf.a(']');
 
@@ -507,6 +650,402 @@ else if (val instanceof Map && ((Map)val).size() > MAX_COL_SIZE) {
         }
     }
 
+    /**
+     * @param arrType Type of the array.
+     * @param arr Array object.
+     * @return String representation of an array.
+     */
+    private static String arrayToString(Class arrType, Object arr) {
+        if (arrType.equals(byte[].class))
+            return Arrays.toString((byte[])arr);
+        if (arrType.equals(boolean[].class))
+            return Arrays.toString((boolean[])arr);
+        if (arrType.equals(short[].class))
+            return Arrays.toString((short[])arr);
+        if (arrType.equals(int[].class))
+            return Arrays.toString((int[])arr);
+        if (arrType.equals(long[].class))
+            return Arrays.toString((long[])arr);
+        if (arrType.equals(float[].class))
+            return Arrays.toString((float[])arr);
+        if (arrType.equals(double[].class))
+            return Arrays.toString((double[])arr);
+        if (arrType.equals(char[].class))
+            return Arrays.toString((char[])arr);
+
+        return Arrays.toString((Object[])arr);
+    }
+
+    /**
+     * Produces uniformed output of string with context properties
+     *
+     * @param str Output prefix or {@code null} if empty.
+     * @param name Property name.
+     * @param val Property value.
+     * @return String presentation.
+     */
+    public static String toString(String str, String name, @Nullable Object val) {
+        return toString(str, name, val, false);
+    }
+
+    /**
+     * Produces uniformed output of string with context properties
+     *
+     * @param str Output prefix or {@code null} if empty.
+     * @param name Property name.
+     * @param val Property value.
+     * @param sens Property sensitive flag.
+     * @return String presentation.
+     */
+    public static String toString(String str, String name, @Nullable Object val, boolean sens) {
+        assert name != null;
+
+        Queue<GridToStringThreadLocal> queue = threadCache.get();
+
+        assert queue != null;
+
+        // Since string() methods can be chain-called from the same thread we
+        // have to keep a list of thread-local objects and remove/add them
+        // in each string() apply.
+        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
+
+        Object[] propNames = tmp.getAdditionalNames();
+        Object[] propVals = tmp.getAdditionalValues();
+        boolean[] propSens = tmp.getAdditionalSensitives();
+
+        propNames[0] = name;
+        propVals[0] = val;
+        propSens[0] = sens;
+
+        try {
+            return toStringImpl(str, tmp.getStringBuilder(), propNames, propVals, propSens, 1);
+        }
+        finally {
+            queue.offer(tmp);
+        }
+    }
+
+    /**
+     * Produces uniformed output of string with context properties
+     *
+     * @param str Output prefix or {@code null} if empty.
+     * @param name0 Property name.
+     * @param val0 Property value.
+     * @param name1 Property name.
+     * @param val1 Property value.
+     * @return String presentation.
+     */
+    public static String toString(String str, String name0, @Nullable Object val0, String name1,
+        @Nullable Object val1) {
+        return toString(str, name0, val0, false, name1, val1, false);
+    }
+
+    /**
+     * Produces uniformed output of string with context properties
+     *
+     * @param str Output prefix or {@code null} if empty.
+     * @param name0 Property name.
+     * @param val0 Property value.
+     * @param sens0 Property sensitive flag.
+     * @param name1 Property name.
+     * @param val1 Property value.
+     * @param sens1 Property sensitive flag.
+     * @return String presentation.
+     */
+    public static String toString(String str,
+        String name0, @Nullable Object val0, boolean sens0,
+        String name1, @Nullable Object val1, boolean sens1) {
+        assert name0 != null;
+        assert name1 != null;
+
+        Queue<GridToStringThreadLocal> queue = threadCache.get();
+
+        assert queue != null;
+
+        // Since string() methods can be chain-called from the same thread we
+        // have to keep a list of thread-local objects and remove/add them
+        // in each string() apply.
+        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
+
+        Object[] propNames = tmp.getAdditionalNames();
+        Object[] propVals = tmp.getAdditionalValues();
+        boolean[] propSens = tmp.getAdditionalSensitives();
+
+        propNames[0] = name0;
+        propVals[0] = val0;
+        propSens[0] = sens0;
+        propNames[1] = name1;
+        propVals[1] = val1;
+        propSens[1] = sens1;
+
+        try {
+            return toStringImpl(str, tmp.getStringBuilder(), propNames, propVals, propSens, 2);
+        }
+        finally {
+            queue.offer(tmp);
+        }
+    }
+
+    /**
+     * Produces uniformed output of string with context properties
+     *
+     * @param str Output prefix or {@code null} if empty.
+     * @param name0 Property name.
+     * @param val0 Property value.
+     * @param sens0 Property sensitive flag.
+     * @param name1 Property name.
+     * @param val1 Property value.
+     * @param sens1 Property sensitive flag.
+     * @param name2 Property name.
+     * @param val2 Property value.
+     * @param sens2 Property sensitive flag.
+     * @return String presentation.
+     */
+    public static String toString(String str,
+        String name0, @Nullable Object val0, boolean sens0,
+        String name1, @Nullable Object val1, boolean sens1,
+        String name2, @Nullable Object val2, boolean sens2) {
+        assert name0 != null;
+        assert name1 != null;
+        assert name2 != null;
+
+        Queue<GridToStringThreadLocal> queue = threadCache.get();
+
+        assert queue != null;
+
+        // Since string() methods can be chain-called from the same thread we
+        // have to keep a list of thread-local objects and remove/add them
+        // in each string() apply.
+        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
+
+        Object[] propNames = tmp.getAdditionalNames();
+        Object[] propVals = tmp.getAdditionalValues();
+        boolean[] propSens = tmp.getAdditionalSensitives();
+
+        propNames[0] = name0;
+        propVals[0] = val0;
+        propSens[0] = sens0;
+        propNames[1] = name1;
+        propVals[1] = val1;
+        propSens[1] = sens1;
+        propNames[2] = name2;
+        propVals[2] = val2;
+        propSens[2] = sens2;
+
+        try {
+            return toStringImpl(str, tmp.getStringBuilder(), propNames, propVals, propSens, 3);
+        }
+        finally {
+            queue.offer(tmp);
+        }
+    }
+
+    /**
+     * Produces uniformed output of string with context properties
+     *
+     * @param str Output prefix or {@code null} if empty.
+     * @param name0 Property name.
+     * @param val0 Property value.
+     * @param sens0 Property sensitive flag.
+     * @param name1 Property name.
+     * @param val1 Property value.
+     * @param sens1 Property sensitive flag.
+     * @param name2 Property name.
+     * @param val2 Property value.
+     * @param sens2 Property sensitive flag.
+     * @param name3 Property name.
+     * @param val3 Property value.
+     * @param sens3 Property sensitive flag.
+     * @return String presentation.
+     */
+    public static String toString(String str,
+        String name0, @Nullable Object val0, boolean sens0,
+        String name1, @Nullable Object val1, boolean sens1,
+        String name2, @Nullable Object val2, boolean sens2,
+        String name3, @Nullable Object val3, boolean sens3) {
+        assert name0 != null;
+        assert name1 != null;
+        assert name2 != null;
+        assert name3 != null;
+
+        Queue<GridToStringThreadLocal> queue = threadCache.get();
+
+        assert queue != null;
+
+        // Since string() methods can be chain-called from the same thread we
+        // have to keep a list of thread-local objects and remove/add them
+        // in each string() apply.
+        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
+
+        Object[] propNames = tmp.getAdditionalNames();
+        Object[] propVals = tmp.getAdditionalValues();
+        boolean[] propSens = tmp.getAdditionalSensitives();
+
+        propNames[0] = name0;
+        propVals[0] = val0;
+        propSens[0] = sens0;
+        propNames[1] = name1;
+        propVals[1] = val1;
+        propSens[1] = sens1;
+        propNames[2] = name2;
+        propVals[2] = val2;
+        propSens[2] = sens2;
+        propNames[3] = name3;
+        propVals[3] = val3;
+        propSens[3] = sens3;
+
+        try {
+            return toStringImpl(str, tmp.getStringBuilder(), propNames, propVals, propSens, 4);
+        }
+        finally {
+            queue.offer(tmp);
+        }
+    }
+
+    /**
+     * Produces uniformed output of string with context properties
+     *
+     * @param str Output prefix or {@code null} if empty.
+     * @param name0 Property name.
+     * @param val0 Property value.
+     * @param sens0 Property sensitive flag.
+     * @param name1 Property name.
+     * @param val1 Property value.
+     * @param sens1 Property sensitive flag.
+     * @param name2 Property name.
+     * @param val2 Property value.
+     * @param sens2 Property sensitive flag.
+     * @param name3 Property name.
+     * @param val3 Property value.
+     * @param sens3 Property sensitive flag.
+     * @param name4 Property name.
+     * @param val4 Property value.
+     * @param sens4 Property sensitive flag.
+     * @return String presentation.
+     */
+    public static String toString(String str,
+        String name0, @Nullable Object val0, boolean sens0,
+        String name1, @Nullable Object val1, boolean sens1,
+        String name2, @Nullable Object val2, boolean sens2,
+        String name3, @Nullable Object val3, boolean sens3,
+        String name4, @Nullable Object val4, boolean sens4) {
+        assert name0 != null;
+        assert name1 != null;
+        assert name2 != null;
+        assert name3 != null;
+        assert name4 != null;
+
+        Queue<GridToStringThreadLocal> queue = threadCache.get();
+
+        assert queue != null;
+
+        // Since string() methods can be chain-called from the same thread we
+        // have to keep a list of thread-local objects and remove/add them
+        // in each string() apply.
+        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
+
+        Object[] propNames = tmp.getAdditionalNames();
+        Object[] propVals = tmp.getAdditionalValues();
+        boolean[] propSens = tmp.getAdditionalSensitives();
+
+        propNames[0] = name0;
+        propVals[0] = val0;
+        propSens[0] = sens0;
+        propNames[1] = name1;
+        propVals[1] = val1;
+        propSens[1] = sens1;
+        propNames[2] = name2;
+        propVals[2] = val2;
+        propSens[2] = sens2;
+        propNames[3] = name3;
+        propVals[3] = val3;
+        propSens[3] = sens3;
+        propNames[4] = name4;
+        propVals[4] = val4;
+        propSens[4] = sens4;
+
+        try {
+            return toStringImpl(str, tmp.getStringBuilder(), propNames, propVals, propSens, 5);
+        }
+        finally {
+            queue.offer(tmp);
+        }
+    }
+
+    /**
+     * Creates an uniformed string presentation for the binary-like object.
+     *
+     * @param str Output prefix or {@code null} if empty.
+     * @param buf String builder buffer.
+     * @param propNames Names of object properties.
+     * @param propVals Property values.
+     * @param propSens Sensitive flag of values or {@code null} if all values is not sensitive.
+     * @param propCnt Properties count.
+     * @return String presentation of the object.
+     */
+    private static String toStringImpl(String str, SB buf, Object[] propNames, Object[] propVals,
+        boolean[] propSens, int propCnt) {
+
+        buf.setLength(0);
+
+        if (str != null)
+            buf.a(str).a(" ");
+
+        buf.a("[");
+
+        appendVals(buf, true, propNames, propVals, propSens, propCnt);
+
+        buf.a(']');
+
+        return buf.toString();
+    }
+
+    /**
+     * Append additional values to the buffer.
+     *
+     * @param buf Buffer.
+     * @param first First value flag.
+     * @param addNames Names of additional values to be included.
+     * @param addVals Additional values to be included.
+     * @param addSens Sensitive flag of values or {@code null} if all values are not sensitive.
+     * @param addLen How many additional values will be included.
+     */
+    private static void appendVals(SB buf,
+        boolean first,
+        Object[] addNames,
+        Object[] addVals,
+        boolean[] addSens,
+        int addLen)
+    {
+        if (addLen > 0) {
+            for (int i = 0; i < addLen; i++) {
+                Object addVal = addVals[i];
+
+                if (addVal != null) {
+                    if (addSens != null && addSens[i] && !INCLUDE_SENSITIVE)
+                        continue;
+
+                    GridToStringInclude incAnn = addVal.getClass().getAnnotation(GridToStringInclude.class);
+
+                    if (incAnn != null && incAnn.sensitive() && !INCLUDE_SENSITIVE)
+                        continue;
+
+                    Class<?> cls = addVal.getClass();
+
+                    if (cls.isArray())
+                        addVal = arrayToString(cls, addVal);
+                }
+
+                if (!first)
+                    buf.a(", ");
+                else
+                    first = false;
+
+                buf.a(addNames[i]).a('=').a(addVal);
+            }
+        }
+    }
+
     /**
      * @param cls Class.
      * @param <T> Type of the object.
@@ -518,7 +1057,7 @@ private static <T> GridToStringClassDescriptor getClassDescriptor(Class<T> cls)
 
         String key = cls.getName() + System.identityHashCode(cls.getClassLoader());
 
-        GridToStringClassDescriptor cd = null;
+        GridToStringClassDescriptor cd;
 
         rwLock.readLock().lock();
 
@@ -537,9 +1076,15 @@ private static <T> GridToStringClassDescriptor getClassDescriptor(Class<T> cls)
 
                 Class<?> type = f.getType();
 
-                if (f.isAnnotationPresent(GridToStringInclude.class) ||
-                    type.isAnnotationPresent(GridToStringInclude.class))
-                    add = true;
+                final GridToStringInclude incFld = f.getAnnotation(GridToStringInclude.class);
+                final GridToStringInclude incType = type.getAnnotation(GridToStringInclude.class);
+
+                if (incFld != null || incType != null) {
+                    // Information is not sensitive when both the field and the field type are not sensitive.
+                    // When @GridToStringInclude is not present then the flag is false by default for that attribute.
+                    final boolean notSens = (incFld == null || !incFld.sensitive()) && (incType == null || !incType.sensitive());
+                    add = notSens || INCLUDE_SENSITIVE;
+                }
                 else if (!f.isAnnotationPresent(GridToStringExclude.class) &&
                     !f.getType().isAnnotationPresent(GridToStringExclude.class)) {
                     if (
@@ -573,8 +1118,9 @@ else if (!f.isAnnotationPresent(GridToStringExclude.class) &&
                     GridToStringFieldDescriptor fd = new GridToStringFieldDescriptor(f.getName());
 
                     // Get order, if any.
-                    if (f.isAnnotationPresent(GridToStringOrder.class))
-                        fd.setOrder(f.getAnnotation(GridToStringOrder.class).value());
+                    final GridToStringOrder annOrder = f.getAnnotation(GridToStringOrder.class);
+                    if (annOrder != null)
+                        fd.setOrder(annOrder.value());
 
                     cd.addField(fd);
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringInclude.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringInclude.java
index ec502a73d62b7..5ea1fe68ca62e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringInclude.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringInclude.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.util.tostring;
 
+import org.apache.ignite.IgniteSystemProperties;
+
 import java.lang.annotation.Documented;
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
@@ -32,5 +34,13 @@
 @Retention(RetentionPolicy.RUNTIME)
 @Target({ElementType.FIELD, ElementType.TYPE})
 public @interface GridToStringInclude {
-    // No-op.
+    /**
+     * A flag indicating a sensitive information stored in the field or fields of the class.<br/>
+     * Such information will be included in {@code toString()} output ONLY when the system property
+     * {@link IgniteSystemProperties#IGNITE_TO_STRING_INCLUDE_SENSITIVE IGNITE_TO_STRING_INCLUDE_SENSITIVE}
+     * is set to {@code true}.
+     *
+     * @return Attribute value.
+     */
+    boolean sensitive() default false;
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringThreadLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringThreadLocal.java
index ab9145298adf9..33fc6a0e8e7df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringThreadLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringThreadLocal.java
@@ -20,7 +20,7 @@
 import org.apache.ignite.internal.util.typedef.internal.SB;
 
 /**
- * Helper wrapper containing StringBuilder and additional values. Stored as a thread-lcal variable.
+ * Helper wrapper containing StringBuilder and additional values. Stored as a thread-local variable.
  */
 class GridToStringThreadLocal {
     /** */
@@ -32,6 +32,9 @@ class GridToStringThreadLocal {
     /** */
     private Object[] addVals = new Object[5];
 
+    /** */
+    private boolean[] addSens = new boolean[5];
+
     /**
      * @return String builder.
      */
@@ -52,4 +55,11 @@ Object[] getAdditionalNames() {
     Object[] getAdditionalValues() {
         return addVals;
     }
+
+    /**
+     * @return Additional values.
+     */
+    boolean[] getAdditionalSensitives() {
+        return addSens;
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
index 150c2454762b5..bf7278236d126 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
@@ -60,6 +60,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
 import org.apache.ignite.internal.util.typedef.P2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -224,7 +225,11 @@ public void testCircularReference() throws Exception {
 
         String typeName = nameMapper.typeName(TestReferenceObject.class.getName());
 
-        assertTrue("Unexpected toString: " + str, str.startsWith(typeName) && str.contains("obj=" + typeName + " ["));
+        assertTrue("Unexpected toString: " + str,
+            S.INCLUDE_SENSITIVE ?
+            str.startsWith(typeName) && str.contains("obj=" + typeName + " [") :
+            str.startsWith("BinaryObject") && str.contains("idHash=") && str.contains("hash=")
+        );
 
         TestReferenceObject obj1_r = po.deserialize();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java
index 4f5fe9b2500e1..6a0eae2d17b48 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java
@@ -54,7 +54,13 @@ public void testToStringWithAdditions() throws Exception {
 
         IgniteLogger log = log();
 
-        log.info(obj.toStringWithAdditional());
+        String manual = obj.toStringWithAdditionalManual();
+        log.info(manual);
+
+        String automatic = obj.toStringWithAdditionalAutomatic();
+        log.info(automatic);
+
+        assert manual.equals(automatic);
     }
 
     /**
@@ -117,7 +123,7 @@ public void testToStringPerformance() {
     /**
      * Test class.
      */
-    private class TestClass1 {
+    private static class TestClass1 {
         /** */
         @SuppressWarnings("unused")
         @GridToStringOrder(0)
@@ -129,6 +135,7 @@ private class TestClass1 {
 
         /** */
         @SuppressWarnings("unused")
+        @GridToStringInclude(sensitive = true)
         private long longVar;
 
         /** */
@@ -180,7 +187,8 @@ String toStringManual() {
             buf.append("id=").append(id).append(", ");
             buf.append("uuidVar=").append(uuidVar).append(", ");
             buf.append("intVar=").append(intVar).append(", ");
-            buf.append("longVar=").append(longVar).append(", ");
+            if (S.INCLUDE_SENSITIVE)
+                buf.append("longVar=").append(longVar).append(", ");
             buf.append("boolVar=").append(boolVar).append(", ");
             buf.append("byteVar=").append(byteVar).append(", ");
             buf.append("name=").append(name).append(", ");
@@ -200,10 +208,23 @@ String toStringAutomatic() {
         }
 
         /**
-         * @return String with additional parameters.
+         * @return Automatic string with additional parameters.
+         */
+        String toStringWithAdditionalAutomatic() {
+            return S.toString(TestClass1.class, this, "newParam1", 1, false, "newParam2", 2, true);
+        }
+
+        /**
+         * @return Manual string with additional parameters.
          */
-        String toStringWithAdditional() {
-            return S.toString(TestClass1.class, this, "newParam1", 1, "newParam2", 2);
+        String toStringWithAdditionalManual() {
+            StringBuilder s = new StringBuilder(toStringManual());
+            s.setLength(s.length() - 1);
+            s.append(", newParam1=").append(1);
+            if (S.INCLUDE_SENSITIVE)
+                s.append(", newParam2=").append(2);
+            s.append(']');
+            return s.toString();
         }
     }
 }
\ No newline at end of file

From 5769f44367cae5908cd291f226e9fccd68fe1c39 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 27 Dec 2016 15:14:13 +0700
Subject: [PATCH 078/446] Fixed Visor queries for BinaryObjects.

---
 .../query/VisorQueryScanSubstringFilter.java  |  5 +-
 .../internal/visor/query/VisorQueryUtils.java | 60 +++++++++++++++++++
 2 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryScanSubstringFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryScanSubstringFilter.java
index 43eb6dda64e7f..171698b52a3dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryScanSubstringFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryScanSubstringFilter.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.visor.query;
 
+import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.lang.IgniteBiPredicate;
 
 /**
@@ -52,8 +53,8 @@ public VisorQueryScanSubstringFilter(boolean caseSensitive, String ptrn) {
      * @return {@code true} when string presentation of key or value contain specified string.
      */
     @Override public boolean apply(Object key, Object val) {
-        String k = key.toString();
-        String v = val.toString();
+        String k = key instanceof BinaryObject ? VisorQueryUtils.binaryToString((BinaryObject)key) : key.toString();
+        String v = val instanceof BinaryObject ? VisorQueryUtils.binaryToString((BinaryObject)val) : val.toString();
 
         if (caseSensitive)
             return k.contains(ptrn) || v.contains(ptrn);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
index 0b8cf837a339c..5faeac0754abf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
@@ -25,7 +25,13 @@
 import java.util.Date;
 import java.util.List;
 import javax.cache.Cache;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryType;
+import org.apache.ignite.internal.binary.BinaryObjectEx;
 import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.SB;
 
 /**
  * Contains utility methods for Visor query tasks and jobs.
@@ -77,12 +83,19 @@ private static String typeOf(Object o) {
     private static String valueOf(Object o) {
         if (o == null)
             return "null";
+
         if (o instanceof byte[])
             return "size=" + ((byte[])o).length;
+
         if (o instanceof Byte[])
             return "size=" + ((Byte[])o).length;
+
         if (o instanceof Object[])
             return "size=" + ((Object[])o).length + ", values=[" + mkString((Object[])o, 120) + "]";
+
+        if (o instanceof BinaryObject)
+            return binaryToString((BinaryObject)o);
+
         return o.toString();
     }
 
@@ -167,6 +180,51 @@ private static boolean isKnownType(Object obj) {
             obj instanceof URL;
     }
 
+    /**
+     * Convert Binary object to string.
+     *
+     * @param obj Binary object.
+     * @return String representation of Binary object.
+     */
+    public static String binaryToString(BinaryObject obj) {
+        int hash = obj.hashCode();
+
+        if (obj instanceof BinaryObjectEx) {
+            BinaryObjectEx objEx = (BinaryObjectEx)obj;
+
+            BinaryType meta;
+
+            try {
+                meta = ((BinaryObjectEx)obj).rawType();
+            }
+            catch (BinaryObjectException ignore) {
+                meta = null;
+            }
+
+            if (meta != null) {
+                SB buf = new SB(meta.typeName());
+
+                if (meta.fieldNames() != null) {
+                    buf.a(" [hash=").a(hash);
+
+                    for (String name : meta.fieldNames()) {
+                        Object val = objEx.field(name);
+
+                        buf.a(", ").a(name).a('=').a(val);
+                    }
+
+                    buf.a(']');
+
+                    return buf.toString();
+                }
+            }
+        }
+
+        return S.toString(obj.getClass().getSimpleName(),
+            "hash", hash, false,
+            "typeId", obj.type().typeId(), true);
+    }
+
     /**
      * Collects rows from sql query future, first time creates meta and column names arrays.
      *
@@ -193,6 +251,8 @@ public static List<Object[]> fetchSqlQueryRows(VisorQueryCursor<List<?>> cur, in
                     row[i] = null;
                 else if (isKnownType(o))
                     row[i] = o;
+                else if (o instanceof BinaryObject)
+                    row[i] = binaryToString((BinaryObject)o);
                 else
                     row[i] = o.getClass().isArray() ? "binary" : o.toString();
             }

From 5494dfb8dd222bf7aee8214b6bb201d3ae8a1ec5 Mon Sep 17 00:00:00 2001
From: Ignite Teamcity <ignite@apache.org>
Date: Tue, 27 Dec 2016 14:50:58 +0300
Subject: [PATCH 079/446] 1.8.0-SNAPSHOT

---
 modules/platforms/cpp/configure.ac                            | 2 +-
 modules/platforms/cpp/configure.acrel                         | 2 +-
 modules/platforms/cpp/examples/configure.ac                   | 2 +-
 modules/platforms/cpp/odbc/install/ignite-odbc-amd64.wxs      | 2 +-
 modules/platforms/cpp/odbc/install/ignite-odbc-x86.wxs        | 2 +-
 .../Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs     | 4 ++--
 .../dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs    | 4 ++--
 .../Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs       | 4 ++--
 .../Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs | 4 ++--
 .../Properties/AssemblyInfo.cs                                | 4 ++--
 .../Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs       | 4 ++--
 .../dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs      | 4 ++--
 .../dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs      | 4 ++--
 .../dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs   | 4 ++--
 .../dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs      | 4 ++--
 .../platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs | 4 ++--
 .../Apache.Ignite.Examples/Properties/AssemblyInfo.cs         | 4 ++--
 .../Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs      | 4 ++--
 18 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/modules/platforms/cpp/configure.ac b/modules/platforms/cpp/configure.ac
index 9dc9f529a067d..500bdbff2cdc2 100644
--- a/modules/platforms/cpp/configure.ac
+++ b/modules/platforms/cpp/configure.ac
@@ -19,7 +19,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([Apache Ignite C++], [1.8.0.14218], [dev@ignite.apache.org], [ignite], [ignite.apache.org])
+AC_INIT([Apache Ignite C++], [1.8.0.16695], [dev@ignite.apache.org], [ignite], [ignite.apache.org])
 
 AC_CANONICAL_HOST
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/modules/platforms/cpp/configure.acrel b/modules/platforms/cpp/configure.acrel
index 036f124b84204..984aa38c89d1f 100644
--- a/modules/platforms/cpp/configure.acrel
+++ b/modules/platforms/cpp/configure.acrel
@@ -19,7 +19,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([Apache Ignite C++], [1.6.0.8653], [dev@ignite.apache.org], [ignite], [ignite.apache.org])
+AC_INIT([Apache Ignite C++], [1.8.0.16695], [dev@ignite.apache.org], [ignite], [ignite.apache.org])
 
 AC_CANONICAL_HOST
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/modules/platforms/cpp/examples/configure.ac b/modules/platforms/cpp/examples/configure.ac
index 82a3727fbf4fa..6f08490aa08ef 100644
--- a/modules/platforms/cpp/examples/configure.ac
+++ b/modules/platforms/cpp/examples/configure.ac
@@ -19,7 +19,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([Apache Ignite C++ Examples], [1.8.0.14218], [dev@ignite.apache.org], [ignite-examples], [ignite.apache.org])
+AC_INIT([Apache Ignite C++ Examples], [1.8.0.16695], [dev@ignite.apache.org], [ignite-examples], [ignite.apache.org])
 
 AC_CANONICAL_HOST
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/modules/platforms/cpp/odbc/install/ignite-odbc-amd64.wxs b/modules/platforms/cpp/odbc/install/ignite-odbc-amd64.wxs
index cb6ab5410b205..2c12d9a2eac08 100644
--- a/modules/platforms/cpp/odbc/install/ignite-odbc-amd64.wxs
+++ b/modules/platforms/cpp/odbc/install/ignite-odbc-amd64.wxs
@@ -21,7 +21,7 @@
 	<Product Name='Apache Ignite ODBC 64-bit Driver' Manufacturer='The Apache Software Foundation'
 		Id='F3E308E4-910C-4AF5-82DE-2ACF4D64830E' 
 		UpgradeCode='1D7AEFDF-6CD2-4FB5-88F2-811A89832D6D'
-		Language='1033' Codepage='1252' Version='1.6.7.0'>
+		Language='1033' Codepage='1252' Version='1.8.0.16695'>
 		
 		<Package Id='*' Keywords='Installer' Description="Apache Ignite ODBC 64-bit Driver Installer"
 			Comments='Apache, Apache Ignite, the Apache feather and the Apache Ignite logo are trademarks of The Apache Software Foundation.'
diff --git a/modules/platforms/cpp/odbc/install/ignite-odbc-x86.wxs b/modules/platforms/cpp/odbc/install/ignite-odbc-x86.wxs
index 6ee1cdf10e2ea..9e77e2ad14337 100644
--- a/modules/platforms/cpp/odbc/install/ignite-odbc-x86.wxs
+++ b/modules/platforms/cpp/odbc/install/ignite-odbc-x86.wxs
@@ -21,7 +21,7 @@
 	<Product Name='Apache Ignite ODBC 32-bit Driver' Manufacturer='The Apache Software Foundation'
 		Id='D39CBABA-1E21-4701-AA5C-91EDA07B383B' 
 		UpgradeCode='743902A4-365C-424E-B226-5B2898A3941E'
-		Language='1033' Codepage='1252' Version='1.6.7.0'>
+		Language='1033' Codepage='1252' Version='1.8.0.16695'>
 		
 		<Package Id='*' Keywords='Installer' Description="Apache Ignite ODBC 32-bit Driver Installer"
 			Comments='Apache, Apache Ignite, the Apache feather and the Apache Ignite logo are trademarks of The Apache Software Foundation.'
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
index afaa6f07669fe..d6cb3df14b072 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
@@ -35,8 +35,8 @@
 
 [assembly: Guid("18ea4c71-a11d-4ab1-8042-418f7559d84f")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
 
 [assembly: CLSCompliant(true)]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
index 2a7da675982fb..9bd5c4726d7d4 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
@@ -33,8 +33,8 @@
 
 [assembly: Guid("13ea96fc-cc83-4164-a7c0-4f30ed797460")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
 
 [assembly: CLSCompliant(true)]
diff --git a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
index 85af146eba846..91f79bde52888 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
@@ -31,6 +31,6 @@
 
 [assembly: Guid("8fae8395-7e91-411a-a78f-44d6d3fed0fc")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
index 34fca37b4b157..ad0e91549fdd0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
@@ -30,6 +30,6 @@
 [assembly: ComVisible(false)]
 [assembly: Guid("134707f6-155d-47f6-9eb2-c67abbf3c009")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
index 4aa03f1d9035b..a8c4305674e86 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
@@ -45,6 +45,6 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
index 9eb2e2490e07a..7bf322fa81dae 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
@@ -31,6 +31,6 @@
 
 [assembly: Guid("de8dd5cc-7c7f-4a09-80d5-7086d9416a7b")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
index 9fcbeb0862325..240f2731708f3 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
@@ -33,8 +33,8 @@
 
 [assembly: Guid("97db45a8-f922-456a-a819-7b3c6e5e03ba")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
 
 [assembly: CLSCompliant(true)]
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
index d47bef9a1d931..76596da4a3ffa 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
@@ -33,8 +33,8 @@
 // The following GUID is for the ID of the typelib if this project is exposed to COM
 [assembly: Guid("5b571661-17f4-4f29-8c7d-0edb38ca9b55")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
 
 [assembly: CLSCompliant(true)]
diff --git a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
index bb8e830dbf864..824252e90449d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
@@ -33,8 +33,8 @@
 // The following GUID is for the ID of the typelib if this project is exposed to COM
 [assembly: Guid("6f82d669-382e-4435-8092-68c4440146d8")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
 
 [assembly: CLSCompliant(true)]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
index 50220d205321d..f28278c55723b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
@@ -33,8 +33,8 @@
 // The following GUID is for the ID of the typelib if this project is exposed to COM
 [assembly: Guid("c6b58e4a-a2e9-4554-ad02-68ce6da5cfb7")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
 
 [assembly: CLSCompliant(true)]
diff --git a/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
index 82e27b130de6f..85030264cf06f 100644
--- a/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
@@ -31,6 +31,6 @@
 
 [assembly: Guid("0f9702ec-da7d-4ce5-b4b7-73310c885355")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
index 4f5503960de9f..5834c57bca37f 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
@@ -31,6 +31,6 @@
 
 [assembly: Guid("41a0cb95-3435-4c78-b867-900b28e2c9ee")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
index 471e7e972319c..230e9d8434d53 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
@@ -31,6 +31,6 @@
 
 [assembly: Guid("ce65ec7c-d3cf-41ad-8f45-f90d5af68d77")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
+[assembly: AssemblyVersion("1.8.0.16695")]
+[assembly: AssemblyFileVersion("1.8.0.16695")]
 [assembly: AssemblyInformationalVersion("1.8.0")]

From b8cb82de65a529040ea18b0dc03fa7109c69bb4a Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Thu, 29 Dec 2016 14:48:45 +0700
Subject: [PATCH 080/446] IGNITE-4442 Implemented cache affinity configuration.

(cherry picked from commit f4a1e6c)
---
 modules/web-console/backend/app/mongo.js      | 19 +++++
 .../generator/AbstractTransformer.js          |  5 ++
 .../modules/configuration/generator/Beans.js  |  4 +
 .../generator/ConfigurationGenerator.js       | 36 ++++++++
 .../states/configuration/caches/affinity.jade | 82 +++++++++++++++++++
 .../states/configuration/caches/memory.jade   |  4 +-
 .../frontend/views/configuration/caches.jade  |  1 +
 7 files changed, 149 insertions(+), 2 deletions(-)
 create mode 100644 modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade

diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index 58ab11983aca1..dd71f3a8bf57e 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -140,6 +140,25 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
         cacheMode: {type: String, enum: ['PARTITIONED', 'REPLICATED', 'LOCAL']},
         atomicityMode: {type: String, enum: ['ATOMIC', 'TRANSACTIONAL']},
 
+        affinity: {
+            kind: {type: String, enum: ['Default', 'Rendezvous', 'Fair', 'Custom']},
+            Rendezvous: {
+                affinityBackupFilter: String,
+                partitions: Number,
+                excludeNeighbors: Boolean
+            },
+            Fair: {
+                affinityBackupFilter: String,
+                partitions: Number,
+                excludeNeighbors: Boolean
+            },
+            Custom: {
+                className: String
+            }
+        },
+
+        affinityMapper: String,
+
         nodeFilter: {
             kind: {type: String, enum: ['Default', 'Exclude', 'IGFS', 'OnNodes', 'Custom']},
             Exclude: {
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
index f5afe59805114..40d937e1d39dd 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
@@ -210,6 +210,11 @@ export default class AbstractTransformer {
         return this.toSection(this.generator.cacheGeneral(cache));
     }
 
+    // Generate cache memory group.
+    static cacheAffinity(cache) {
+        return this.toSection(this.generator.cacheAffinity(cache));
+    }
+
     // Generate cache memory group.
     static cacheMemory(cache) {
         return this.toSection(this.generator.cacheMemory(cache));
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Beans.js b/modules/web-console/frontend/app/modules/configuration/generator/Beans.js
index ca1934282510e..0972eace6801c 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Beans.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Beans.js
@@ -116,6 +116,10 @@ export class Bean extends EmptyBean {
         return this._property(this.arguments, 'int', model, null, _.nonNil);
     }
 
+    boolConstructorArgument(model) {
+        return this._property(this.arguments, 'boolean', model, null, _.nonNil);
+    }
+
     classConstructorArgument(model) {
         return this._property(this.arguments, 'java.lang.Class', model, null, _.nonEmpty);
     }
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 8770bf6063c73..de2b750675fd8 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -1453,6 +1453,41 @@ export default class IgniteConfigurationGenerator {
         return ccfg;
     }
 
+    // Generation of constructor for affinity function.
+    static cacheAffinityFunction(cls, func) {
+        const affBean = new Bean(cls, 'affinityFunction', func);
+
+        affBean.boolConstructorArgument('excludeNeighbors')
+            .intProperty('partitions')
+            .emptyBeanProperty('affinityBackupFilter');
+
+        return affBean;
+    }
+
+    // Generate cache memory group.
+    static cacheAffinity(cache, ccfg = this.cacheConfigurationBean(cache)) {
+        switch (_.get(cache, 'affinity.kind')) {
+            case 'Rendezvous':
+                ccfg.beanProperty('affinity', this.cacheAffinityFunction('org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction', cache.affinity.Rendezvous));
+
+                break;
+            case 'Fair':
+                ccfg.beanProperty('affinity', this.cacheAffinityFunction('org.apache.ignite.cache.affinity.fair.FairAffinityFunction', cache.affinity.Fair));
+
+                break;
+            case 'Custom':
+                ccfg.emptyBeanProperty('affinity.Custom.className', 'affinity');
+
+                break;
+            default:
+                // No-op.
+        }
+
+        ccfg.emptyBeanProperty('affinityMapper');
+
+        return ccfg;
+    }
+
     // Generate cache memory group.
     static cacheMemory(cache, ccfg = this.cacheConfigurationBean(cache)) {
         ccfg.enumProperty('memoryMode');
@@ -1728,6 +1763,7 @@ export default class IgniteConfigurationGenerator {
 
     static cacheConfiguration(cache, ccfg = this.cacheConfigurationBean(cache)) {
         this.cacheGeneral(cache, ccfg);
+        this.cacheAffinity(cache, ccfg);
         this.cacheMemory(cache, ccfg);
         this.cacheQuery(cache, cache.domains, ccfg);
         this.cacheStore(cache, cache.domains, ccfg);
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade
new file mode 100644
index 0000000000000..3c4746bbca794
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade
@@ -0,0 +1,82 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins.jade
+
+-var form = 'affinity'
+-var model = 'backupItem'
+-var affModel = model + '.affinity'
+-var affMapModel = model + '.affinityMapper'
+-var rendezvousAff = affModel + '.kind === "Rendezvous"'
+-var fairAff = affModel + '.kind === "Fair"'
+-var customAff = affModel + '.kind === "Custom"'
+-var customAffMapper = affMapModel + '.kind === "Custom"'
+-var rendPartitionsRequired = rendezvousAff + ' && ' + affModel + '.Rendezvous.affinityBackupFilter'
+-var fairPartitionsRequired = fairAff + ' && ' + affModel + '.Fair.affinityBackupFilter'
+
+.panel.panel-default(ng-form=form novalidate)
+    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+        ignite-form-panel-chevron
+        label Affinity Collocation
+        ignite-form-field-tooltip.tipLabel
+            | Collocate data with data to improve performance and scalability of your application#[br]
+            | #[a(href="http://apacheignite.gridgain.org/docs/affinity-collocation" target="_blank") More info]
+        ignite-form-revert
+    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
+        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+            .col-sm-6
+                .settings-row
+                    +dropdown('Function:', affModel + '.kind', '"AffinityKind"', 'true', 'Default',
+                        '[\
+                            {value: "Rendezvous", label: "Rendezvous"},\
+                            {value: "Fair", label: "Fair"},\
+                            {value: "Custom", label: "Custom"},\
+                            {value: undefined, label: "Default"}\
+                        ]',
+                        'Key topology resolver to provide mapping from keys to nodes\
+                        <ul>\
+                            <li>Rendezvous - Based on Highest Random Weight algorithm<br/></li>\
+                            <li>Fair - Tries to ensure that all nodes get equal number of partitions with minimum amount of reassignments between existing nodes<br/></li>\
+                            <li>Custom - Custom implementation of key affinity fynction<br/></li>\
+                            <li>Default - By default rendezvous affinity function  with 1024 partitions is used<br/></li>\
+                        </ul>')
+                .panel-details(ng-if=rendezvousAff)
+                    .details-row
+                        +number-required('Partitions', affModel + '.Rendezvous.partitions', '"RendPartitions"', 'true', rendPartitionsRequired, '1024', '1', 'Number of partitions')
+                    .details-row
+                        +java-class('Backup filter', affModel + '.Rendezvous.affinityBackupFilter', '"RendAffinityBackupFilter"', 'true', 'false',
+                            'Backups will be selected from all nodes that pass this filter')
+                    .details-row
+                        +checkbox('Exclude neighbors', affModel + '.Rendezvous.excludeNeighbors', '"RendExcludeNeighbors"',
+                            'Exclude same - host - neighbors from being backups of each other and specified number of backups')
+                .panel-details(ng-if=fairAff)
+                    .details-row
+                        +number-required('Partitions', affModel + '.Fair.partitions', '"FairPartitions"', 'true', fairPartitionsRequired, '256', '1', 'Number of partitions')
+                    .details-row
+                        +java-class('Backup filter', affModel + '.Fair.affinityBackupFilter', '"FairAffinityBackupFilter"', 'true', 'false',
+                            'Backups will be selected from all nodes that pass this filter')
+                    .details-row
+                        +checkbox('Exclude neighbors', affModel + '.Fair.excludeNeighbors', '"FairExcludeNeighbors"',
+                            'Exclude same - host - neighbors from being backups of each other and specified number of backups')
+                .panel-details(ng-if=customAff)
+                    .details-row
+                        +java-class('Class name:', affModel + '.Custom.className', '"AffCustomClassName"', 'true', customAff,
+                            'Custom key affinity function implementation class name')
+                .settings-row
+                    +java-class('Mapper:', model + '.affinityMapper', '"AffMapCustomClassName"', 'true', 'false',
+                        'Provide custom affinity key for any given key')
+            .col-sm-6
+                +preview-xml-java(model, 'cacheAffinity')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade
index f2d3e2bfa656c..e8dfb3a15a1fc 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade
@@ -61,7 +61,7 @@ include /app/helpers/jade/mixins.jade
                                 Note that in this mode entries can be evicted only to swap\
                             </li>\
                         </ul>')
-                .settings-row(data-ng-show=model + '.memoryMode !== "OFFHEAP_VALUES"')
+                .settings-row(ng-show=model + '.memoryMode !== "OFFHEAP_VALUES"')
                     +dropdown-required('Off-heap memory:', model + '.offHeapMode', '"offHeapMode"', 'true',
                         model + '.memoryMode === "OFFHEAP_TIERED"',
                         'Disabled',
@@ -76,7 +76,7 @@ include /app/helpers/jade/mixins.jade
                             <li>Limited - Off-heap storage has limited size</li>\
                             <li>Unlimited - Off-heap storage grow infinitely (it is up to user to properly add and remove entries from cache to ensure that off-heap storage does not grow infinitely)</li>\
                         </ul>')
-                .settings-row(data-ng-if=model + '.offHeapMode === 1 && ' + model + '.memoryMode !== "OFFHEAP_VALUES"')
+                .settings-row(ng-if=model + '.offHeapMode === 1 && ' + model + '.memoryMode !== "OFFHEAP_VALUES"')
                     +number-required('Off-heap memory max size:', model + '.offHeapMaxMemory', '"offHeapMaxMemory"', 'true',
                         model + '.offHeapMode === 1', 'Enter off-heap memory size', '1',
                         'Maximum amount of memory available to off-heap storage in bytes')
diff --git a/modules/web-console/frontend/views/configuration/caches.jade b/modules/web-console/frontend/views/configuration/caches.jade
index 4a4cf2e1a1d24..73a6309d223c6 100644
--- a/modules/web-console/frontend/views/configuration/caches.jade
+++ b/modules/web-console/frontend/views/configuration/caches.jade
@@ -44,6 +44,7 @@ include /app/helpers/jade/mixins.jade
                         +advanced-options-toggle-default
 
                         div(ng-show='ui.expanded')
+                            include /app/modules/states/configuration/caches/affinity.jade
                             include /app/modules/states/configuration/caches/concurrency.jade
                             include /app/modules/states/configuration/caches/near-cache-client.jade
                             include /app/modules/states/configuration/caches/near-cache-server.jade

From 6c38eb28623271a3604ee8d966deb88677a3adb1 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Thu, 29 Dec 2016 12:20:20 +0300
Subject: [PATCH 081/446] IGNITE-4167: Changed
 IGNITE_TO_STRING_INCLUDE_SENSITIVE default value to "true".

---
 .../util/tostring/GridToStringBuilder.java     | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
index 333f95e64ae58..6807b3f8ad0ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
@@ -17,6 +17,13 @@
 
 package org.apache.ignite.internal.util.tostring;
 
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
 import java.io.Externalizable;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -34,12 +41,8 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.SB;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE;
 
 /**
  * Provides auto-generation framework for {@code toString()} output.
@@ -87,7 +90,8 @@ public class GridToStringBuilder {
     public static final int MAX_COL_SIZE = 100;
 
     /** {@link IgniteSystemProperties#IGNITE_TO_STRING_INCLUDE_SENSITIVE} */
-    public static final boolean INCLUDE_SENSITIVE = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE, false);
+    public static final boolean INCLUDE_SENSITIVE =
+        IgniteSystemProperties.getBoolean(IGNITE_TO_STRING_INCLUDE_SENSITIVE, true);
 
     /** */
     private static ThreadLocal<Queue<GridToStringThreadLocal>> threadCache = new ThreadLocal<Queue<GridToStringThreadLocal>>() {

From 864a95e13f1262f14351df0883d0a1abd1bf70c7 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 29 Dec 2016 14:45:08 +0300
Subject: [PATCH 082/446] Removed duplicated benchmark.

---
 .../cache/IgniteIoTestBenchmark.java          | 73 -------------------
 1 file changed, 73 deletions(-)
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java
deleted file mode 100644
index bee45e0a339c9..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.cache;
-
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-import org.yardstickframework.BenchmarkConfiguration;
-import org.yardstickframework.BenchmarkUtils;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class IgniteIoTestBenchmark extends IgniteAbstractBenchmark {
-    /** */
-    private List<ClusterNode> targetNodes;
-
-    /** */
-    private IgniteKernal ignite;
-
-    /** {@inheritDoc} */
-    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
-        super.setUp(cfg);
-
-        ignite = (IgniteKernal)ignite();
-
-        targetNodes = new ArrayList<>();
-
-        ClusterNode loc = ignite().cluster().localNode();
-
-        Collection<ClusterNode> nodes = ignite().cluster().forServers().nodes();
-
-        for (ClusterNode node : nodes) {
-            if (!loc.equals(node))
-                targetNodes.add(node);
-        }
-
-        if (targetNodes.isEmpty())
-            throw new IgniteException("Failed to find remote server nodes [nodes=" + nodes + ']');
-
-        BenchmarkUtils.println(cfg, "Initialized target nodes: " + targetNodes + ']');
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
-        ClusterNode node = targetNodes.get(nextRandom(targetNodes.size()));
-
-        ignite.sendIoTest(node, null, false).get();
-
-        return true;
-    }
-}

From da5b68cc89ba6eeff25beb66e3a4d8c2b9871ab3 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 29 Dec 2016 15:46:59 +0300
Subject: [PATCH 083/446] For communication spi disabled pairedConnections by
 default, implemented NIO sessions balancing for this mode.

---
 .../internal/util/nio/GridNioServer.java      | 159 +++++++++++++++---
 .../tcp/TcpCommunicationSpi.java              |  20 ++-
 .../tcp/TcpCommunicationSpiMBean.java         |   5 +-
 ...unicationBalancePairedConnectionsTest.java |  28 +++
 .../IgniteCommunicationBalanceTest.java       |  25 ++-
 ...MessageRecoveryPairedConnectionsTest.java} |   6 +-
 .../testsuites/IgniteCacheTestSuite.java      |   6 +-
 7 files changed, 206 insertions(+), 43 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalancePairedConnectionsTest.java
 rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/{IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java => IgniteCacheAtomicMessageRecoveryPairedConnectionsTest.java} (87%)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index bc1f173a29af4..a59adba9d16e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -227,6 +227,9 @@ public class GridNioServer<T> {
     /** */
     private final IgniteRunnable balancer;
 
+    /** */
+    private final boolean readWriteSelectorsAssign;
+
     /**
      * @param addr Address.
      * @param port Port.
@@ -250,7 +253,7 @@ public class GridNioServer<T> {
      * @param writerFactory Writer factory.
      * @param skipRecoveryPred Skip recovery predicate.
      * @param msgQueueLsnr Message queue size listener.
-     * @param balancing NIO sessions balancing flag.
+     * @param readWriteSelectorsAssign If {@code true} then in/out connections are assigned to even/odd workers.
      * @param filters Filters for this server.
      * @throws IgniteCheckedException If failed.
      */
@@ -275,7 +278,7 @@ private GridNioServer(
         GridNioMessageWriterFactory writerFactory,
         IgnitePredicate<Message> skipRecoveryPred,
         IgniteBiInClosure<GridNioSession, Integer> msgQueueLsnr,
-        boolean balancing,
+        boolean readWriteSelectorsAssign,
         GridNioFilter... filters
     ) throws IgniteCheckedException {
         if (port != -1)
@@ -300,6 +303,7 @@ private GridNioServer(
         this.sndQueueLimit = sndQueueLimit;
         this.msgQueueLsnr = msgQueueLsnr;
         this.selectorSpins = selectorSpins;
+        this.readWriteSelectorsAssign = readWriteSelectorsAssign;
 
         filterChain = new GridNioFilterChain<>(log, lsnr, new HeadFilter(), filters);
 
@@ -359,10 +363,16 @@ private GridNioServer(
 
         IgniteRunnable balancer0 = null;
 
-        if (balancing && balancePeriod > 0) {
+        if (balancePeriod > 0) {
             boolean rndBalance = IgniteSystemProperties.getBoolean(IGNITE_IO_BALANCE_RANDOM_BALANCE, false);
 
-            balancer0 = rndBalance ? new RandomBalancer() : new SizeBasedBalancer(balancePeriod);
+            if (rndBalance)
+                balancer0 = new RandomBalancer();
+            else {
+                balancer0 = readWriteSelectorsAssign ?
+                    new ReadWriteSizeBasedBalancer(balancePeriod) :
+                    new SizeBasedBalancer(balancePeriod);
+            }
         }
 
         this.balancer = balancer0;
@@ -823,21 +833,31 @@ private synchronized void offerBalanced(NioOperationFuture req) {
         int balanceIdx;
 
         if (workers > 1) {
-            if (req.accepted()) {
-                balanceIdx = readBalanceIdx;
+            if (readWriteSelectorsAssign) {
+                if (req.accepted()) {
+                    balanceIdx = readBalanceIdx;
 
-                readBalanceIdx += 2;
+                    readBalanceIdx += 2;
 
-                if (readBalanceIdx >= workers)
-                    readBalanceIdx = 0;
+                    if (readBalanceIdx >= workers)
+                        readBalanceIdx = 0;
+                }
+                else {
+                    balanceIdx = writeBalanceIdx;
+
+                    writeBalanceIdx += 2;
+
+                    if (writeBalanceIdx >= workers)
+                        writeBalanceIdx = 1;
+                }
             }
             else {
-                balanceIdx = writeBalanceIdx;
+                balanceIdx = readBalanceIdx;
 
-                writeBalanceIdx += 2;
+                readBalanceIdx++;
 
-                if (writeBalanceIdx >= workers)
-                    writeBalanceIdx = 1;
+                if (readBalanceIdx >= workers)
+                    readBalanceIdx = 0;
             }
         }
         else
@@ -3124,8 +3144,8 @@ public static class Builder<T> {
         /** */
         private long selectorSpins;
 
-        /** NIO sessions balancing flag. */
-        private boolean balancing;
+        /** */
+        private boolean readWriteSelectorsAssign;
 
         /**
          * Finishes building the instance.
@@ -3155,7 +3175,7 @@ public GridNioServer<T> build() throws IgniteCheckedException {
                 writerFactory,
                 skipRecoveryPred,
                 msgQueueLsnr,
-                balancing,
+                readWriteSelectorsAssign,
                 filters != null ? Arrays.copyOf(filters, filters.length) : EMPTY_FILTERS
             );
 
@@ -3169,11 +3189,11 @@ public GridNioServer<T> build() throws IgniteCheckedException {
         }
 
         /**
-         * @param balancing NIO sessions balancing flag.
+         * @param readWriteSelectorsAssign {@code True} to assign in/out connections even/odd workers.
          * @return This for chaining.
          */
-        public Builder<T> balancing(boolean balancing) {
-            this.balancing = balancing;
+        public Builder<T> readWriteSelectorsAssign(boolean readWriteSelectorsAssign) {
+            this.readWriteSelectorsAssign = readWriteSelectorsAssign;
 
             return this;
         }
@@ -3415,7 +3435,7 @@ public Builder<T> messageQueueSizeListener(IgniteBiInClosure<GridNioSession, Int
     /**
      *
      */
-    private class SizeBasedBalancer implements IgniteRunnable {
+    private class ReadWriteSizeBasedBalancer implements IgniteRunnable {
         /** */
         private static final long serialVersionUID = 0L;
 
@@ -3428,7 +3448,7 @@ private class SizeBasedBalancer implements IgniteRunnable {
         /**
          * @param balancePeriod Period.
          */
-        SizeBasedBalancer(long balancePeriod) {
+        ReadWriteSizeBasedBalancer(long balancePeriod) {
             this.balancePeriod = balancePeriod;
         }
 
@@ -3558,6 +3578,100 @@ private class SizeBasedBalancer implements IgniteRunnable {
         }
     }
 
+    /**
+     *
+     */
+    private class SizeBasedBalancer implements IgniteRunnable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private long lastBalance;
+
+        /** */
+        private final long balancePeriod;
+
+        /**
+         * @param balancePeriod Period.
+         */
+        SizeBasedBalancer(long balancePeriod) {
+            this.balancePeriod = balancePeriod;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            long now = U.currentTimeMillis();
+
+            if (lastBalance + balancePeriod < now) {
+                lastBalance = now;
+
+                long maxBytes0 = -1, minBytes0 = -1;
+                int maxBytesIdx = -1, minBytesIdx = -1;
+
+                for (int i = 0; i < clientWorkers.size(); i++) {
+                    GridNioServer.AbstractNioClientWorker worker = clientWorkers.get(i);
+
+                    int sesCnt = worker.workerSessions.size();
+
+                    long bytes0 = worker.bytesRcvd0 + worker.bytesSent0;
+
+                    if ((maxBytes0 == -1 || bytes0 > maxBytes0) && bytes0 > 0 && sesCnt > 1) {
+                        maxBytes0 = bytes0;
+                        maxBytesIdx = i;
+                    }
+
+                    if (minBytes0 == -1 || bytes0 < minBytes0) {
+                        minBytes0 = bytes0;
+                        minBytesIdx = i;
+                    }
+                }
+
+                if (log.isDebugEnabled())
+                    log.debug("Balancing data [min0=" + minBytes0 + ", minIdx=" + minBytesIdx +
+                        ", max0=" + maxBytes0 + ", maxIdx=" + maxBytesIdx + ']');
+
+                if (maxBytes0 != -1 && minBytes0 != -1) {
+                    GridSelectorNioSessionImpl ses = null;
+
+                    long bytesDiff = maxBytes0 - minBytes0;
+                    long delta = bytesDiff;
+                    double threshold = bytesDiff * 0.9;
+
+                    GridConcurrentHashSet<GridSelectorNioSessionImpl> sessions =
+                        clientWorkers.get(maxBytesIdx).workerSessions;
+
+                    for (GridSelectorNioSessionImpl ses0 : sessions) {
+                        long bytesSent0 = ses0.bytesSent0();
+
+                        if (bytesSent0 < threshold &&
+                            (ses == null || delta > U.safeAbs(bytesSent0 - bytesDiff / 2))) {
+                            ses = ses0;
+                            delta = U.safeAbs(bytesSent0 - bytesDiff / 2);
+                        }
+                    }
+
+                    if (ses != null) {
+                        if (log.isDebugEnabled())
+                            log.debug("Will move session to less loaded worker [ses=" + ses +
+                                ", from=" + maxBytesIdx + ", to=" + minBytesIdx + ']');
+
+                        moveSession(ses, maxBytesIdx, minBytesIdx);
+                    }
+                    else {
+                        if (log.isDebugEnabled())
+                            log.debug("Unable to find session to move.");
+                    }
+                }
+
+                for (int i = 0; i < clientWorkers.size(); i++) {
+                    GridNioServer.AbstractNioClientWorker worker = clientWorkers.get(i);
+
+                    worker.reset0();
+                }
+            }
+        }
+    }
+
     /**
      * For tests only.
      */
@@ -3625,6 +3739,9 @@ private GridNioSession randomSession(GridNioServer.AbstractNioClientWorker worke
      *
      */
     interface SessionChangeRequest {
+        /**
+         * @return Session.
+         */
         GridNioSession session();
 
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index c35b5efb302f9..ae0e6f0a85905 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -293,7 +293,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
 
     /**
      * Default count of selectors for TCP server equals to
-     * {@code "Math.min(8, Runtime.getRuntime().availableProcessors())"}.
+     * {@code "Math.max(4, Runtime.getRuntime().availableProcessors() / 2)"}.
      */
     public static final int DFLT_SELECTORS_CNT = Math.max(4, Runtime.getRuntime().availableProcessors() / 2);
 
@@ -979,7 +979,7 @@ class ConnectClosure implements IgniteInClosure<Boolean> {
     private IpcSharedMemoryServerEndpoint shmemSrv;
 
     /** */
-    private boolean usePairedConnections = true;
+    private boolean usePairedConnections;
 
     /** */
     private int connectionsPerNode = DFLT_CONN_PER_NODE;
@@ -1193,10 +1193,8 @@ public void setLocalPortRange(int locPortRange) {
      * Set this to {@code false} if each connection of {@link #getConnectionsPerNode()}
      * should be used for outgoing and incoming messages. In this case total number
      * of connections between local and each remote node is {@link #getConnectionsPerNode()}.
-     * In this case load NIO selectors load
-     * balancing of {@link GridNioServer} will be disabled.
      * <p>
-     * Default is {@code true}.
+     * Default is {@code false}.
      *
      * @param usePairedConnections {@code true} to use paired connections and {@code false} otherwise.
      * @see #getConnectionsPerNode()
@@ -2057,16 +2055,20 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
                         .writerFactory(writerFactory)
                         .skipRecoveryPredicate(skipRecoveryPred)
                         .messageQueueSizeListener(queueSizeMonitor)
-                        .balancing(usePairedConnections) // Current balancing logic assumes separate in/out connections.
+                        .readWriteSelectorsAssign(usePairedConnections)
                         .build();
 
                 boundTcpPort = port;
 
                 // Ack Port the TCP server was bound to.
-                if (log.isInfoEnabled())
+                if (log.isInfoEnabled()) {
                     log.info("Successfully bound communication NIO server to TCP port " +
-                        "[port=" + boundTcpPort + ", locHost=" + locHost + ", selectorsCnt=" + selectorsCnt +
-                        ", selectorSpins=" + srvr.selectorSpins() + ']');
+                        "[port=" + boundTcpPort +
+                        ", locHost=" + locHost +
+                        ", selectorsCnt=" + selectorsCnt +
+                        ", selectorSpins=" + srvr.selectorSpins() +
+                        ", pairedConn=" + usePairedConnections + ']');
+                }
 
                 srvr.idleTimeout(idleConnTimeout);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java
index c7a1a535c69a8..c56e18c10052b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java
@@ -51,10 +51,9 @@ public interface TcpCommunicationSpiMBean extends IgniteSpiManagementMBean {
      * is {@link #getConnectionsPerNode()} * 2.
      * <p>
      * Returns {@code false} if each connection of {@link #getConnectionsPerNode()}
-     * should be used for outgoing and incoming messages. In this case load NIO selectors load
-     * balancing of {@link GridNioServer} will be disabled.
+     * should be used for outgoing and incoming messages.
      * <p>
-     * Default is {@code true}.
+     * Default is {@code false}.
      *
      * @return {@code true} to use paired connections and {@code false} otherwise.
      * @see #getConnectionsPerNode()
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalancePairedConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalancePairedConnectionsTest.java
new file mode 100644
index 0000000000000..4544030b59130
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalancePairedConnectionsTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+/**
+ *
+ */
+public class IgniteCommunicationBalancePairedConnectionsTest extends IgniteCommunicationBalanceTest {
+    /** {@inheritDoc} */
+    @Override protected boolean usePairedConnections() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
index e142aefdb2343..4271417554fa7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
@@ -63,6 +63,7 @@ public class IgniteCommunicationBalanceTest extends GridCommonAbstractTest {
 
         commSpi.setSharedMemoryPort(-1);
         commSpi.setConnectionsPerNode(connectionsPerNode());
+        commSpi.setUsePairedConnections(usePairedConnections());
 
         if (selectors > 0)
             commSpi.setSelectorsCount(selectors);
@@ -74,6 +75,13 @@ public class IgniteCommunicationBalanceTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /**
+     * @return Value for {@link TcpCommunicationSpi#setUsePairedConnections(boolean)}.
+     */
+    protected boolean usePairedConnections() {
+        return false;
+    }
+
     /**
      * @return Connections per node.
      */
@@ -97,7 +105,7 @@ public void testBalance1() throws Exception {
         try {
             selectors = 4;
 
-            final int SRVS = 4;
+            final int SRVS = 6;
 
             startGridsMultiThreaded(SRVS);
 
@@ -105,7 +113,7 @@ public void testBalance1() throws Exception {
 
             final Ignite client = startGrid(SRVS);
 
-            for (int i = 0; i < 4; i++) {
+            for (int i = 0; i < SRVS; i++) {
                 ClusterNode node = client.cluster().node(ignite(i).cluster().localNode().id());
 
                 client.compute(client.cluster().forNode(node)).call(new DummyCallable(null));
@@ -151,7 +159,10 @@ public void testBalance1() throws Exception {
                             }
                         }
 
-                        return srv.readerMoveCount() > readMoveCnt && srv.writerMoveCount() > writeMoveCnt;
+                        if (usePairedConnections())
+                            return srv.readerMoveCount() > readMoveCnt && srv.writerMoveCount() > writeMoveCnt;
+                        else
+                            return srv.readerMoveCount() > readMoveCnt || srv.writerMoveCount() > writeMoveCnt;
                     }
                 }, 30_000);
 
@@ -165,8 +176,12 @@ public void testBalance1() throws Exception {
                     ", rc2=" + readMoveCnt2 +
                     ", wc2=" + writeMoveCnt2 + ']');
 
-                assertTrue(readMoveCnt2 > readMoveCnt1);
-                assertTrue(writeMoveCnt2 > writeMoveCnt1);
+                if (usePairedConnections()) {
+                    assertTrue(readMoveCnt2 > readMoveCnt1);
+                    assertTrue(writeMoveCnt2 > writeMoveCnt1);
+                }
+                else
+                    assertTrue(readMoveCnt2 > readMoveCnt1 || writeMoveCnt2 > writeMoveCnt1);
 
                 readMoveCnt1 = readMoveCnt2;
                 writeMoveCnt1 = writeMoveCnt2;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryPairedConnectionsTest.java
similarity index 87%
rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java
rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryPairedConnectionsTest.java
index 71772efeadc99..dffb96627cc99 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryPairedConnectionsTest.java
@@ -26,16 +26,16 @@
 /**
  *
  */
-public class IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest extends IgniteCacheAtomicMessageRecoveryTest {
+public class IgniteCacheAtomicMessageRecoveryPairedConnectionsTest extends IgniteCacheAtomicMessageRecoveryTest {
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
         TcpCommunicationSpi commSpi = (TcpCommunicationSpi)cfg.getCommunicationSpi();
 
-        assertTrue(commSpi.isUsePairedConnections());
+        assertFalse(commSpi.isUsePairedConnections());
 
-        commSpi.setUsePairedConnections(false);
+        commSpi.setUsePairedConnections(true);
 
         return cfg;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 1e73e794d5d55..092d95eff12bd 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreMultithreadedSelfTest;
 import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreSelfTest;
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceMultipleConnectionsTest;
+import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalancePairedConnectionsTest;
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceTest;
 import org.apache.ignite.internal.managers.communication.IgniteIoTestMessagesTest;
 import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest;
@@ -134,7 +135,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.CacheTxNearUpdateTopologyChangeTest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheEntrySetIterationPreloadingSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecovery10ConnectionsTest;
-import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecoveryPairedConnectionsTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecoveryTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheConnectionRecovery10ConnectionsTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheConnectionRecoveryTest;
@@ -301,7 +302,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridCacheEntrySetIterationPreloadingSelfTest.class);
         suite.addTestSuite(GridCacheMixedPartitionExchangeSelfTest.class);
         suite.addTestSuite(IgniteCacheAtomicMessageRecoveryTest.class);
-        suite.addTestSuite(IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.class);
+        suite.addTestSuite(IgniteCacheAtomicMessageRecoveryPairedConnectionsTest.class);
         suite.addTestSuite(IgniteCacheAtomicMessageRecovery10ConnectionsTest.class);
         suite.addTestSuite(IgniteCacheTxMessageRecoveryTest.class);
         suite.addTestSuite(IgniteCacheMessageWriteTimeoutTest.class);
@@ -339,6 +340,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
 
         suite.addTestSuite(IgniteVariousConnectionNumberTest.class);
         suite.addTestSuite(IgniteCommunicationBalanceTest.class);
+        suite.addTestSuite(IgniteCommunicationBalancePairedConnectionsTest.class);
         suite.addTestSuite(IgniteCommunicationBalanceMultipleConnectionsTest.class);
         suite.addTestSuite(IgniteIoTestMessagesTest.class);
 

From 71412cecd861119965a873520da96078f99c94e2 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Fri, 30 Dec 2016 13:41:34 +0300
Subject: [PATCH 084/446] IGNITE-4424 REPLICATED cache isn't synced across
 nodes

---
 .../GridNearAtomicAbstractUpdateFuture.java   |  34 ++-
 .../GridNearAtomicSingleUpdateFuture.java     |  44 ++--
 .../atomic/GridNearAtomicUpdateFuture.java    |  57 ++---
 .../AtomicPutAllChangingTopologyTest.java     | 212 ++++++++++++++++++
 .../IgniteCacheFailoverTestSuite.java         |   3 +
 5 files changed, 284 insertions(+), 66 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
index 2fbabaa54fc9c..c92e0f5ec5b42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
@@ -212,14 +212,18 @@ public void map() {
             // Cannot remap.
             remapCnt = 1;
 
-            map(topVer);
+            GridCacheVersion futVer = addAtomicFuture(topVer);
+
+            if (futVer != null)
+                map(topVer, futVer);
         }
     }
 
     /**
      * @param topVer Topology version.
+     * @param futVer Future version
      */
-    protected abstract void map(AffinityTopologyVersion topVer);
+    protected abstract void map(AffinityTopologyVersion topVer, GridCacheVersion futVer);
 
     /**
      * Maps future on ready topology.
@@ -302,7 +306,7 @@ protected void mapSingle(UUID nodeId, GridNearAtomicAbstractUpdateRequest req) {
      * @param req Request.
      * @param e Error.
      */
-    protected void onSendError(GridNearAtomicAbstractUpdateRequest req, IgniteCheckedException e) {
+    protected final void onSendError(GridNearAtomicAbstractUpdateRequest req, IgniteCheckedException e) {
         synchronized (mux) {
             GridNearAtomicUpdateResponse res = new GridNearAtomicUpdateResponse(cctx.cacheId(),
                 req.nodeId(),
@@ -314,4 +318,28 @@ protected void onSendError(GridNearAtomicAbstractUpdateRequest req, IgniteChecke
             onResult(req.nodeId(), res, true);
         }
     }
+
+    /**
+     * Adds future prevents topology change before operation complete.
+     * Should be invoked before topology lock released.
+     *
+     * @param topVer Topology version.
+     * @return Future version in case future added.
+     */
+    protected final GridCacheVersion addAtomicFuture(AffinityTopologyVersion topVer) {
+        GridCacheVersion futVer = cctx.versions().next(topVer);
+
+        synchronized (mux) {
+            assert this.futVer == null : this;
+            assert this.topVer == AffinityTopologyVersion.ZERO : this;
+
+            this.topVer = topVer;
+            this.futVer = futVer;
+        }
+
+        if (storeFuture() && !cctx.mvcc().addAtomicFuture(futVer, this))
+            return null;
+
+        return futVer;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
index bd231cf0003d8..7376affca4455 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
@@ -348,14 +348,7 @@ else if (res.error() != null) {
                 @Override public void apply(final IgniteInternalFuture<AffinityTopologyVersion> fut) {
                     cctx.kernalContext().closure().runLocalSafe(new Runnable() {
                         @Override public void run() {
-                            try {
-                                AffinityTopologyVersion topVer = fut.get();
-
-                                map(topVer);
-                            }
-                            catch (IgniteCheckedException e) {
-                                onDone(e);
-                            }
+                            mapOnTopology();
                         }
                     });
                 }
@@ -388,7 +381,9 @@ private void updateNear(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicU
     @Override protected void mapOnTopology() {
         cache.topology().readLock();
 
-        AffinityTopologyVersion topVer = null;
+        AffinityTopologyVersion topVer;
+
+        GridCacheVersion futVer;
 
         try {
             if (cache.topology().stopping()) {
@@ -410,6 +405,8 @@ private void updateNear(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicU
                 }
 
                 topVer = fut.topologyVersion();
+
+                futVer = addAtomicFuture(topVer);
             }
             else {
                 if (waitTopFut) {
@@ -435,11 +432,12 @@ private void updateNear(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicU
             cache.topology().readUnlock();
         }
 
-        map(topVer);
+        if (futVer != null)
+            map(topVer, futVer);
     }
 
     /** {@inheritDoc} */
-    protected void map(AffinityTopologyVersion topVer) {
+    @Override protected void map(AffinityTopologyVersion topVer, GridCacheVersion futVer) {
         Collection<ClusterNode> topNodes = CU.affinityNodes(cctx, topVer);
 
         if (F.isEmpty(topNodes)) {
@@ -449,11 +447,6 @@ protected void map(AffinityTopologyVersion topVer) {
             return;
         }
 
-        Exception err = null;
-        GridNearAtomicAbstractUpdateRequest singleReq0 = null;
-
-        GridCacheVersion futVer = cctx.versions().next(topVer);
-
         GridCacheVersion updVer;
 
         // Assign version on near node in CLOCK ordering mode even if fastMap is false.
@@ -470,16 +463,17 @@ protected void map(AffinityTopologyVersion topVer) {
         else
             updVer = null;
 
+        Exception err = null;
+        GridNearAtomicAbstractUpdateRequest singleReq0 = null;
+
         try {
             singleReq0 = mapSingleUpdate(topVer, futVer, updVer);
 
             synchronized (mux) {
-                assert this.futVer == null : this;
-                assert this.topVer == AffinityTopologyVersion.ZERO : this;
+                assert this.futVer == futVer || (this.isDone() && this.error() != null);
+                assert this.topVer == topVer;
 
-                this.topVer = topVer;
                 this.updVer = updVer;
-                this.futVer = futVer;
 
                 resCnt = 0;
 
@@ -496,14 +490,6 @@ protected void map(AffinityTopologyVersion topVer) {
             return;
         }
 
-        if (storeFuture()) {
-            if (!cctx.mvcc().addAtomicFuture(futVer, this)) {
-                assert isDone() : this;
-
-                return;
-            }
-        }
-
         // Optimize mapping for single key.
         mapSingle(singleReq0.nodeId(), singleReq0);
     }
@@ -511,7 +497,7 @@ protected void map(AffinityTopologyVersion topVer) {
     /**
      * @return Future version.
      */
-    GridCacheVersion onFutureDone() {
+    private GridCacheVersion onFutureDone() {
         GridCacheVersion ver0;
 
         GridFutureAdapter<Void> fut0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index cd6411725748b..950e5bdd2b9b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -456,14 +456,7 @@ else if (!nodeErr)
                 @Override public void apply(final IgniteInternalFuture<AffinityTopologyVersion> fut) {
                     cctx.kernalContext().closure().runLocalSafe(new Runnable() {
                         @Override public void run() {
-                            try {
-                                AffinityTopologyVersion topVer = fut.get();
-
-                                map(topVer, remapKeys);
-                            }
-                            catch (IgniteCheckedException e) {
-                                onDone(e);
-                            }
+                            mapOnTopology();
                         }
                     });
                 }
@@ -497,7 +490,9 @@ private void updateNear(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdat
     @Override protected void mapOnTopology() {
         cache.topology().readLock();
 
-        AffinityTopologyVersion topVer = null;
+        AffinityTopologyVersion topVer;
+
+        GridCacheVersion futVer;
 
         try {
             if (cache.topology().stopping()) {
@@ -519,6 +514,8 @@ private void updateNear(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdat
                 }
 
                 topVer = fut.topologyVersion();
+
+                futVer = addAtomicFuture(topVer);
             }
             else {
                 if (waitTopFut) {
@@ -544,7 +541,8 @@ private void updateNear(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdat
             cache.topology().readUnlock();
         }
 
-        map(topVer, null);
+        if (futVer != null)
+            map(topVer, futVer, remapKeys);
     }
 
     /**
@@ -602,15 +600,18 @@ private void doUpdate(Map<UUID, GridNearAtomicFullUpdateRequest> mappings) {
     }
 
     /** {@inheritDoc} */
-    protected void map(AffinityTopologyVersion topVer) {
-        map(topVer, null);
+    @Override protected void map(AffinityTopologyVersion topVer, GridCacheVersion futVer) {
+        map(topVer, futVer, null);
     }
 
     /**
      * @param topVer Topology version.
+     * @param futVer Future ID.
      * @param remapKeys Keys to remap.
      */
-    void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> remapKeys) {
+    void map(AffinityTopologyVersion topVer,
+        GridCacheVersion futVer,
+        @Nullable Collection<KeyCacheObject> remapKeys) {
         Collection<ClusterNode> topNodes = CU.affinityNodes(cctx, topVer);
 
         if (F.isEmpty(topNodes)) {
@@ -620,14 +621,6 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
             return;
         }
 
-        Exception err = null;
-        GridNearAtomicFullUpdateRequest singleReq0 = null;
-        Map<UUID, GridNearAtomicFullUpdateRequest> mappings0 = null;
-
-        int size = keys.size();
-
-        GridCacheVersion futVer = cctx.versions().next(topVer);
-
         GridCacheVersion updVer;
 
         // Assign version on near node in CLOCK ordering mode even if fastMap is false.
@@ -644,6 +637,12 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
         else
             updVer = null;
 
+        Exception err = null;
+        GridNearAtomicFullUpdateRequest singleReq0 = null;
+        Map<UUID, GridNearAtomicFullUpdateRequest> mappings0 = null;
+
+        int size = keys.size();
+
         try {
             if (size == 1 && !fastMap) {
                 assert remapKeys == null || remapKeys.size() == 1;
@@ -676,12 +675,10 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
             }
 
             synchronized (mux) {
-                assert this.futVer == null : this;
-                assert this.topVer == AffinityTopologyVersion.ZERO : this;
+                assert this.futVer == futVer || (this.isDone() && this.error() != null);
+                assert this.topVer == topVer;
 
-                this.topVer = topVer;
                 this.updVer = updVer;
-                this.futVer = futVer;
 
                 resCnt = 0;
 
@@ -701,14 +698,6 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
             return;
         }
 
-        if (storeFuture()) {
-            if (!cctx.mvcc().addAtomicFuture(futVer, this)) {
-                assert isDone() : this;
-
-                return;
-            }
-        }
-
         // Optimize mapping for single key.
         if (singleReq0 != null)
             mapSingle(singleReq0.nodeId(), singleReq0);
@@ -725,7 +714,7 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
     /**
      * @return Future version.
      */
-    GridCacheVersion onFutureDone() {
+    private GridCacheVersion onFutureDone() {
         GridCacheVersion ver0;
 
         GridFutureAdapter<Void> fut0;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java
new file mode 100644
index 0000000000000..878cb17b5c3e2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.atomic;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.affinity.fair.FairAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CachePeekMode.BACKUP;
+import static org.apache.ignite.cache.CachePeekMode.PRIMARY;
+import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/** */
+public class AtomicPutAllChangingTopologyTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int NODES_CNT = 3;
+
+    /** */
+    public static final String CACHE_NAME = "test-cache";
+
+    /** */
+    private static final int CACHE_SIZE = 20_000;
+
+    /** */
+    private static volatile CountDownLatch FILLED_LATCH;
+
+    /**
+     * @return Cache configuration.
+     */
+    private CacheConfiguration<Integer, Integer> cacheConfig() {
+        return new CacheConfiguration<Integer, Integer>()
+            .setAtomicityMode(ATOMIC)
+            .setCacheMode(REPLICATED)
+            .setAffinity(new FairAffinityFunction(false, 1))
+            .setWriteSynchronizationMode(FULL_SYNC)
+            .setRebalanceMode(SYNC)
+            .setName(CACHE_NAME);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPutAllOnChangingTopology() throws Exception {
+        List<IgniteInternalFuture> futs = new LinkedList<>();
+
+        for (int i = 1; i < NODES_CNT; i++)
+            futs.add(startNodeAsync(i));
+
+        futs.add(startSeedNodeAsync());
+
+        boolean failed = false;
+
+        for (IgniteInternalFuture fut : futs) {
+            try {
+                fut.get();
+            }
+            catch (Throwable th) {
+                log.error("Check failed.", th);
+
+                failed = true;
+            }
+        }
+
+        if (failed)
+            throw new RuntimeException("Test Failed.");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        FILLED_LATCH = new CountDownLatch(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @return Future.
+     * @throws IgniteCheckedException If failed.
+     */
+    private IgniteInternalFuture startSeedNodeAsync() throws IgniteCheckedException {
+        return GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Boolean call() throws Exception {
+                Ignite node = startGrid(0);
+
+                log.info("Creating cache.");
+
+                IgniteCache<Integer, Integer> cache = node.getOrCreateCache(cacheConfig());
+
+                log.info("Created cache.");
+
+                Map<Integer, Integer> data = new HashMap<>(CACHE_SIZE);
+
+                for (int i = 0; i < CACHE_SIZE; i++)
+                    data.put(i, i);
+
+                log.info("Filling.");
+
+                cache.putAll(data);
+
+                log.info("Filled.");
+
+                FILLED_LATCH.countDown();
+
+                checkCacheState(node, cache);
+
+                return true;
+            }
+        });
+    }
+
+    /**
+     * @param nodeId Node index.
+     * @return Future.
+     * @throws IgniteCheckedException If failed.
+     */
+    private IgniteInternalFuture startNodeAsync(final int nodeId) throws IgniteCheckedException {
+        return GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Boolean call() throws Exception {
+                Ignite node = startGrid(nodeId);
+
+                log.info("Getting cache.");
+
+                IgniteCache<Integer, Integer> cache = node.getOrCreateCache(cacheConfig());
+
+                log.info("Got cache.");
+
+                FILLED_LATCH.await();
+
+                log.info("Got Filled.");
+
+                cache.put(1, nodeId);
+
+                checkCacheState(node, cache);
+
+                return true;
+            }
+        });
+    }
+
+    /**
+     * @param node Node.
+     * @param cache Cache.
+     * @throws Exception If failed.
+     */
+    private void checkCacheState(Ignite node, IgniteCache<Integer, Integer> cache) throws Exception {
+        int locSize = cache.localSize(PRIMARY, BACKUP);
+        int locSize2 = -1;
+
+        if (locSize != CACHE_SIZE) {
+            U.sleep(5000);
+
+            // Rechecking.
+            locSize2 = cache.localSize(PRIMARY, BACKUP);
+        }
+
+        assertEquals("Wrong cache size on node [node=" + node.configuration().getGridName() +
+            ", expected= " + CACHE_SIZE +
+            ", actual=" + locSize +
+            ", actual2=" + locSize2 + "]",
+            locSize, CACHE_SIZE);
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
index c9e507d33e11c..5bc67292b33de 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheTxNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridNearCacheTxNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteAtomicLongChangingTopologySelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.AtomicPutAllChangingTopologyTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicClientInvalidPartitionHandlingSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicClientRemoveFailureTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicInvalidPartitionHandlingSelfTest;
@@ -95,6 +96,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridCacheTxNodeFailureSelfTest.class);
         suite.addTestSuite(GridNearCacheTxNodeFailureSelfTest.class);
 
+        suite.addTestSuite(AtomicPutAllChangingTopologyTest.class);
+
         return suite;
     }
 }

From ac92bdb44ed0fb02893c1e7e9df4443a0a26d331 Mon Sep 17 00:00:00 2001
From: Ivan Veselovskiy <iveselovskiy@gridgain.com>
Date: Mon, 19 Dec 2016 11:57:00 +0300
Subject: [PATCH 085/446] IGNITE-3966: Hadoop: add Hadoop native library to
 Java arguments automatically. This closes #1320.

---
 bin/ignite.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/bin/ignite.sh b/bin/ignite.sh
index 05d2d5ff95101..efa37159cfc4e 100755
--- a/bin/ignite.sh
+++ b/bin/ignite.sh
@@ -127,6 +127,15 @@ if [ "${ENABLE_ASSERTIONS}" = "1" ]; then
     JVM_OPTS="${JVM_OPTS} -ea"
 fi
 
+#
+# If this is a Hadoop edition, and HADOOP_HOME set, add the native library location:
+#
+if [ -d "${IGNITE_HOME}/libs/ignite-hadoop/" ] && [ -n "${HADOOP_HOME}" ] && [ -d "${HADOOP_HOME}/lib/native/" ]; then
+   if [[ "${JVM_OPTS}${JVM_XOPTS}" != *-Djava.library.path=* ]]; then
+      JVM_OPTS="${JVM_OPTS} -Djava.library.path=${HADOOP_HOME}/lib/native/"
+   fi
+fi
+
 #
 # Set main class to start service (grid node by default).
 #

From 1776e9eb04895bbb870f7c6aa89ca96db2af31b7 Mon Sep 17 00:00:00 2001
From: tledkov-gridgain <tledkov@gridgain.com>
Date: Thu, 29 Dec 2016 11:14:10 +0300
Subject: [PATCH 086/446] IGNITE-4458: Hadoop: "striped" shuffle mode is
 default from now on. This closes #1390.

---
 .../ignite/internal/processors/hadoop/HadoopJobProperty.java    | 2 +-
 .../internal/processors/hadoop/shuffle/HadoopShuffleJob.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index 4398acd0be5fe..a3115bf1c064a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -105,7 +105,7 @@ public enum HadoopJobProperty {
     /**
      * Whether to stripe mapper output for remote reducers.
      * <p>
-     * Defaults to {@code false}.
+     * Defaults to {@code true}.
      */
     SHUFFLE_MAPPER_STRIPED_OUTPUT("ignite.shuffle.mapper.striped.output"),
 
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 1c546a1037d60..7713d6d627a2a 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -169,7 +169,7 @@ public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUn
         this.embedded = embedded;
 
         // No stripes for combiner.
-        boolean stripeMappers0 = get(job.info(), SHUFFLE_MAPPER_STRIPED_OUTPUT, false);
+        boolean stripeMappers0 = get(job.info(), SHUFFLE_MAPPER_STRIPED_OUTPUT, true);
 
         if (stripeMappers0) {
             if (job.info().hasCombiner()) {

From e7d781ee3221107d9a819dd70cb5776558a59e2a Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Thu, 5 Jan 2017 11:30:42 +0300
Subject: [PATCH 087/446] IGNITE-4516: Hadoop: implemented optional compression
 for remote shuffle output. This closes #1399.

---
 .../processors/hadoop/HadoopJobProperty.java  |  7 +++
 .../shuffle/HadoopDirectShuffleMessage.java   | 34 ++++++++++-
 .../hadoop/shuffle/HadoopShuffleJob.java      | 57 ++++++++++++++++---
 .../direct/HadoopDirectDataOutput.java        | 14 +++++
 .../direct/HadoopDirectDataOutputContext.java | 48 ++++++++++++++--
 .../direct/HadoopDirectDataOutputState.java   | 14 ++++-
 .../hadoop/impl/HadoopTeraSortTest.java       | 32 ++++++++++-
 7 files changed, 188 insertions(+), 18 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index a3115bf1c064a..60992d56e836d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -102,6 +102,13 @@ public enum HadoopJobProperty {
      */
     SHUFFLE_MSG_SIZE("ignite.shuffle.message.size"),
 
+    /**
+     * Whether shuffle message should be compressed with GZIP.
+     * <p>
+     * Defaults to {@code false}.
+     */
+    SHUFFLE_MSG_GZIP("ignite.shuffle.message.gzip"),
+
     /**
      * Whether to stripe mapper output for remote reducers.
      * <p>
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
index e81dc5f2fa9d0..f596100375093 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
@@ -57,6 +57,9 @@ public class HadoopDirectShuffleMessage implements Message, HadoopMessage {
     @GridDirectTransient
     private transient int bufLen;
 
+    /** Data length. */
+    private int dataLen;
+
     /**
      * Default constructor.
      */
@@ -72,8 +75,9 @@ public HadoopDirectShuffleMessage() {
      * @param cnt Count.
      * @param buf Buffer.
      * @param bufLen Buffer length.
+     * @param dataLen Data length.
      */
-    public HadoopDirectShuffleMessage(HadoopJobId jobId, int reducer, int cnt, byte[] buf, int bufLen) {
+    public HadoopDirectShuffleMessage(HadoopJobId jobId, int reducer, int cnt, byte[] buf, int bufLen, int dataLen) {
         assert jobId != null;
 
         this.jobId = jobId;
@@ -81,6 +85,7 @@ public HadoopDirectShuffleMessage(HadoopJobId jobId, int reducer, int cnt, byte[
         this.cnt = cnt;
         this.buf = buf;
         this.bufLen = bufLen;
+        this.dataLen = dataLen;
     }
 
     /**
@@ -111,6 +116,13 @@ public byte[] buffer() {
         return buf;
     }
 
+    /**
+     * @return Data length.
+     */
+    public int dataLength() {
+        return dataLen;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
         writer.setBuffer(buf);
@@ -147,6 +159,12 @@ public byte[] buffer() {
 
                 writer.incrementState();
 
+            case 4:
+                if (!writer.writeInt("dataLen", dataLen))
+                    return false;
+
+                writer.incrementState();
+
         }
 
         return true;
@@ -194,6 +212,14 @@ public byte[] buffer() {
 
                 reader.incrementState();
 
+            case 4:
+                dataLen = reader.readInt("dataLen");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
         }
 
         return reader.afterMessageRead(HadoopDirectShuffleMessage.class);
@@ -206,7 +232,7 @@ public byte[] buffer() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 4;
+        return 5;
     }
 
     /** {@inheritDoc} */
@@ -222,6 +248,8 @@ public byte[] buffer() {
         out.writeInt(cnt);
 
         U.writeByteArray(out, buf);
+
+        out.writeInt(dataLen);
     }
 
     /** {@inheritDoc} */
@@ -234,6 +262,8 @@ public byte[] buffer() {
 
         buf = U.readByteArray(in);
         bufLen = buf != null ? buf.length : 0;
+
+        dataLen = in.readInt();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 7713d6d627a2a..318ead36f9376 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -56,6 +56,8 @@
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -63,10 +65,12 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReferenceArray;
+import java.util.zip.GZIPInputStream;
 
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.PARTITION_HASHMAP_SIZE;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_JOB_THROTTLE;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MSG_GZIP;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MSG_SIZE;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_REDUCER_NO_SORTING;
 import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.get;
@@ -78,6 +82,9 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
     /** */
     private static final int DFLT_SHUFFLE_MSG_SIZE = 1024 * 1024;
 
+    /** */
+    private static final boolean DFLT_SHUFFLE_MSG_GZIP = false;
+
     /** */
     private final HadoopJob job;
 
@@ -130,6 +137,9 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
     /** Message size. */
     private final int msgSize;
 
+    /** Whether to GZIP shuffle messages. */
+    private final boolean msgGzip;
+
     /** Whether to strip mappers for remote execution. */
     private final boolean stripeMappers;
 
@@ -190,6 +200,7 @@ public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUn
         stripeMappers = stripeMappers0;
 
         msgSize = get(job.info(), SHUFFLE_MSG_SIZE, DFLT_SHUFFLE_MSG_SIZE);
+        msgGzip = get(job.info(), SHUFFLE_MSG_GZIP, DFLT_SHUFFLE_MSG_GZIP);
 
         locReducersCtx = new AtomicReferenceArray<>(totalReducerCnt);
 
@@ -360,22 +371,26 @@ public void onShuffleMessage(T src, HadoopShuffleMessage msg) throws IgniteCheck
      * @throws IgniteCheckedException Exception.
      */
     public void onDirectShuffleMessage(T src, HadoopDirectShuffleMessage msg) throws IgniteCheckedException {
-        assert msg.buffer() != null;
+        byte[] buf = extractBuffer(msg);
 
-        HadoopTaskContext taskCtx = locReducersCtx.get(msg.reducer()).get();
+        assert buf != null;
+
+        int rdc = msg.reducer();
+
+        HadoopTaskContext taskCtx = locReducersCtx.get(rdc).get();
 
         HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
 
-        perfCntr.onShuffleMessage(msg.reducer(), U.currentTimeMillis());
+        perfCntr.onShuffleMessage(rdc, U.currentTimeMillis());
 
-        HadoopMultimap map = getOrCreateMap(locMaps, msg.reducer());
+        HadoopMultimap map = getOrCreateMap(locMaps, rdc);
 
         HadoopSerialization keySer = taskCtx.keySerialization();
         HadoopSerialization valSer = taskCtx.valueSerialization();
 
         // Add data from message to the map.
         try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
-            HadoopDirectDataInput in = new HadoopDirectDataInput(msg.buffer());
+            HadoopDirectDataInput in = new HadoopDirectDataInput(buf);
 
             Object key = null;
             Object val = null;
@@ -392,6 +407,31 @@ public void onDirectShuffleMessage(T src, HadoopDirectShuffleMessage msg) throws
             sendFinishResponse(src, msg.jobId());
     }
 
+    /**
+     * Extract buffer from direct shuffle message.
+     *
+     * @param msg Message.
+     * @return Buffer.
+     */
+    private byte[] extractBuffer(HadoopDirectShuffleMessage msg) throws IgniteCheckedException {
+        if (msgGzip) {
+            byte[] res = new byte[msg.dataLength()];
+
+            try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(msg.buffer()), res.length)) {
+                int len = in.read(res, 0, res.length);
+
+                assert len == res.length;
+            }
+            catch (IOException e) {
+                throw new IgniteCheckedException("Failed to uncompress direct shuffle message.", e);
+            }
+
+            return res;
+        }
+        else
+            return msg.buffer();
+    }
+
     /**
      * @param ack Shuffle ack.
      */
@@ -595,7 +635,8 @@ private void collectUpdatesAndSend(int rmtMapIdx, boolean flush) throws IgniteCh
      * @param rmtDirectCtx Remote direct context.
      * @param reset Whether to perform reset.
      */
-    private void sendShuffleMessage(int rmtMapIdx, @Nullable HadoopDirectDataOutputContext rmtDirectCtx, boolean reset) {
+    private void sendShuffleMessage(int rmtMapIdx, @Nullable HadoopDirectDataOutputContext rmtDirectCtx,
+        boolean reset) {
         if (rmtDirectCtx == null)
             return;
 
@@ -612,7 +653,7 @@ private void sendShuffleMessage(int rmtMapIdx, @Nullable HadoopDirectDataOutputC
             rmtDirectCtx.reset();
 
         HadoopDirectShuffleMessage msg = new HadoopDirectShuffleMessage(job.id(), rmtRdcIdx, cnt,
-            state.buffer(), state.bufferLength());
+            state.buffer(), state.bufferLength(), state.dataLength());
 
         T nodeId = reduceAddrs[rmtRdcIdx];
 
@@ -983,7 +1024,7 @@ private PartitionedOutput(HadoopTaskContext taskCtx) throws IgniteCheckedExcepti
                     HadoopDirectDataOutputContext rmtDirectCtx = rmtDirectCtxs[idx];
 
                     if (rmtDirectCtx == null) {
-                        rmtDirectCtx = new HadoopDirectDataOutputContext(msgSize, taskCtx);
+                        rmtDirectCtx = new HadoopDirectDataOutputContext(msgSize, msgGzip, taskCtx);
 
                         rmtDirectCtxs[idx] = rmtDirectCtx;
                     }
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java
index 151e5528092a4..5840994d52772 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java
@@ -169,6 +169,13 @@ public byte[] buffer() {
         return buf;
     }
 
+    /**
+     * @return Buffer length (how much memory is allocated).
+     */
+    public int bufferLength() {
+        return bufSize;
+    }
+
     /**
      * @return Position.
      */
@@ -183,6 +190,13 @@ public boolean readyForFlush() {
         return pos >= flushSize;
     }
 
+    /**
+     * Reset the stream.
+     */
+    public void reset() {
+        pos = 0;
+    }
+
     /**
      * Ensure that the given amount of bytes is available within the stream, then shift the position.
      *
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java
index bc70ef3dcebd7..454278ba617f3 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java
@@ -18,16 +18,29 @@
 package org.apache.ignite.internal.processors.hadoop.shuffle.direct;
 
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
 
+import java.io.IOException;
+import java.util.zip.GZIPOutputStream;
+
 /**
  * Hadoop data output context for direct communication.
  */
 public class HadoopDirectDataOutputContext {
+    /** Initial allocation size for GZIP output. We start with very low value, but then it will grow if needed. */
+    private static final int GZIP_OUT_MIN_ALLOC_SIZE = 1024;
+
+    /** GZIP buffer size. We should remove it when we implement efficient direct GZIP output. */
+    private static final int GZIP_BUFFER_SIZE = 8096;
+
     /** Flush size. */
     private final int flushSize;
 
+    /** Whether to perform GZIP. */
+    private final boolean gzip;
+
     /** Key serialization. */
     private final HadoopSerialization keySer;
 
@@ -37,6 +50,9 @@ public class HadoopDirectDataOutputContext {
     /** Data output. */
     private HadoopDirectDataOutput out;
 
+    /** Data output for GZIP. */
+    private HadoopDirectDataOutput gzipOut;
+
     /** Number of keys written. */
     private int cnt;
 
@@ -44,17 +60,22 @@ public class HadoopDirectDataOutputContext {
      * Constructor.
      *
      * @param flushSize Flush size.
+     * @param gzip Whether to perform GZIP.
      * @param taskCtx Task context.
      * @throws IgniteCheckedException If failed.
      */
-    public HadoopDirectDataOutputContext(int flushSize, HadoopTaskContext taskCtx)
+    public HadoopDirectDataOutputContext(int flushSize, boolean gzip, HadoopTaskContext taskCtx)
         throws IgniteCheckedException {
         this.flushSize = flushSize;
+        this.gzip = gzip;
 
         keySer = taskCtx.keySerialization();
         valSer = taskCtx.valueSerialization();
 
         out = new HadoopDirectDataOutput(flushSize);
+
+        if (gzip)
+            gzipOut = new HadoopDirectDataOutput(Math.max(flushSize / 8, GZIP_OUT_MIN_ALLOC_SIZE));
     }
 
     /**
@@ -85,16 +106,35 @@ public int count() {
      * @return State.
      */
     public HadoopDirectDataOutputState state() {
-        return new HadoopDirectDataOutputState(out.buffer(), out.position());
+        if (gzip) {
+            try {
+                try (GZIPOutputStream gzip = new GZIPOutputStream(gzipOut, GZIP_BUFFER_SIZE)) {
+                    gzip.write(out.buffer(), 0, out.position());
+                }
+
+                return new HadoopDirectDataOutputState(gzipOut.buffer(), gzipOut.position(), out.position());
+            }
+            catch (IOException e) {
+                throw new IgniteException("Failed to compress.", e);
+            }
+        }
+        else
+            return new HadoopDirectDataOutputState(out.buffer(), out.position(), out.position());
     }
 
     /**
      * Reset buffer.
      */
     public void reset() {
-        int allocSize = Math.max(flushSize, out.position());
+        if (gzip) {
+            // In GZIP mode we do not expose normal output to the outside. Hence, no need for reallocation, just reset.
+            out.reset();
+
+            gzipOut = new HadoopDirectDataOutput(gzipOut.bufferLength());
+        }
+        else
+            out = new HadoopDirectDataOutput(flushSize, out.bufferLength());
 
-        out = new HadoopDirectDataOutput(flushSize, allocSize);
         cnt = 0;
     }
 }
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java
index a9c12e38b10fe..cadde7a4d0a39 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java
@@ -27,15 +27,20 @@ public class HadoopDirectDataOutputState {
     /** Buffer length. */
     private final int bufLen;
 
+    /** Data length. */
+    private final int dataLen;
+
     /**
      * Constructor.
      *
      * @param buf Buffer.
      * @param bufLen Buffer length.
+     * @param dataLen Data length.
      */
-    public HadoopDirectDataOutputState(byte[] buf, int bufLen) {
+    public HadoopDirectDataOutputState(byte[] buf, int bufLen, int dataLen) {
         this.buf = buf;
         this.bufLen = bufLen;
+        this.dataLen = dataLen;
     }
 
     /**
@@ -51,4 +56,11 @@ public byte[] buffer() {
     public int bufferLength() {
         return bufLen;
     }
+
+    /**
+     * @return Original data length.
+     */
+    public int dataLength() {
+        return dataLen;
+    }
 }
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
index b1fa91f0e4707..d2371807ebbbc 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
@@ -137,8 +137,10 @@ protected String getUser() {
 
     /**
      * Does actual test TeraSort job Through Ignite API
+     *
+     * @param gzip Whether to use GZIP.
      */
-    protected final void teraSort() throws Exception {
+    protected final void teraSort(boolean gzip) throws Exception {
         System.out.println("TeraSort ===============================================================");
 
         getFileSystem().delete(new Path(sortOutDir), true);
@@ -164,6 +166,10 @@ protected final void teraSort() throws Exception {
         jobConf.set("mapred.max.split.size", String.valueOf(splitSize));
 
         jobConf.setBoolean(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), true);
+        jobConf.setInt(HadoopJobProperty.SHUFFLE_MSG_SIZE.propertyName(), 4096);
+
+        if (gzip)
+            jobConf.setBoolean(HadoopJobProperty.SHUFFLE_MSG_GZIP.propertyName(), true);
 
         jobConf.set(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(),
             TextPartiallyRawComparator.class.getName());
@@ -347,12 +353,32 @@ private void teraValidate() throws Exception {
 
     /**
      * Runs generate/sort/validate phases of the terasort sample.
-     * @throws Exception
+     *
+     * @throws Exception If failed.
      */
     public void testTeraSort() throws Exception {
+        checkTeraSort(false);
+    }
+
+    /**
+     * Runs generate/sort/validate phases of the terasort sample.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTeraSortGzip() throws Exception {
+        checkTeraSort(true);
+    }
+
+    /**
+     * Check terasort.
+     *
+     * @param gzip GZIP flag.
+     * @throws Exception If failed.
+     */
+    private void checkTeraSort(boolean gzip) throws Exception {
         teraGenerate();
 
-        teraSort();
+        teraSort(gzip);
 
         teraValidate();
     }

From b6005b07b3e28907c8ff5cb6ebcce25bdb23bb48 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Thu, 5 Jan 2017 14:48:06 +0300
Subject: [PATCH 088/446] IGNITE-4461: Hadoop: added automatic resolution of
 "raw" comparator for Text class.

---
 .../hadoop/impl/v2/HadoopV2TaskContext.java   | 64 +++++++++++++------
 1 file changed, 46 insertions(+), 18 deletions(-)

diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
index e9cae1c068a52..d328550a7a2f7 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.hadoop.io.PartiallyRawComparator;
+import org.apache.ignite.hadoop.io.TextPartiallyRawComparator;
 import org.apache.ignite.internal.processors.hadoop.HadoopClassLoader;
 import org.apache.ignite.internal.processors.hadoop.HadoopCommonUtils;
 import org.apache.ignite.internal.processors.hadoop.HadoopExternalSplit;
@@ -76,6 +77,8 @@
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.Callable;
 
@@ -99,6 +102,9 @@ public class HadoopV2TaskContext extends HadoopTaskContext {
     private static final HadoopLazyConcurrentMap<FsCacheKey, FileSystem> fsMap
         = createHadoopLazyConcurrentMap();
 
+    /** Default partial comparator mappings. */
+    private static final Map<String, String> PARTIAL_COMPARATORS = new HashMap<>();
+
     /**
      * This method is called with reflection upon Job finish with class loader of each task.
      * This will clean up all the Fs created for specific task.
@@ -111,24 +117,6 @@ public static void close() throws IgniteCheckedException {
         fsMap.close();
     }
 
-    /**
-     * Check for combiner grouping support (available since Hadoop 2.3).
-     */
-    static {
-        boolean ok;
-
-        try {
-            JobContext.class.getDeclaredMethod("getCombinerKeyGroupingComparator");
-
-            ok = true;
-        }
-        catch (NoSuchMethodException ignore) {
-            ok = false;
-        }
-
-        COMBINE_KEY_GROUPING_SUPPORTED = ok;
-    }
-
     /** Flag is set if new context-object code is used for running the mapper. */
     private final boolean useNewMapper;
 
@@ -153,6 +141,23 @@ public static void close() throws IgniteCheckedException {
     /** Counters for task. */
     private final HadoopCounters cntrs = new HadoopCountersImpl();
 
+    static {
+        boolean ok;
+
+        try {
+            JobContext.class.getDeclaredMethod("getCombinerKeyGroupingComparator");
+
+            ok = true;
+        }
+        catch (NoSuchMethodException ignore) {
+            ok = false;
+        }
+
+        COMBINE_KEY_GROUPING_SUPPORTED = ok;
+
+        PARTIAL_COMPARATORS.put(Text.class.getName(), TextPartiallyRawComparator.class.getName());
+    }
+
     /**
      * @param taskInfo Task info.
      * @param job Job.
@@ -181,6 +186,8 @@ public HadoopV2TaskContext(HadoopTaskInfo taskInfo, HadoopJob job, HadoopJobId j
             // For map-reduce jobs prefer local writes.
             jobConf.setBooleanIfUnset(PARAM_IGFS_PREFER_LOCAL_WRITES, true);
 
+            initializePartiallyRawComparator(jobConf);
+
             jobCtx = new JobContextImpl(jobConf, new JobID(jobId.globalId().toString(), jobId.localId()));
 
             useNewMapper = jobConf.getUseNewMapper();
@@ -447,6 +454,7 @@ private HadoopSerialization getSerialization(Class<?> cls, Configuration jobConf
     }
 
     /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
     @Override public Comparator<Object> groupComparator() {
         Comparator<?> res;
 
@@ -581,4 +589,24 @@ private Object readExternalSplit(HadoopExternalSplit split) throws IgniteChecked
             throw new IgniteCheckedException(e);
         }
     }
+
+    /**
+     * Try initializing partially raw comparator for job.
+     *
+     * @param conf Configuration.
+     */
+    private void initializePartiallyRawComparator(JobConf conf) {
+        String clsName = conf.get(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(), null);
+
+        if (clsName == null) {
+            Class keyCls = conf.getMapOutputKeyClass();
+
+            if (keyCls != null) {
+                clsName = PARTIAL_COMPARATORS.get(keyCls.getName());
+
+                if (clsName != null)
+                    conf.set(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(), clsName);
+            }
+        }
+    }
 }
\ No newline at end of file

From 626f1d5614930bca0ab744a6049e5f83f321d46a Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 23 Dec 2016 12:18:07 +0300
Subject: [PATCH 089/446] IGNITE-4142: Fixed assertion while update metrics.
 This closes #1372.

---
 .../java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java    | 2 +-
 .../java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 8928f28d4fd27..0f5f74181516e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -1920,7 +1920,7 @@ private void processHeartbeatMessage(TcpDiscoveryHeartbeatMessage msg) {
 
                         TcpDiscoveryHeartbeatMessage.MetricsSet metricsSet = e.getValue();
 
-                        Map<Integer, CacheMetrics> cacheMetrics = msg.hasCacheMetrics() ?
+                        Map<Integer, CacheMetrics> cacheMetrics = msg.hasCacheMetrics(nodeId) ?
                             msg.cacheMetrics().get(nodeId) : Collections.<Integer, CacheMetrics>emptyMap();
 
                         updateMetrics(nodeId, metricsSet.metrics(), cacheMetrics, tstamp);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 204b685984be3..c79133379c267 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -4927,7 +4927,7 @@ private void processHeartbeatMessage(TcpDiscoveryHeartbeatMessage msg) {
 
                         TcpDiscoveryHeartbeatMessage.MetricsSet metricsSet = e.getValue();
 
-                        Map<Integer, CacheMetrics> cacheMetrics = msg.hasCacheMetrics() ?
+                        Map<Integer, CacheMetrics> cacheMetrics = msg.hasCacheMetrics(nodeId) ?
                             msg.cacheMetrics().get(nodeId) : Collections.<Integer, CacheMetrics>emptyMap();
 
                         updateMetrics(nodeId, metricsSet.metrics(), cacheMetrics, tstamp);

From 07752ec85dc3bf18297b728f9a31185d44ad3687 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 11 Jan 2017 13:51:23 +0300
Subject: [PATCH 090/446] Improved cache.get benchmarks.

---
 .../cache/IgniteGetAllBenchmark.java          | 42 +++++++++++++++++++
 .../yardstick/cache/IgniteGetBenchmark.java   | 38 ++++++++++++++++-
 2 files changed, 79 insertions(+), 1 deletion(-)
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetAllBenchmark.java

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetAllBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetAllBenchmark.java
new file mode 100644
index 0000000000000..2f76b7cd30344
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetAllBenchmark.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Ignite benchmark that performs getAll operations.
+ */
+public class IgniteGetAllBenchmark extends IgniteGetBenchmark {
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        Set<Integer> keys = U.newHashSet(args.batch());
+
+        while (keys.size() < args.batch()) {
+            int key = nextRandom(args.range());
+
+            keys.add(key);
+        }
+
+        cache.getAll(keys);
+
+        return true;
+    }
+}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
index 8a86e2f4bf6c3..918f57103fce9 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
@@ -19,11 +19,47 @@
 
 import java.util.Map;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.yardstick.cache.model.SampleValue;
+import org.yardstickframework.BenchmarkConfiguration;
+
+import static org.yardstickframework.BenchmarkUtils.println;
 
 /**
  * Ignite benchmark that performs get operations.
  */
 public class IgniteGetBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
+    /** */
+    private static final String CACHE_NAME = "atomic";
+
+    /** {@inheritDoc} */
+    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        if (args.preloadAmount() > args.range())
+            throw new IllegalArgumentException("Preloading amount (\"-pa\", \"--preloadAmount\") " +
+                "must by less then the range (\"-r\", \"--range\").");
+
+        println(cfg, "Loading data...");
+
+        long start = System.nanoTime();
+
+        try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(CACHE_NAME)) {
+            for (int i = 0; i < args.preloadAmount(); i++) {
+                dataLdr.addData(i, new SampleValue(i));
+
+                if (i % 100000 == 0) {
+                    if (Thread.currentThread().isInterrupted())
+                        break;
+
+                    println("Loaded entries: " + i);
+                }
+            }
+        }
+
+        println(cfg, "Finished populating query data in " + ((System.nanoTime() - start) / 1_000_000) + " ms.");
+    }
+
     /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
         int key = nextRandom(args.range());
@@ -35,6 +71,6 @@ public class IgniteGetBenchmark extends IgniteCacheAbstractBenchmark<Integer, Ob
 
     /** {@inheritDoc} */
     @Override protected IgniteCache<Integer, Object> cache() {
-        return ignite().cache("atomic");
+        return ignite().cache(CACHE_NAME);
     }
 }

From 660cd1e072496c323f5bdfc797f2a7cb7d74480d Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 11 Jan 2017 17:34:46 +0300
Subject: [PATCH 091/446] Added cache.getAll benchmarks for offheap cache.

---
 .../cache/IgniteGetAllOffHeapBenchmark.java   | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetAllOffHeapBenchmark.java

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetAllOffHeapBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetAllOffHeapBenchmark.java
new file mode 100644
index 0000000000000..face4bda30d77
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetAllOffHeapBenchmark.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import org.apache.ignite.IgniteCache;
+
+/**
+ *
+ */
+public class IgniteGetAllOffHeapBenchmark extends IgniteGetAllBenchmark {
+    /** {@inheritDoc} */
+    @Override protected IgniteCache<Integer, Object> cache() {
+        return ignite().cache("atomic-offheap");
+    }
+}

From 074a4a08012822f3404807015bc63052c20e7439 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 11 Jan 2017 17:54:55 +0300
Subject: [PATCH 092/446] Added cache.getAll benchmarks for offheap cache.

---
 .../ignite/yardstick/cache/IgniteGetBenchmark.java    | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
index 918f57103fce9..9dd5058b4d290 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
@@ -29,9 +29,6 @@
  * Ignite benchmark that performs get operations.
  */
 public class IgniteGetBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
-    /** */
-    private static final String CACHE_NAME = "atomic";
-
     /** {@inheritDoc} */
     @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
         super.setUp(cfg);
@@ -40,11 +37,13 @@ public class IgniteGetBenchmark extends IgniteCacheAbstractBenchmark<Integer, Ob
             throw new IllegalArgumentException("Preloading amount (\"-pa\", \"--preloadAmount\") " +
                 "must by less then the range (\"-r\", \"--range\").");
 
-        println(cfg, "Loading data...");
+        String cacheName = cache().getName();
+
+        println(cfg, "Loading data for cache: " + cacheName);
 
         long start = System.nanoTime();
 
-        try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(CACHE_NAME)) {
+        try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(cacheName)) {
             for (int i = 0; i < args.preloadAmount(); i++) {
                 dataLdr.addData(i, new SampleValue(i));
 
@@ -71,6 +70,6 @@ public class IgniteGetBenchmark extends IgniteCacheAbstractBenchmark<Integer, Ob
 
     /** {@inheritDoc} */
     @Override protected IgniteCache<Integer, Object> cache() {
-        return ignite().cache(CACHE_NAME);
+        return ignite().cache("atomic");
     }
 }

From 1820eb3135c44d6768d53b95d69c2a93c7759b6f Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 11 Jan 2017 18:09:42 +0300
Subject: [PATCH 093/446] Added cache.putAll benchmarks for offheap cache.

---
 .../cache/IgnitePutAllOffHeapBenchmark.java   | 30 +++++++++++++++++++
 .../cache/IgnitePutAllOffHeapTxBenchmark.java | 30 +++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapBenchmark.java
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapTxBenchmark.java

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapBenchmark.java
new file mode 100644
index 0000000000000..4ac67fde2bdba
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapBenchmark.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import org.apache.ignite.IgniteCache;
+
+/**
+ *
+ */
+public class IgnitePutAllOffHeapBenchmark extends IgnitePutAllBenchmark {
+    /** {@inheritDoc} */
+    @Override protected IgniteCache<Integer, Object> cache() {
+        return ignite().cache("atomic-offheap");
+    }
+}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapTxBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapTxBenchmark.java
new file mode 100644
index 0000000000000..559f8e80cf4bb
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapTxBenchmark.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import org.apache.ignite.IgniteCache;
+
+/**
+ *
+ */
+public class IgnitePutAllOffHeapTxBenchmark extends IgnitePutAllTxBenchmark {
+    /** {@inheritDoc} */
+    @Override protected IgniteCache<Integer, Object> cache() {
+        return ignite().cache("tx-offheap");
+    }
+}

From e04d83c4911bbd3e18f03ee1e80a99cd49fae87f Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 11 Jan 2017 18:15:53 +0300
Subject: [PATCH 094/446] Added cache.putAll benchmarks for offheap cache.

---
 ...HeapTxBenchmark.java => IgnitePutAllTxOffHeapBenchmark.java} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/{IgnitePutAllOffHeapTxBenchmark.java => IgnitePutAllTxOffHeapBenchmark.java} (94%)

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapTxBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxOffHeapBenchmark.java
similarity index 94%
rename from modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapTxBenchmark.java
rename to modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxOffHeapBenchmark.java
index 559f8e80cf4bb..1578b90827b81 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllOffHeapTxBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxOffHeapBenchmark.java
@@ -22,7 +22,7 @@
 /**
  *
  */
-public class IgnitePutAllOffHeapTxBenchmark extends IgnitePutAllTxBenchmark {
+public class IgnitePutAllTxOffHeapBenchmark extends IgnitePutAllTxBenchmark {
     /** {@inheritDoc} */
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("tx-offheap");

From fe424591161595de1151a29cf1cdeb50456c3e39 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 12 Jan 2017 17:43:21 +0300
Subject: [PATCH 095/446] Minors

---
 .../IgniteCacheExpiryPolicyAbstractTest.java  | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index 794519a0dae48..4368a1533f6d3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -36,24 +36,30 @@
 import javax.cache.expiry.ModifiedExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheMemoryMode;
 import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
+import org.apache.ignite.internal.processors.cache.GridCacheTestStore;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.PAX;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
@@ -1002,6 +1008,42 @@ public void testNearAccess() throws Exception {
         checkTtl(key, 62_000L, true);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNearExpiresWithCacheStore() throws Exception {
+        factory = CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 1));
+
+        nearCache = true;
+
+        startGridsMultiThreaded(gridCount());
+
+        IgniteConfiguration clientCfg = getConfiguration("client").setClientMode(true);
+
+        ((TcpDiscoverySpi)clientCfg.getDiscoverySpi()).setForceServerMode(false);
+
+        Ignite client = startGrid("client", clientCfg);
+
+        CacheConfiguration ccfg = cacheConfiguration("testCache");
+
+//        ccfg.setExpiryPolicyFactory( CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 1)));
+
+        IgniteCache<Object, Object> cache = client.getOrCreateCache(ccfg);
+
+        Integer key = 1;
+
+        cache.put(key, 1);
+
+        assertEquals(1, cache.localPeek(key, CachePeekMode.NEAR));
+        assertEquals(1, cache.get(key));
+
+        waitExpired(key);
+
+        for(int i = 0; i < gridCount(); i++)
+            assertNull(jcache(i).localPeek(key, CachePeekMode.BACKUP, CachePeekMode.PRIMARY));
+
+        assertEquals(null, cache.get(key));
+    }
     /**
      * @return Test keys.
      * @throws Exception If failed.

From cf9f0a79da7540dc40c3ac860a94158e87e2d7ec Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 12 Jan 2017 17:50:02 +0300
Subject: [PATCH 096/446] Minors

---
 .../cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java        | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index 4368a1533f6d3..9739350d10617 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -1026,6 +1026,7 @@ public void testNearExpiresWithCacheStore() throws Exception {
 
         CacheConfiguration ccfg = cacheConfiguration("testCache");
 
+        ccfg.setCacheStoreFactory(FactoryBuilder.factoryOf(GridCacheTestStore.class));
 //        ccfg.setExpiryPolicyFactory( CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 1)));
 
         IgniteCache<Object, Object> cache = client.getOrCreateCache(ccfg);

From 55f7594fd0595a4269c3972446ba4ebe30c12442 Mon Sep 17 00:00:00 2001
From: Alexandr Kuramshin <akuramshin@gridgain.com>
Date: Fri, 13 Jan 2017 12:26:39 +0300
Subject: [PATCH 097/446] ignite-4293 Do not need store deserialized value for
 BinaryMarshaller

(cherry picked from commit d10946b)
---
 .../cacheobject/IgniteCacheObjectProcessorImpl.java         | 5 ++---
 .../org/apache/ignite/cache/store/jdbc/model/Person.java    | 2 +-
 .../processors/cache/CacheEntryProcessorCopySelfTest.java   | 6 ++++--
 .../processors/cache/GridCacheBasicStoreAbstractTest.java   | 2 +-
 .../dht/GridCacheDhtEvictionsDisabledSelfTest.java          | 5 +----
 5 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java
index 208ec62448176..614c6129929db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java
@@ -244,9 +244,8 @@ protected final int partition(CacheObjectContext ctx, @Nullable GridCacheContext
 
         CacheMemoryMode memMode = ccfg.getMemoryMode();
 
-        boolean storeVal = ctx.config().isPeerClassLoadingEnabled() ||
-            GridQueryProcessor.isEnabled(ccfg) ||
-            !ccfg.isCopyOnRead();
+        boolean storeVal = !ccfg.isCopyOnRead() || (!isBinaryEnabled(ccfg) &&
+            (GridQueryProcessor.isEnabled(ccfg) || ctx.config().isPeerClassLoadingEnabled()));
 
         CacheObjectContext res = new CacheObjectContext(ctx,
             ccfg.getName(),
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java
index ddf309b1e6a0c..52ddfc8ecdcbb 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java
@@ -196,7 +196,7 @@ public void setSalary(Integer salary) {
     @Override public String toString() {
         return "Person [id=" + id +
             ", orgId=" + orgId +
-            ", birthday=" + birthday.getTime() +
+            ", birthday=" + (birthday == null ? null : birthday.getTime()) +
             ", name=" + name +
             "]";
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEntryProcessorCopySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEntryProcessorCopySelfTest.java
index 21395e61c75e6..f44889ba53dfd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEntryProcessorCopySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEntryProcessorCopySelfTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.cache.CacheEntryProcessor;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -93,8 +94,9 @@ private void doTestMutableEntry(boolean p2pEnabled) throws Exception {
             doTest(true, false, OLD_VAL, 1);
 
             // One deserialization due to copyOnRead == true.
-            // Additional deserialization in case p2p enabled due to storeValue == true on update entry.
-            doTest(true, true, NEW_VAL, p2pEnabled ? 2 : 1);
+            // Additional deserialization in case p2p enabled and not BinaryMarshaller due to storeValue == true on update entry.
+            doTest(true, true, NEW_VAL, p2pEnabled &&
+                !(grid.configuration().getMarshaller() instanceof BinaryMarshaller) ? 2 : 1);
 
             // No deserialization.
             doTest(false, false, NEW_VAL, 0);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicStoreAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicStoreAbstractTest.java
index 8ddd737aaec33..026b6183bb208 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicStoreAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicStoreAbstractTest.java
@@ -571,7 +571,7 @@ public void testReload() throws Exception {
 
             assert cached != null;
 
-            assert cached == val : "Cached value mismatch [expected=" + val + ", cached=" + cached + ']';
+            assert cached.equals(val) : "Cached value mismatch [expected=" + val + ", cached=" + cached + ']';
 
             // Make sure that value is coming from cache, not from store.
             checkLastMethod(null);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEvictionsDisabledSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEvictionsDisabledSelfTest.java
index 3f3f84fa69807..e8a6cfb87f79a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEvictionsDisabledSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEvictionsDisabledSelfTest.java
@@ -117,10 +117,7 @@ private void checkNodes(Ignite g) throws Exception {
             assertNotNull(v1);
             assertNotNull(v2);
 
-            if (affinity(cache).mapKeyToNode(key).isLocal())
-                assertSame(v1, v2);
-            else
-                assertEquals(v1, v2);
+            assertEquals(v1, v2);
         }
     }
 }
\ No newline at end of file

From 1665a615030201a7c9a51fd479868c3533b103b5 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Fri, 30 Dec 2016 13:41:34 +0300
Subject: [PATCH 098/446] IGNITE-4424 REPLICATED cache isn't synced across
 nodes

---
 .../GridNearAtomicAbstractUpdateFuture.java   |  34 ++-
 .../GridNearAtomicSingleUpdateFuture.java     |  44 ++--
 .../atomic/GridNearAtomicUpdateFuture.java    |  57 ++---
 .../AtomicPutAllChangingTopologyTest.java     | 212 ++++++++++++++++++
 .../IgniteCacheFailoverTestSuite.java         |   3 +
 5 files changed, 284 insertions(+), 66 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
index 2fbabaa54fc9c..c92e0f5ec5b42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
@@ -212,14 +212,18 @@ public void map() {
             // Cannot remap.
             remapCnt = 1;
 
-            map(topVer);
+            GridCacheVersion futVer = addAtomicFuture(topVer);
+
+            if (futVer != null)
+                map(topVer, futVer);
         }
     }
 
     /**
      * @param topVer Topology version.
+     * @param futVer Future version
      */
-    protected abstract void map(AffinityTopologyVersion topVer);
+    protected abstract void map(AffinityTopologyVersion topVer, GridCacheVersion futVer);
 
     /**
      * Maps future on ready topology.
@@ -302,7 +306,7 @@ protected void mapSingle(UUID nodeId, GridNearAtomicAbstractUpdateRequest req) {
      * @param req Request.
      * @param e Error.
      */
-    protected void onSendError(GridNearAtomicAbstractUpdateRequest req, IgniteCheckedException e) {
+    protected final void onSendError(GridNearAtomicAbstractUpdateRequest req, IgniteCheckedException e) {
         synchronized (mux) {
             GridNearAtomicUpdateResponse res = new GridNearAtomicUpdateResponse(cctx.cacheId(),
                 req.nodeId(),
@@ -314,4 +318,28 @@ protected void onSendError(GridNearAtomicAbstractUpdateRequest req, IgniteChecke
             onResult(req.nodeId(), res, true);
         }
     }
+
+    /**
+     * Adds future prevents topology change before operation complete.
+     * Should be invoked before topology lock released.
+     *
+     * @param topVer Topology version.
+     * @return Future version in case future added.
+     */
+    protected final GridCacheVersion addAtomicFuture(AffinityTopologyVersion topVer) {
+        GridCacheVersion futVer = cctx.versions().next(topVer);
+
+        synchronized (mux) {
+            assert this.futVer == null : this;
+            assert this.topVer == AffinityTopologyVersion.ZERO : this;
+
+            this.topVer = topVer;
+            this.futVer = futVer;
+        }
+
+        if (storeFuture() && !cctx.mvcc().addAtomicFuture(futVer, this))
+            return null;
+
+        return futVer;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
index bd231cf0003d8..7376affca4455 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
@@ -348,14 +348,7 @@ else if (res.error() != null) {
                 @Override public void apply(final IgniteInternalFuture<AffinityTopologyVersion> fut) {
                     cctx.kernalContext().closure().runLocalSafe(new Runnable() {
                         @Override public void run() {
-                            try {
-                                AffinityTopologyVersion topVer = fut.get();
-
-                                map(topVer);
-                            }
-                            catch (IgniteCheckedException e) {
-                                onDone(e);
-                            }
+                            mapOnTopology();
                         }
                     });
                 }
@@ -388,7 +381,9 @@ private void updateNear(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicU
     @Override protected void mapOnTopology() {
         cache.topology().readLock();
 
-        AffinityTopologyVersion topVer = null;
+        AffinityTopologyVersion topVer;
+
+        GridCacheVersion futVer;
 
         try {
             if (cache.topology().stopping()) {
@@ -410,6 +405,8 @@ private void updateNear(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicU
                 }
 
                 topVer = fut.topologyVersion();
+
+                futVer = addAtomicFuture(topVer);
             }
             else {
                 if (waitTopFut) {
@@ -435,11 +432,12 @@ private void updateNear(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicU
             cache.topology().readUnlock();
         }
 
-        map(topVer);
+        if (futVer != null)
+            map(topVer, futVer);
     }
 
     /** {@inheritDoc} */
-    protected void map(AffinityTopologyVersion topVer) {
+    @Override protected void map(AffinityTopologyVersion topVer, GridCacheVersion futVer) {
         Collection<ClusterNode> topNodes = CU.affinityNodes(cctx, topVer);
 
         if (F.isEmpty(topNodes)) {
@@ -449,11 +447,6 @@ protected void map(AffinityTopologyVersion topVer) {
             return;
         }
 
-        Exception err = null;
-        GridNearAtomicAbstractUpdateRequest singleReq0 = null;
-
-        GridCacheVersion futVer = cctx.versions().next(topVer);
-
         GridCacheVersion updVer;
 
         // Assign version on near node in CLOCK ordering mode even if fastMap is false.
@@ -470,16 +463,17 @@ protected void map(AffinityTopologyVersion topVer) {
         else
             updVer = null;
 
+        Exception err = null;
+        GridNearAtomicAbstractUpdateRequest singleReq0 = null;
+
         try {
             singleReq0 = mapSingleUpdate(topVer, futVer, updVer);
 
             synchronized (mux) {
-                assert this.futVer == null : this;
-                assert this.topVer == AffinityTopologyVersion.ZERO : this;
+                assert this.futVer == futVer || (this.isDone() && this.error() != null);
+                assert this.topVer == topVer;
 
-                this.topVer = topVer;
                 this.updVer = updVer;
-                this.futVer = futVer;
 
                 resCnt = 0;
 
@@ -496,14 +490,6 @@ protected void map(AffinityTopologyVersion topVer) {
             return;
         }
 
-        if (storeFuture()) {
-            if (!cctx.mvcc().addAtomicFuture(futVer, this)) {
-                assert isDone() : this;
-
-                return;
-            }
-        }
-
         // Optimize mapping for single key.
         mapSingle(singleReq0.nodeId(), singleReq0);
     }
@@ -511,7 +497,7 @@ protected void map(AffinityTopologyVersion topVer) {
     /**
      * @return Future version.
      */
-    GridCacheVersion onFutureDone() {
+    private GridCacheVersion onFutureDone() {
         GridCacheVersion ver0;
 
         GridFutureAdapter<Void> fut0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index cd6411725748b..950e5bdd2b9b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -456,14 +456,7 @@ else if (!nodeErr)
                 @Override public void apply(final IgniteInternalFuture<AffinityTopologyVersion> fut) {
                     cctx.kernalContext().closure().runLocalSafe(new Runnable() {
                         @Override public void run() {
-                            try {
-                                AffinityTopologyVersion topVer = fut.get();
-
-                                map(topVer, remapKeys);
-                            }
-                            catch (IgniteCheckedException e) {
-                                onDone(e);
-                            }
+                            mapOnTopology();
                         }
                     });
                 }
@@ -497,7 +490,9 @@ private void updateNear(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdat
     @Override protected void mapOnTopology() {
         cache.topology().readLock();
 
-        AffinityTopologyVersion topVer = null;
+        AffinityTopologyVersion topVer;
+
+        GridCacheVersion futVer;
 
         try {
             if (cache.topology().stopping()) {
@@ -519,6 +514,8 @@ private void updateNear(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdat
                 }
 
                 topVer = fut.topologyVersion();
+
+                futVer = addAtomicFuture(topVer);
             }
             else {
                 if (waitTopFut) {
@@ -544,7 +541,8 @@ private void updateNear(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdat
             cache.topology().readUnlock();
         }
 
-        map(topVer, null);
+        if (futVer != null)
+            map(topVer, futVer, remapKeys);
     }
 
     /**
@@ -602,15 +600,18 @@ private void doUpdate(Map<UUID, GridNearAtomicFullUpdateRequest> mappings) {
     }
 
     /** {@inheritDoc} */
-    protected void map(AffinityTopologyVersion topVer) {
-        map(topVer, null);
+    @Override protected void map(AffinityTopologyVersion topVer, GridCacheVersion futVer) {
+        map(topVer, futVer, null);
     }
 
     /**
      * @param topVer Topology version.
+     * @param futVer Future ID.
      * @param remapKeys Keys to remap.
      */
-    void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> remapKeys) {
+    void map(AffinityTopologyVersion topVer,
+        GridCacheVersion futVer,
+        @Nullable Collection<KeyCacheObject> remapKeys) {
         Collection<ClusterNode> topNodes = CU.affinityNodes(cctx, topVer);
 
         if (F.isEmpty(topNodes)) {
@@ -620,14 +621,6 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
             return;
         }
 
-        Exception err = null;
-        GridNearAtomicFullUpdateRequest singleReq0 = null;
-        Map<UUID, GridNearAtomicFullUpdateRequest> mappings0 = null;
-
-        int size = keys.size();
-
-        GridCacheVersion futVer = cctx.versions().next(topVer);
-
         GridCacheVersion updVer;
 
         // Assign version on near node in CLOCK ordering mode even if fastMap is false.
@@ -644,6 +637,12 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
         else
             updVer = null;
 
+        Exception err = null;
+        GridNearAtomicFullUpdateRequest singleReq0 = null;
+        Map<UUID, GridNearAtomicFullUpdateRequest> mappings0 = null;
+
+        int size = keys.size();
+
         try {
             if (size == 1 && !fastMap) {
                 assert remapKeys == null || remapKeys.size() == 1;
@@ -676,12 +675,10 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
             }
 
             synchronized (mux) {
-                assert this.futVer == null : this;
-                assert this.topVer == AffinityTopologyVersion.ZERO : this;
+                assert this.futVer == futVer || (this.isDone() && this.error() != null);
+                assert this.topVer == topVer;
 
-                this.topVer = topVer;
                 this.updVer = updVer;
-                this.futVer = futVer;
 
                 resCnt = 0;
 
@@ -701,14 +698,6 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
             return;
         }
 
-        if (storeFuture()) {
-            if (!cctx.mvcc().addAtomicFuture(futVer, this)) {
-                assert isDone() : this;
-
-                return;
-            }
-        }
-
         // Optimize mapping for single key.
         if (singleReq0 != null)
             mapSingle(singleReq0.nodeId(), singleReq0);
@@ -725,7 +714,7 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
     /**
      * @return Future version.
      */
-    GridCacheVersion onFutureDone() {
+    private GridCacheVersion onFutureDone() {
         GridCacheVersion ver0;
 
         GridFutureAdapter<Void> fut0;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java
new file mode 100644
index 0000000000000..878cb17b5c3e2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.atomic;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.affinity.fair.FairAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CachePeekMode.BACKUP;
+import static org.apache.ignite.cache.CachePeekMode.PRIMARY;
+import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/** */
+public class AtomicPutAllChangingTopologyTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int NODES_CNT = 3;
+
+    /** */
+    public static final String CACHE_NAME = "test-cache";
+
+    /** */
+    private static final int CACHE_SIZE = 20_000;
+
+    /** */
+    private static volatile CountDownLatch FILLED_LATCH;
+
+    /**
+     * @return Cache configuration.
+     */
+    private CacheConfiguration<Integer, Integer> cacheConfig() {
+        return new CacheConfiguration<Integer, Integer>()
+            .setAtomicityMode(ATOMIC)
+            .setCacheMode(REPLICATED)
+            .setAffinity(new FairAffinityFunction(false, 1))
+            .setWriteSynchronizationMode(FULL_SYNC)
+            .setRebalanceMode(SYNC)
+            .setName(CACHE_NAME);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPutAllOnChangingTopology() throws Exception {
+        List<IgniteInternalFuture> futs = new LinkedList<>();
+
+        for (int i = 1; i < NODES_CNT; i++)
+            futs.add(startNodeAsync(i));
+
+        futs.add(startSeedNodeAsync());
+
+        boolean failed = false;
+
+        for (IgniteInternalFuture fut : futs) {
+            try {
+                fut.get();
+            }
+            catch (Throwable th) {
+                log.error("Check failed.", th);
+
+                failed = true;
+            }
+        }
+
+        if (failed)
+            throw new RuntimeException("Test Failed.");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        FILLED_LATCH = new CountDownLatch(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @return Future.
+     * @throws IgniteCheckedException If failed.
+     */
+    private IgniteInternalFuture startSeedNodeAsync() throws IgniteCheckedException {
+        return GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Boolean call() throws Exception {
+                Ignite node = startGrid(0);
+
+                log.info("Creating cache.");
+
+                IgniteCache<Integer, Integer> cache = node.getOrCreateCache(cacheConfig());
+
+                log.info("Created cache.");
+
+                Map<Integer, Integer> data = new HashMap<>(CACHE_SIZE);
+
+                for (int i = 0; i < CACHE_SIZE; i++)
+                    data.put(i, i);
+
+                log.info("Filling.");
+
+                cache.putAll(data);
+
+                log.info("Filled.");
+
+                FILLED_LATCH.countDown();
+
+                checkCacheState(node, cache);
+
+                return true;
+            }
+        });
+    }
+
+    /**
+     * @param nodeId Node index.
+     * @return Future.
+     * @throws IgniteCheckedException If failed.
+     */
+    private IgniteInternalFuture startNodeAsync(final int nodeId) throws IgniteCheckedException {
+        return GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Boolean call() throws Exception {
+                Ignite node = startGrid(nodeId);
+
+                log.info("Getting cache.");
+
+                IgniteCache<Integer, Integer> cache = node.getOrCreateCache(cacheConfig());
+
+                log.info("Got cache.");
+
+                FILLED_LATCH.await();
+
+                log.info("Got Filled.");
+
+                cache.put(1, nodeId);
+
+                checkCacheState(node, cache);
+
+                return true;
+            }
+        });
+    }
+
+    /**
+     * @param node Node.
+     * @param cache Cache.
+     * @throws Exception If failed.
+     */
+    private void checkCacheState(Ignite node, IgniteCache<Integer, Integer> cache) throws Exception {
+        int locSize = cache.localSize(PRIMARY, BACKUP);
+        int locSize2 = -1;
+
+        if (locSize != CACHE_SIZE) {
+            U.sleep(5000);
+
+            // Rechecking.
+            locSize2 = cache.localSize(PRIMARY, BACKUP);
+        }
+
+        assertEquals("Wrong cache size on node [node=" + node.configuration().getGridName() +
+            ", expected= " + CACHE_SIZE +
+            ", actual=" + locSize +
+            ", actual2=" + locSize2 + "]",
+            locSize, CACHE_SIZE);
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
index c9e507d33e11c..5bc67292b33de 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheTxNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridNearCacheTxNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteAtomicLongChangingTopologySelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.AtomicPutAllChangingTopologyTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicClientInvalidPartitionHandlingSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicClientRemoveFailureTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicInvalidPartitionHandlingSelfTest;
@@ -95,6 +96,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridCacheTxNodeFailureSelfTest.class);
         suite.addTestSuite(GridNearCacheTxNodeFailureSelfTest.class);
 
+        suite.addTestSuite(AtomicPutAllChangingTopologyTest.class);
+
         return suite;
     }
 }

From 496fb173d55a8ea0702fcb70a4e807f61d8fc53e Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Mon, 16 Jan 2017 13:19:51 +0300
Subject: [PATCH 099/446] IGNITE-4513 Improve debug logging.

---
 .../processors/cache/GridCacheUtils.java      |  3 +
 .../CacheContinuousQueryHandler.java          | 81 ++++++++++++++++---
 modules/core/src/test/config/log4j-test.xml   |  6 ++
 3 files changed, 79 insertions(+), 11 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index d32f4c1867bcf..8ee77e36c3a77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -122,6 +122,9 @@ public class GridCacheUtils {
     /** Marshaller system cache name. */
     public static final String MARSH_CACHE_NAME = "ignite-marshaller-sys-cache";
 
+    /** */
+    public static final String CONTINUOUS_QRY_LOG_CATEGORY = "org.apache.ignite.continuous.query";
+
     /** */
     public static final String CACHE_MSG_LOG_CATEGORY = "org.apache.ignite.cache.msg";
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 10784fc6b05fe..83edab444f8c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -352,7 +352,7 @@ public void keepBinary(boolean keepBinary) {
 
         assert !skipPrimaryCheck || loc;
 
-        log = ctx.log(CacheContinuousQueryHandler.class);
+        log = ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY);
 
         CacheContinuousQueryListener<K, V> lsnr = new CacheContinuousQueryListener<K, V>() {
             @Override public void onExecution() {
@@ -384,7 +384,10 @@ public void keepBinary(boolean keepBinary) {
                 final boolean recordIgniteEvt,
                 GridDhtAtomicAbstractUpdateFuture fut) {
                 if (ignoreExpired && evt.getEventType() == EventType.EXPIRED)
-                    return ;
+                    return;
+
+                if (log.isDebugEnabled())
+                    log.debug("Entry updated on affinity node [evt=" + evt + ", primary=" + primary + ']');
 
                 final GridCacheContext<K, V> cctx = cacheContext(ctx);
 
@@ -407,6 +410,10 @@ public void keepBinary(boolean keepBinary) {
                 else {
                     final boolean notify = filter(evt, primary);
 
+                    if (log.isDebugEnabled())
+                        log.debug("Filter invoked for event [evt=" + evt + ", primary=" + primary
+                            + ", notify=" + notify + ']');
+
                     if (primary || skipPrimaryCheck) {
                         if (fut == null)
                             onEntryUpdate(evt, notify, loc, recordIgniteEvt);
@@ -474,7 +481,8 @@ public void keepBinary(boolean keepBinary) {
                     backupQueue0.clear();
                 }
                 catch (IgniteCheckedException e) {
-                    U.error(ctx.log(getClass()), "Failed to send backup event notification to node: " + nodeId, e);
+                    U.error(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY),
+                        "Failed to send backup event notification to node: " + nodeId, e);
                 }
             }
 
@@ -653,7 +661,7 @@ private void notifyCallback0(UUID nodeId,
         final GridCacheContext cctx = cacheContext(ctx);
 
         if (cctx == null) {
-            IgniteLogger log = ctx.log(CacheContinuousQueryHandler.class);
+            IgniteLogger log = ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY);
 
             if (log.isDebugEnabled())
                 log.debug("Failed to notify callback, cache is not found: " + cacheId);
@@ -689,7 +697,7 @@ private void notifyCallback0(UUID nodeId,
                 if (ignoreClsNotFound)
                     assert internal;
                 else
-                    U.error(ctx.log(getClass()), "Failed to unmarshal entry.", ex);
+                    U.error(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY), "Failed to unmarshal entry.", ex);
             }
         }
 
@@ -799,8 +807,12 @@ private void onEntryUpdate(CacheContinuousQueryEvent evt, boolean notify, boolea
 
                 CacheContinuousQueryEntry e = handleEntry(entry);
 
-                if (e != null)
+                if (e != null) {
+                    if (log.isDebugEnabled())
+                        log.debug("Send the following event to listener: " + e);
+
                     ctx.continuous().addNotification(nodeId, routineId, entry, topic, sync, true);
+                }
             }
         }
         catch (ClusterTopologyCheckedException ex) {
@@ -809,7 +821,7 @@ private void onEntryUpdate(CacheContinuousQueryEvent evt, boolean notify, boolea
                     "[node=" + nodeId + ", err=" + ex + ']');
         }
         catch (IgniteCheckedException ex) {
-            U.error(ctx.log(getClass()), "Failed to send event notification to node: " + nodeId, ex);
+            U.error(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY), "Failed to send event notification to node: " + nodeId, ex);
         }
 
         if (recordIgniteEvt && notify) {
@@ -875,7 +887,7 @@ else if (initUpdCntrs != null)
                     partCntr = initUpdCntrs.get(partId);
             }
 
-            rec = new PartitionRecovery(ctx.log(getClass()), initTopVer0, partCntr);
+            rec = new PartitionRecovery(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY), initTopVer0, partCntr);
 
             PartitionRecovery oldRec = rcvs.putIfAbsent(partId, rec);
 
@@ -984,12 +996,27 @@ private static class PartitionRecovery {
             List<CacheEntryEvent<? extends K, ? extends V>> entries;
 
             synchronized (pendingEvts) {
+                if (log.isDebugEnabled()) {
+                    log.debug("Handling event [lastFiredEvt=" + lastFiredEvt +
+                        ", curTop=" + curTop +
+                        ", entUpdCnt=" + entry.updateCounter() +
+                        ", partId=" + entry.partition() +
+                        ", pendingEvts=" + pendingEvts + ']');
+                }
+
                 // Received first event.
                 if (curTop == AffinityTopologyVersion.NONE) {
                     lastFiredEvt = entry.updateCounter();
 
                     curTop = entry.topologyVersion();
 
+                    if (log.isDebugEnabled()) {
+                        log.debug("First event [lastFiredEvt=" + lastFiredEvt +
+                            ", curTop=" + curTop +
+                            ", entUpdCnt=" + entry.updateCounter() +
+                            ", partId=" + entry.partition() + ']');
+                    }
+
                     return !entry.isFiltered() ?
                         F.<CacheEntryEvent<? extends K, ? extends V>>
                             asList(new CacheContinuousQueryEvent<K, V>(cache, cctx, entry)) :
@@ -1014,6 +1041,13 @@ private static class PartitionRecovery {
                         if (!entry.isFiltered())
                             entries.add(new CacheContinuousQueryEvent<K, V>(cache, cctx, entry));
 
+                        if (log.isDebugEnabled())
+                            log.debug("Partition was lost [lastFiredEvt=" + lastFiredEvt +
+                                ", curTop=" + curTop +
+                                ", entUpdCnt=" + entry.updateCounter() +
+                                ", partId=" + entry.partition() +
+                                ", pendingEvts=" + pendingEvts + ']');
+
                         return entries;
                     }
 
@@ -1039,8 +1073,16 @@ private static class PartitionRecovery {
                     return Collections.emptyList();
                 }
 
-                if (pendingEvts.isEmpty())
+                if (pendingEvts.isEmpty()) {
+                    if (log.isDebugEnabled()) {
+                        log.debug("Nothing sent to listener [lastFiredEvt=" + lastFiredEvt +
+                            ", curTop=" + curTop +
+                            ", entUpdCnt=" + entry.updateCounter() +
+                            ", partId=" + entry.partition() + ']');
+                    }
+
                     return Collections.emptyList();
+                }
 
                 Iterator<Map.Entry<Long, CacheContinuousQueryEntry>> iter = pendingEvts.entrySet().iterator();
 
@@ -1057,6 +1099,14 @@ private static class PartitionRecovery {
 
                         iter.remove();
                     }
+
+                    if (log.isDebugEnabled()) {
+                        log.debug("Pending events reached max of buffer size [lastFiredEvt=" + lastFiredEvt +
+                            ", curTop=" + curTop +
+                            ", entUpdCnt=" + entry.updateCounter() +
+                            ", partId=" + entry.partition() +
+                            ", pendingEvts=" + pendingEvts + ']');
+                    }
                 }
                 else {
                     // Elements are consistently.
@@ -1077,6 +1127,15 @@ private static class PartitionRecovery {
                 }
             }
 
+            if (log.isDebugEnabled()) {
+                log.debug("Will send to listener the following events [entries=" + entries +
+                    ", lastFiredEvt=" + lastFiredEvt +
+                    ", curTop=" + curTop +
+                    ", entUpdCnt=" + entry.updateCounter() +
+                    ", partId=" + entry.partition() +
+                    ", pendingEvts=" + pendingEvts + ']');
+            }
+
             return entries;
         }
     }
@@ -1254,14 +1313,14 @@ private void sendBackupAcknowledge(final IgniteBiTuple<Map<Integer, Long>, Set<A
                                     cctx.io().send(node, msg, GridIoPolicy.SYSTEM_POOL);
                                 }
                                 catch (ClusterTopologyCheckedException e) {
-                                    IgniteLogger log = ctx.log(getClass());
+                                    IgniteLogger log = ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY);
 
                                     if (log.isDebugEnabled())
                                         log.debug("Failed to send acknowledge message, node left " +
                                             "[msg=" + msg + ", node=" + node + ']');
                                 }
                                 catch (IgniteCheckedException e) {
-                                    IgniteLogger log = ctx.log(getClass());
+                                    IgniteLogger log = ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY);
 
                                     U.error(log, "Failed to send acknowledge message " +
                                         "[msg=" + msg + ", node=" + node + ']', e);
diff --git a/modules/core/src/test/config/log4j-test.xml b/modules/core/src/test/config/log4j-test.xml
index 276de8ccdbabf..638b260cb6e4a 100644
--- a/modules/core/src/test/config/log4j-test.xml
+++ b/modules/core/src/test/config/log4j-test.xml
@@ -102,6 +102,12 @@
     </category>
     -->
 
+    <!--
+    <category name="org.apache.ignite.continuous.query">
+        <level value="DEBUG"/>
+    </category>
+    -->
+
     <!-- Disable all open source debugging. -->
     <category name="org">
         <level value="INFO"/>

From c3eae9fecff5ad01390170c034dca39c216a097c Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Mon, 16 Jan 2017 10:33:16 +0700
Subject: [PATCH 100/446] IGNITE-4518 Fixed parallel load of cache. - Fixes
 #1426.

Signed-off-by: Andrey Novikov <anovikov@gridgain.com>

(cherry picked from commit 79401b2)
---
 .../store/jdbc/dialect/BasicJdbcDialect.java  | 31 +++++++-----
 .../store/jdbc/CacheJdbcPojoStoreTest.java    | 48 ++++++++++++++++++-
 2 files changed, 66 insertions(+), 13 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/dialect/BasicJdbcDialect.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/dialect/BasicJdbcDialect.java
index 3ab112af41516..139f3fca7504d 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/dialect/BasicJdbcDialect.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/dialect/BasicJdbcDialect.java
@@ -173,13 +173,15 @@ private static String where(Collection<String> keyCols, int keyCnt) {
         if (appendLowerBound) {
             sb.a("(");
 
-            for (int cnt = keyCols.size(); cnt > 0; cnt--) {
-                for (int j = 0; j < cnt; j++)
-                    if (j == cnt - 1)
-                        sb.a(cols[j]).a(" > ? ");
+            for (int keyCnt = keyCols.size(); keyCnt > 0; keyCnt--) {
+                for (int idx = 0; idx < keyCnt; idx++) {
+                    if (idx == keyCnt - 1)
+                        sb.a(cols[idx]).a(" > ? ");
                     else
-                        sb.a(cols[j]).a(" = ? AND ");
-                if (cnt != 1)
+                        sb.a(cols[idx]).a(" = ? AND ");
+                }
+
+                if (keyCnt != 1)
                     sb.a("OR ");
             }
 
@@ -192,13 +194,18 @@ private static String where(Collection<String> keyCols, int keyCnt) {
         if (appendUpperBound) {
             sb.a("(");
 
-            for (int cnt = keyCols.size(); cnt > 0; cnt--) {
-                for (int j = 0; j < cnt; j++)
-                    if (j == cnt - 1)
-                        sb.a(cols[j]).a(" <= ? ");
+            for (int keyCnt = keyCols.size(); keyCnt > 0; keyCnt--) {
+                for (int idx = 0, lastIdx = keyCnt - 1; idx < keyCnt; idx++) {
+                    sb.a(cols[idx]);
+
+                    // For composite key when not all of the key columns are constrained should use < (strictly less).
+                    if (idx == lastIdx)
+                        sb.a(keyCnt == keyCols.size() ? " <= ? " : " < ? ");
                     else
-                        sb.a(cols[j]).a(" = ? AND ");
-                if (cnt != 1)
+                        sb.a(" = ? AND ");
+                }
+
+                if (keyCnt != 1)
                     sb.a(" OR ");
             }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
index d8f75d3dcb6ee..4a0b1daf646d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
@@ -216,7 +216,7 @@ public CacheJdbcPojoStoreTest() throws Exception {
 
         stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " +
             "Person_Complex (id integer not null, org_id integer not null, city_id integer not null, " +
-            "name varchar(50), salary integer, PRIMARY KEY(id))");
+            "name varchar(50), salary integer, PRIMARY KEY(id, org_id, city_id))");
 
         conn.commit();
 
@@ -349,6 +349,52 @@ else if (k instanceof PersonComplexKey && v instanceof Person) {
         assertTrue(prnComplexKeys.isEmpty());
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testParallelLoad() throws Exception {
+        Connection conn = store.openConnection(false);
+
+        PreparedStatement prnComplexStmt = conn.prepareStatement("INSERT INTO Person_Complex(id, org_id, city_id, name, salary) VALUES (?, ?, ?, ?, ?)");
+
+        for (int i = 0; i < 8; i++) {
+
+            prnComplexStmt.setInt(1, (i >> 2) & 1);
+            prnComplexStmt.setInt(2, (i >> 1) & 1);
+            prnComplexStmt.setInt(3, i % 2);
+
+            prnComplexStmt.setString(4, "name");
+            prnComplexStmt.setInt(5, 1000 + i * 500);
+
+            prnComplexStmt.addBatch();
+        }
+
+        prnComplexStmt.executeBatch();
+
+        U.closeQuiet(prnComplexStmt);
+
+        conn.commit();
+
+        U.closeQuiet(conn);
+
+        final Collection<PersonComplexKey> prnComplexKeys = new ConcurrentLinkedQueue<>();
+
+        IgniteBiInClosure<Object, Object> c = new CI2<Object, Object>() {
+            @Override public void apply(Object k, Object v) {
+                if (k instanceof PersonComplexKey && v instanceof Person)
+                    prnComplexKeys.add((PersonComplexKey)k);
+                else
+                    fail("Unexpected entry [key=" + k + ", value=" + v + "]");
+            }
+        };
+
+        store.setParallelLoadCacheMinimumThreshold(2);
+
+        store.loadCache(c);
+
+        assertEquals(8, prnComplexKeys.size());
+    }
+
     /**
      * @throws Exception If failed.
      */

From 71176473f9fd0aa2088ba4e611ba4b7fc45e76b8 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Mon, 16 Jan 2017 11:22:34 +0700
Subject: [PATCH 101/446] IGNITE-4545 Added cache for router hostnames. - Fixes
 #1428.

Signed-off-by: Andrey Novikov <anovikov@gridgain.com>

(cherry picked from commit 27ba69b)
---
 .../GridClientConnectionManagerAdapter.java   |  7 ++-
 .../impl/connection/GridClientTopology.java   | 53 +++++++++++++++----
 2 files changed, 49 insertions(+), 11 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
index 6ea7c227bddde..0928c9061cf43 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
@@ -85,6 +85,9 @@ public abstract class GridClientConnectionManagerAdapter implements GridClientCo
     /** Class logger. */
     private final Logger log;
 
+    /** All local enabled MACs. */
+    private final Collection<String> macs;
+
     /** NIO server. */
     private GridNioServer srv;
 
@@ -166,6 +169,8 @@ protected GridClientConnectionManagerAdapter(UUID clientId,
         if (marshId == null && cfg.getMarshaller() == null)
             throw new GridClientException("Failed to start client (marshaller is not configured).");
 
+        macs = U.allLocalMACs();
+
         if (cfg.getProtocol() == GridClientProtocol.TCP) {
             try {
                 IgniteLogger gridLog = new JavaLogger(false);
@@ -315,7 +320,7 @@ protected GridClientConnectionManagerAdapter(UUID clientId,
         }
 
         boolean sameHost = node.attributes().isEmpty() ||
-            F.containsAny(U.allLocalMACs(), node.attribute(ATTR_MACS).toString().split(", "));
+            F.containsAny(macs, node.attribute(ATTR_MACS).toString().split(", "));
 
         Collection<InetSocketAddress> srvs = new LinkedHashSet<>();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientTopology.java
index effd5b3943148..97aa5861308db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientTopology.java
@@ -21,7 +21,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.Map;
 import java.util.Set;
@@ -61,12 +60,18 @@ public class GridClientTopology {
     /** Cached last error prevented topology from update. */
     private GridClientException lastError;
 
+    /** Router addresses from configuration.  */
+    private final String routers;
+
     /**
      * Set of router addresses to infer direct connectivity
      * when client is working in router connection mode.
      * {@code null} when client is working in direct connection node.
      */
-    private final Set<String> routerAddrs;
+    private final Set<InetSocketAddress> routerAddrs;
+
+    /** List of all known local MACs */
+    private final Collection<String> macsCache;
 
     /** Protocol. */
     private final GridClientProtocol prot;
@@ -96,8 +101,38 @@ public GridClientTopology(GridClientConfiguration cfg) {
         metricsCache = cfg.isEnableMetricsCache();
         attrCache = cfg.isEnableAttributesCache();
         prot = cfg.getProtocol();
-        routerAddrs = (!cfg.getRouters().isEmpty() && cfg.getServers().isEmpty()) ?
-            new HashSet<>(cfg.getRouters()) : null;
+
+        if (!cfg.getRouters().isEmpty() && cfg.getServers().isEmpty()) {
+            routers = cfg.getRouters().toString();
+
+            routerAddrs = U.newHashSet(cfg.getRouters().size());
+
+            for (String router : cfg.getRouters()) {
+                int portIdx = router.lastIndexOf(":");
+
+                if (portIdx > 0) {
+                    String hostName = router.substring(0, portIdx);
+
+                    try {
+                        int port = Integer.parseInt(router.substring(portIdx + 1));
+
+                        InetSocketAddress inetSockAddr = new InetSocketAddress(hostName, port);
+
+                        routerAddrs.add(inetSockAddr);
+                    }
+                    catch (Exception ignore) {
+                        // No-op.
+                    }
+                }
+            }
+        }
+        else {
+            routers = null;
+
+            routerAddrs = Collections.emptySet();
+        }
+
+        macsCache = U.allLocalMACs();
     }
 
     /**
@@ -279,7 +314,7 @@ public GridClientNode node(UUID id) throws GridClientException {
         try {
             if (lastError != null)
                 throw new GridClientDisconnectedException(
-                    "Topology is failed [protocol=" + prot + ", routers=" + routerAddrs + ']', lastError);
+                    "Topology is failed [protocol=" + prot + ", routers=" + routers + ']', lastError);
             else
                 return nodes.get(id);
         }
@@ -376,19 +411,17 @@ private GridClientNodeImpl prepareNode(final GridClientNodeImpl node) {
             (metricsCache && attrCache) || (node.attributes().isEmpty() && node.metrics() == null);
 
         // Try to bypass object copying.
-        if (noAttrsAndMetrics && routerAddrs == null && node.connectable())
+        if (noAttrsAndMetrics && routerAddrs.isEmpty() && node.connectable())
             return node;
 
         // Return a new node instance based on the original one.
         GridClientNodeImpl.Builder nodeBuilder = GridClientNodeImpl.builder(node, !attrCache, !metricsCache);
 
         for (InetSocketAddress addr : node.availableAddresses(prot, true)) {
-            boolean router = routerAddrs == null ||
-                routerAddrs.contains(addr.getHostName() + ":" + addr.getPort()) ||
-                routerAddrs.contains(addr.getAddress().getHostAddress() + ":" + addr.getPort());
+            boolean router = routerAddrs.isEmpty() || routerAddrs.contains(addr);
 
             boolean reachable = noAttrsAndMetrics || !addr.getAddress().isLoopbackAddress() ||
-                F.containsAny(U.allLocalMACs(), node.attribute(ATTR_MACS).toString().split(", "));
+                F.containsAny(macsCache, node.<String>attribute(ATTR_MACS).split(", "));
 
             if (router && reachable) {
                 nodeBuilder.connectable(true);

From 22b7e76c00a77a06388bcef869f29d1a572a306a Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 17 Jan 2017 12:33:32 +0300
Subject: [PATCH 102/446] ignite-4465 Prevent cache entry eviction while it is
 loaded from store (otherwise loaded value can be not stored in cache).

---
 .../processors/cache/EntryGetResult.java      |  65 +++++++
 .../processors/cache/GridCacheAdapter.java    | 104 ++++++----
 .../processors/cache/GridCacheEntryEx.java    |  30 ++-
 .../processors/cache/GridCacheMapEntry.java   | 100 ++++++++--
 .../dht/GridPartitionedGetFuture.java         |   7 +-
 .../dht/GridPartitionedSingleGetFuture.java   |   7 +-
 .../dht/atomic/GridDhtAtomicCache.java        |   7 +-
 .../dht/colocated/GridDhtColocatedCache.java  |   7 +-
 .../distributed/near/GridNearGetFuture.java   |  13 +-
 .../local/atomic/GridLocalAtomicCache.java    |   7 +-
 .../cache/transactions/IgniteTxHandler.java   |   2 +-
 .../transactions/IgniteTxLocalAdapter.java    |  29 +--
 .../cache/CacheConcurrentReadThroughTest.java | 184 ++++++++++++++++++
 .../CrossCacheTxRandomOperationsTest.java     |  28 ++-
 .../cache/GridCacheTestEntryEx.java           |  21 +-
 .../testsuites/IgniteCacheTestSuite2.java     |   2 +
 16 files changed, 512 insertions(+), 101 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetResult.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetResult.java
new file mode 100644
index 0000000000000..a34ddae6d10af
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetResult.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+
+/**
+ *
+ */
+public class EntryGetResult {
+    /** */
+    private final CacheObject val;
+
+    /** */
+    private final GridCacheVersion ver;
+
+    /** */
+    private final boolean reserved;
+
+    /**
+     * @param val Value.
+     * @param ver Version.
+     */
+    EntryGetResult(CacheObject val, GridCacheVersion ver, boolean reserved) {
+        this.val = val;
+        this.ver = ver;
+        this.reserved = reserved;
+    }
+
+    /**
+     * @return Value.
+     */
+    public CacheObject value() {
+        return val;
+    }
+
+    /**
+     * @return Version.
+     */
+    public GridCacheVersion version() {
+        return ver;
+    }
+
+    /**
+     * @return Reserved flag,
+     */
+    public boolean reserved() {
+        return reserved;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 965c6d12916aa..fd9f396926f0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1860,7 +1860,7 @@ public final IgniteInternalFuture<Map<K, V>> getAllAsync(@Nullable final Collect
      * @param needVer If {@code true} returns values as tuples containing value and version.
      * @return Future.
      */
-    public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
+    protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
         @Nullable final Collection<KeyCacheObject> keys,
         final boolean readThrough,
         boolean checkTx,
@@ -1906,7 +1906,7 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
 
                 final boolean needEntry = storeEnabled || ctx.isSwapOrOffheapEnabled();
 
-                Map<KeyCacheObject, GridCacheVersion> misses = null;
+                Map<KeyCacheObject, EntryGetResult> misses = null;
 
                 for (KeyCacheObject key : keys) {
                     while (true) {
@@ -1920,40 +1920,58 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                         }
 
                         try {
-                            T2<CacheObject, GridCacheVersion> res = entry.innerGetVersioned(
-                                null,
-                                null,
-                                ctx.isSwapOrOffheapEnabled(),
-                                /*unmarshal*/true,
-                                /*update-metrics*/!skipVals,
-                                /*event*/!skipVals,
-                                subjId,
-                                null,
-                                taskName,
-                                expiry,
-                                !deserializeBinary);
-
-                            if (res == null) {
-                                if (storeEnabled) {
-                                    GridCacheVersion ver = entry.version();
+                            EntryGetResult res;
 
+                            boolean evt = !skipVals;
+                            boolean updateMetrics = !skipVals;
+
+                            if (storeEnabled) {
+                                res = entry.innerGetAndReserveForLoad(ctx.isSwapOrOffheapEnabled(),
+                                    updateMetrics,
+                                    evt,
+                                    subjId,
+                                    taskName,
+                                    expiry,
+                                    !deserializeBinary);
+
+                                assert res != null;
+
+                                if (res.value() == null) {
                                     if (misses == null)
                                         misses = new HashMap<>();
 
-                                    misses.put(key, ver);
+                                    misses.put(key, res);
+
+                                    res = null;
                                 }
-                                else
-                                    ctx.evicts().touch(entry, topVer);
                             }
                             else {
+                                res = entry.innerGetVersioned(
+                                    null,
+                                    null,
+                                    ctx.isSwapOrOffheapEnabled(),
+                                    /*unmarshal*/true,
+                                    updateMetrics,
+                                    evt,
+                                    subjId,
+                                    null,
+                                    taskName,
+                                    expiry,
+                                    !deserializeBinary);
+
+                                if (res == null)
+                                    ctx.evicts().touch(entry, topVer);
+                            }
+
+                            if (res != null) {
                                 ctx.addResult(map,
                                     key,
-                                    res.get1(),
+                                    res.value(),
                                     skipVals,
                                     keepCacheObjects,
                                     deserializeBinary,
                                     true,
-                                    needVer ? res.get2() : null);
+                                    needVer ? res.version() : null);
 
                                 if (tx == null || (!tx.implicit() && tx.isolation() == READ_COMMITTED))
                                     ctx.evicts().touch(entry, topVer);
@@ -1973,7 +1991,7 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                 }
 
                 if (storeEnabled && misses != null) {
-                    final Map<KeyCacheObject, GridCacheVersion> loadKeys = misses;
+                    final Map<KeyCacheObject, EntryGetResult> loadKeys = misses;
 
                     final IgniteTxLocalAdapter tx0 = tx;
 
@@ -1984,15 +2002,10 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                             @Override public Map<K1, V1> call() throws Exception {
                                 ctx.store().loadAll(null/*tx*/, loadKeys.keySet(), new CI2<KeyCacheObject, Object>() {
                                     @Override public void apply(KeyCacheObject key, Object val) {
-                                        GridCacheVersion ver = loadKeys.get(key);
-
-                                        if (ver == null) {
-                                            if (log.isDebugEnabled())
-                                                log.debug("Value from storage was never asked for [key=" + key +
-                                                    ", val=" + val + ']');
+                                        EntryGetResult res = loadKeys.get(key);
 
+                                        if (res == null || val == null)
                                             return;
-                                        }
 
                                         loaded.add(key);
 
@@ -2002,14 +2015,16 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                             GridCacheEntryEx entry = entryEx(key);
 
                                             try {
-                                                GridCacheVersion verSet = entry.versionedValue(cacheVal, ver, null);
+                                                GridCacheVersion verSet = entry.versionedValue(cacheVal,
+                                                    res.version(),
+                                                    null);
 
                                                 boolean set = verSet != null;
 
                                                 if (log.isDebugEnabled())
                                                     log.debug("Set value loaded from store into entry [" +
                                                         "set=" + set +
-                                                        ", curVer=" + ver +
+                                                        ", curVer=" + res.version() +
                                                         ", newVer=" + verSet + ", " +
                                                         "entry=" + entry + ']');
 
@@ -2022,7 +2037,7 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                                         keepCacheObjects,
                                                         deserializeBinary,
                                                         false,
-                                                        needVer ? set ? verSet : ver : null);
+                                                        needVer ? set ? verSet : res.version() : null);
                                                 }
 
                                                 if (tx0 == null || (!tx0.implicit() &&
@@ -2045,16 +2060,23 @@ public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                 });
 
                                 if (loaded.size() != loadKeys.size()) {
-                                    for (KeyCacheObject key : loadKeys.keySet()) {
-                                        if (loaded.contains(key))
+                                    boolean needTouch =
+                                        tx0 == null || (!tx0.implicit() && tx0.isolation() == READ_COMMITTED);
+
+                                    for (Map.Entry<KeyCacheObject, EntryGetResult> e : loadKeys.entrySet()) {
+                                        if (loaded.contains(e.getKey()))
                                             continue;
 
-                                        if (tx0 == null || (!tx0.implicit() &&
-                                            tx0.isolation() == READ_COMMITTED)) {
-                                            GridCacheEntryEx entry = peekEx(key);
+                                        if (needTouch || e.getValue().reserved()) {
+                                            GridCacheEntryEx entry = peekEx(e.getKey());
 
-                                            if (entry != null)
-                                                ctx.evicts().touch(entry, topVer);
+                                            if (entry != null) {
+                                                if (e.getValue().reserved())
+                                                    entry.clearReserveForLoad(e.getValue().version());
+
+                                                if (needTouch)
+                                                    ctx.evicts().touch(entry, topVer);
+                                            }
                                         }
                                     }
                                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index d8194fcb30de7..b1d632fd5943d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -34,7 +34,6 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
 import org.apache.ignite.internal.processors.dr.GridDrType;
 import org.apache.ignite.internal.util.lang.GridTuple3;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -323,7 +322,7 @@ public boolean evictInternal(boolean swap, GridCacheVersion obsoleteVer,
      * @throws IgniteCheckedException If loading value failed.
      * @throws GridCacheEntryRemovedException If entry was removed.
      */
-    @Nullable public T2<CacheObject, GridCacheVersion> innerGetVersioned(
+    @Nullable public EntryGetResult innerGetVersioned(
         @Nullable GridCacheVersion ver,
         IgniteInternalTx tx,
         boolean readSwap,
@@ -337,6 +336,33 @@ public boolean evictInternal(boolean swap, GridCacheVersion obsoleteVer,
         boolean keepBinary)
         throws IgniteCheckedException, GridCacheEntryRemovedException;
 
+    /**
+     * @param readSwap Flag indicating whether to check swap memory.
+     * @param updateMetrics If {@code true} then metrics should be updated.
+     * @param evt Flag to signal event notification.
+     * @param subjId Subject ID initiated this read.
+     * @param taskName Task name.
+     * @param expiryPlc Expiry policy.
+     * @param keepBinary Keep binary flag.
+     * @return Cached value and entry version.
+     * @throws IgniteCheckedException If loading value failed.
+     * @throws GridCacheEntryRemovedException If entry was removed.
+     * @return Cached value, entry version and flag indicating if entry was reserved.
+     */
+    public EntryGetResult innerGetAndReserveForLoad(boolean readSwap,
+        boolean updateMetrics,
+        boolean evt,
+        UUID subjId,
+        String taskName,
+        @Nullable IgniteCacheExpiryPolicy expiryPlc,
+        boolean keepBinary) throws IgniteCheckedException, GridCacheEntryRemovedException;
+
+    /**
+     * @param ver Expected entry version.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void clearReserveForLoad(GridCacheVersion ver) throws IgniteCheckedException;
+
     /**
      * Reloads entry from underlying storage.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 31baedacc7956..9f0c2b0a47c56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -105,6 +105,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
     /** */
     private static final byte IS_SWAPPING_REQUIRED = 0x08;
 
+    /** */
+    private static final byte IS_EVICT_DISABLED = 0x10;
+
     /** */
     public static final GridCacheAtomicVersionComparator ATOMIC_VER_COMPARATOR = new GridCacheAtomicVersionComparator();
 
@@ -774,11 +777,37 @@ protected final void releaseSwap() throws IgniteCheckedException {
             taskName,
             expirePlc,
             false,
-            keepBinary);
+            keepBinary,
+            false);
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public T2<CacheObject, GridCacheVersion> innerGetVersioned(
+    @Override public EntryGetResult innerGetAndReserveForLoad(boolean readSwap,
+        boolean updateMetrics,
+        boolean evt,
+        UUID subjId,
+        String taskName,
+        @Nullable IgniteCacheExpiryPolicy expiryPlc,
+        boolean keepBinary) throws IgniteCheckedException, GridCacheEntryRemovedException {
+        return (EntryGetResult)innerGet0(
+            /*ver*/null,
+            /*tx*/null,
+            readSwap,
+            /*readThrough*/false,
+            evt,
+            updateMetrics,
+            /*tmp*/false,
+            subjId,
+            /*transformClo*/null,
+            taskName,
+            expiryPlc,
+            true,
+            keepBinary,
+            /*reserve*/true);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryGetResult innerGetVersioned(
         @Nullable GridCacheVersion ver,
         IgniteInternalTx tx,
         boolean readSwap,
@@ -791,7 +820,7 @@ protected final void releaseSwap() throws IgniteCheckedException {
         @Nullable IgniteCacheExpiryPolicy expiryPlc,
         boolean keepBinary)
         throws IgniteCheckedException, GridCacheEntryRemovedException {
-        return (T2<CacheObject, GridCacheVersion>)innerGet0(ver,
+        return (EntryGetResult)innerGet0(ver,
             tx,
             readSwap,
             false,
@@ -803,7 +832,8 @@ protected final void releaseSwap() throws IgniteCheckedException {
             taskName,
             expiryPlc,
             true,
-            keepBinary);
+            keepBinary,
+            false);
     }
 
     /** {@inheritDoc} */
@@ -821,16 +851,16 @@ private Object innerGet0(
         String taskName,
         @Nullable IgniteCacheExpiryPolicy expiryPlc,
         boolean retVer,
-        boolean keepBinary
+        boolean keepBinary,
+        boolean reserveForLoad
     ) throws IgniteCheckedException, GridCacheEntryRemovedException {
         assert !(retVer && readThrough);
+        assert !(reserveForLoad && readThrough);
 
         // Disable read-through if there is no store.
         if (readThrough && !cctx.readThrough())
             readThrough = false;
 
-        CacheObject ret;
-
         GridCacheVersion startVer;
         GridCacheVersion resVer = null;
 
@@ -838,6 +868,8 @@ private Object innerGet0(
         boolean deferred = false;
         GridCacheVersion ver0 = null;
 
+        Object res = null;
+
         synchronized (this) {
             checkObsolete();
 
@@ -881,7 +913,7 @@ private Object innerGet0(
             else
                 val = null;
 
-            ret = val;
+            CacheObject ret = val;
 
             if (ret == null) {
                 if (updateMetrics && cctx.cache().configuration().isStatisticsEnabled())
@@ -928,15 +960,26 @@ private Object innerGet0(
 
             // Cache version for optimistic check.
             startVer = ver;
-        }
 
-        if (ret != null) {
-            assert tmp || !(ret instanceof BinaryObjectOffheapImpl);
-            assert !obsolete;
-            assert !deferred;
+            if (ret != null) {
+                assert tmp || !(ret instanceof BinaryObjectOffheapImpl);
+                assert !obsolete;
+                assert !deferred;
+
+                // If return value is consistent, then done.
+                res = retVer ? new EntryGetResult(ret, resVer, false) : ret;
+            }
+            else if (reserveForLoad && !obsolete) {
+                assert !readThrough;
+                assert retVer;
+
+                boolean reserve = !evictionDisabled();
 
-            // If return value is consistent, then done.
-            return retVer ? new T2<>(ret, resVer) : ret;
+                if (reserve)
+                    flags |= IS_EVICT_DISABLED;
+
+                res = new EntryGetResult(null, resVer, reserve);
+            }
         }
 
         if (obsolete) {
@@ -948,6 +991,11 @@ private Object innerGet0(
         if (deferred)
             cctx.onDeferredDelete(this, ver0);
 
+        if (res != null)
+            return res;
+
+        CacheObject ret = null;
+
         if (readThrough) {
             IgniteInternalTx tx0 = null;
 
@@ -2926,7 +2974,7 @@ protected void clearReader(UUID nodeId) throws GridCacheEntryRemovedException {
      * @return {@code True} if this entry should not be evicted from cache.
      */
     protected boolean evictionDisabled() {
-        return false;
+        return (flags & IS_EVICT_DISABLED) != 0;
     }
 
     /**
@@ -3008,6 +3056,7 @@ protected final boolean markObsolete0(GridCacheVersion ver, boolean clear, GridC
             value(null);
 
             ver = newVer;
+            flags &= ~IS_EVICT_DISABLED;
 
             if (log.isTraceEnabled()) {
                 log.trace("invalidate releaseSwap [key=" + key +
@@ -3096,6 +3145,7 @@ protected final void update(@Nullable CacheObject val, long expireTime, long ttl
         ttlAndExpireTimeExtras(ttl, expireTime);
 
         this.ver = ver;
+        flags &= ~IS_EVICT_DISABLED;
 
         if (addTracked && expireTime != 0 && (expireTime != oldExpireTime || isStartVersion()) && cctx.config().isEagerTtl())
             cctx.ttl().addTrackedEntry(this);
@@ -3548,12 +3598,24 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
             keepBinary);
     }
 
+    /** {@inheritDoc} */
+    @Override public synchronized void clearReserveForLoad(GridCacheVersion ver) throws IgniteCheckedException {
+        if (obsoleteVersionExtras() != null)
+            return;
+
+        if (ver.equals(this.ver)) {
+            assert evictionDisabled() : this;
+
+            flags &= ~IS_EVICT_DISABLED;
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public synchronized GridCacheVersion versionedValue(CacheObject val,
         GridCacheVersion curVer,
         GridCacheVersion newVer)
-        throws IgniteCheckedException, GridCacheEntryRemovedException {
-
+        throws IgniteCheckedException, GridCacheEntryRemovedException
+    {
         checkObsolete();
 
         if (curVer == null || curVer.equals(ver)) {
@@ -3587,6 +3649,8 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
 
                 return newVer;
             }
+
+            assert !evictionDisabled() : this;
         }
 
         return null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 2e22d9eda13a0..d0f209df94ca0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
@@ -449,7 +450,7 @@ private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
                     GridCacheVersion ver = null;
 
                     if (needVer) {
-                        T2<CacheObject, GridCacheVersion> res = entry.innerGetVersioned(
+                        EntryGetResult res = entry.innerGetVersioned(
                             null,
                             null,
                             /*swap*/true,
@@ -463,8 +464,8 @@ private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
                             !deserializeBinary);
 
                         if (res != null) {
-                            v = res.get1();
-                            ver = res.get2();
+                            v = res.value();
+                            ver = res.version();
                         }
                     }
                     else {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index aeb7eba56d1f5..e188a3523a3a5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
@@ -373,7 +374,7 @@ private boolean localGet(AffinityTopologyVersion topVer, int part) {
                     GridCacheVersion ver = null;
 
                     if (needVer) {
-                        T2<CacheObject, GridCacheVersion> res = entry.innerGetVersioned(
+                        EntryGetResult res = entry.innerGetVersioned(
                             null,
                             null,
                             /*swap*/true,
@@ -387,8 +388,8 @@ private boolean localGet(AffinityTopologyVersion topVer, int part) {
                             true);
 
                         if (res != null) {
-                            v = res.get1();
-                            ver = res.get2();
+                            v = res.value();
+                            ver = res.version();
                         }
                     }
                     else {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 940c74ed863af..94a049ea86c6e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -48,6 +48,7 @@
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheOperationContext;
 import org.apache.ignite.internal.processors.cache.CacheStorePartialUpdateException;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.GridCacheConcurrentMap;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
@@ -1493,7 +1494,7 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
                             GridCacheVersion ver = null;
 
                             if (needVer) {
-                                T2<CacheObject, GridCacheVersion> res = entry.innerGetVersioned(
+                                EntryGetResult res = entry.innerGetVersioned(
                                     null,
                                     null,
                                     /*swap*/true,
@@ -1507,8 +1508,8 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
                                     true);
 
                                 if (res != null) {
-                                    v = res.get1();
-                                    ver = res.get2();
+                                    v = res.value();
+                                    ver = res.version();
                                 }
                             }
                             else {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 176a90f5894db..56af95e6b65c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheOperationContext;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.GridCacheConcurrentMap;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
@@ -484,7 +485,7 @@ public final IgniteInternalFuture<Map<K, V>> loadAsync(
                             GridCacheVersion ver = null;
 
                             if (needVer) {
-                                T2<CacheObject, GridCacheVersion> res = entry.innerGetVersioned(
+                                EntryGetResult res = entry.innerGetVersioned(
                                     null,
                                     null,
                                     /*swap*/true,
@@ -498,8 +499,8 @@ public final IgniteInternalFuture<Map<K, V>> loadAsync(
                                     !deserializeBinary);
 
                                 if (res != null) {
-                                    v = res.get1();
-                                    ver = res.get2();
+                                    v = res.value();
+                                    ver = res.version();
                                 }
                             }
                             else {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index b7fcbbd21a5cc..ab0e88cbf375a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
@@ -437,7 +438,7 @@ private Map<KeyCacheObject, GridNearCacheEntry> map(
                 // First we peek into near cache.
                 if (isNear) {
                     if (needVer) {
-                        T2<CacheObject, GridCacheVersion> res = entry.innerGetVersioned(
+                        EntryGetResult res = entry.innerGetVersioned(
                             null,
                             null,
                             /*swap*/true,
@@ -451,8 +452,8 @@ private Map<KeyCacheObject, GridNearCacheEntry> map(
                             !deserializeBinary);
 
                         if (res != null) {
-                            v = res.get1();
-                            ver = res.get2();
+                            v = res.value();
+                            ver = res.version();
                         }
                     }
                     else {
@@ -577,7 +578,7 @@ private boolean localDhtGet(KeyCacheObject key,
                     boolean isNew = dhtEntry.isNewLocked() || !dhtEntry.valid(topVer);
 
                     if (needVer) {
-                        T2<CacheObject, GridCacheVersion> res = dhtEntry.innerGetVersioned(
+                        EntryGetResult res = dhtEntry.innerGetVersioned(
                             null,
                             null,
                             /*swap*/true,
@@ -591,8 +592,8 @@ private boolean localDhtGet(KeyCacheObject key,
                             !deserializeBinary);
 
                         if (res != null) {
-                            v = res.get1();
-                            ver = res.get2();
+                            v = res.value();
+                            ver = res.version();
                         }
                     }
                     else {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index a419887698326..d1acada062cb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -45,6 +45,7 @@
 import org.apache.ignite.internal.processors.cache.CacheOperationContext;
 import org.apache.ignite.internal.processors.cache.CachePartialUpdateCheckedException;
 import org.apache.ignite.internal.processors.cache.CacheStorePartialUpdateException;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
@@ -516,7 +517,7 @@ private Map<K, V> getAllInternal(@Nullable Collection<? extends K> keys,
                         GridCacheVersion ver;
 
                         if (needVer) {
-                            T2<CacheObject, GridCacheVersion> res = entry.innerGetVersioned(
+                            EntryGetResult res = entry.innerGetVersioned(
                                 null,
                                 null,
                                 /*swap*/swapOrOffheap,
@@ -530,8 +531,8 @@ private Map<K, V> getAllInternal(@Nullable Collection<? extends K> keys,
                                 !deserializeBinary);
 
                             if (res != null) {
-                                v = res.get1();
-                                ver = res.get2();
+                                v = res.value();
+                                ver = res.version();
 
                                 ctx.addResult(
                                     vals,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index cf692640c2365..f784ba2942452 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -1141,7 +1141,7 @@ protected final void processDhtTxFinishRequest(final UUID nodeId, final GridDhtT
         else
             sendReply(nodeId, req, true, null);
 
-        assert req.txState() != null || (ctx.tm().tx(req.version()) == null && ctx.tm().nearTx(req.version()) == null);
+        assert req.txState() != null || (ctx.tm().tx(req.version()) == null && ctx.tm().nearTx(req.version()) == null) : req;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index e2f8438215005..91c9c92075eab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.internal.processors.cache.CacheInvokeEntry;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheOperationContext;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.EntryProcessorResourceInjectorProxy;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
@@ -426,7 +427,7 @@ protected boolean commitAfterLock() {
                         continue;
 
                     try {
-                        T2<CacheObject, GridCacheVersion> res = entry.innerGetVersioned(
+                        EntryGetResult res = entry.innerGetVersioned(
                             null,
                             this,
                             /*readSwap*/true,
@@ -446,7 +447,7 @@ protected boolean commitAfterLock() {
                             misses.put(key, entry.version());
                         }
                         else
-                            c.apply(key, skipVals ? true : res.get1(), res.get2());
+                            c.apply(key, skipVals ? true : res.value(), res.version());
 
                         break;
                     }
@@ -1220,7 +1221,7 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                     F.first(txEntry.entryProcessors()) : null;
 
                             if (needVer) {
-                                T2<CacheObject, GridCacheVersion> res = txEntry.cached().innerGetVersioned(
+                                EntryGetResult res = txEntry.cached().innerGetVersioned(
                                     null,
                                     this,
                                     /*swap*/true,
@@ -1234,8 +1235,8 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                     txEntry.keepBinary());
 
                                 if (res != null) {
-                                    val = res.get1();
-                                    readVer = res.get2();
+                                    val = res.value();
+                                    readVer = res.version();
                                 }
                             }
                             else {
@@ -1303,7 +1304,7 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                 optimistic() ? accessPolicy(cacheCtx, txKey, expiryPlc) : null;
 
                             if (needReadVer) {
-                                T2<CacheObject, GridCacheVersion> res = primaryLocal(entry) ?
+                                EntryGetResult res = primaryLocal(entry) ?
                                     entry.innerGetVersioned(
                                         null,
                                         this,
@@ -1318,8 +1319,8 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                         !deserializeBinary) : null;
 
                                 if (res != null) {
-                                    val = res.get1();
-                                    readVer = res.get2();
+                                    val = res.value();
+                                    readVer = res.version();
                                 }
                             }
                             else {
@@ -1654,7 +1655,7 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                                             F.first(txEntry.entryProcessors()) : null;
 
                                     if (needVer) {
-                                        T2<CacheObject, GridCacheVersion> res = cached.innerGetVersioned(
+                                        EntryGetResult res = cached.innerGetVersioned(
                                             null,
                                             IgniteTxLocalAdapter.this,
                                             /*swap*/cacheCtx.isSwapOrOffheapEnabled(),
@@ -1668,8 +1669,8 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                                             txEntry.keepBinary());
 
                                         if (res != null) {
-                                            val = res.get1();
-                                            readVer = res.get2();
+                                            val = res.value();
+                                            readVer = res.version();
                                         }
                                     }
                                     else{
@@ -2377,7 +2378,7 @@ private boolean enlistWriteEntry(GridCacheContext cacheCtx,
                     if (optimistic() && !implicit()) {
                         try {
                             if (needReadVer) {
-                                T2<CacheObject, GridCacheVersion> res = primaryLocal(entry) ?
+                                EntryGetResult res = primaryLocal(entry) ?
                                     entry.innerGetVersioned(
                                         null,
                                         this,
@@ -2392,8 +2393,8 @@ private boolean enlistWriteEntry(GridCacheContext cacheCtx,
                                         keepBinary) : null;
 
                                 if (res != null) {
-                                    old = res.get1();
-                                    readVer = res.get2();
+                                    old = res.value();
+                                    readVer = res.version();
                                 }
                             }
                             else {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java
new file mode 100644
index 0000000000000..87baa490902b5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.integration.CacheLoaderException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test was added to check fix for IGNITE-4465.
+ */
+public class CacheConcurrentReadThroughTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int SYS_THREADS = 16;
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        cfg.setClientMode(client);
+
+        if (!client) {
+            cfg.setPublicThreadPoolSize(SYS_THREADS);
+            cfg.setSystemThreadPoolSize(SYS_THREADS);
+        }
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentReadThrough() throws Exception {
+        startGrid(0);
+
+        client = true;
+
+        Ignite client = startGrid(1);
+
+        assertTrue(client.configuration().isClientMode());
+
+        IgniteCompute compute = client.compute().withAsync();
+
+        for (int iter = 0; iter < 10; iter++) {
+            CacheConfiguration ccfg = new CacheConfiguration();
+
+            final String cacheName = "test-" + iter;
+
+            ccfg.setName(cacheName);
+            ccfg.setReadThrough(true);
+            ccfg.setCacheStoreFactory(new TestStoreFactory());
+            ccfg.setStatisticsEnabled(true);
+
+            client.createCache(ccfg);
+
+            final Integer key = 1;
+
+            TestCacheStore.loadCnt.set(0);
+
+            Collection<IgniteFuture<?>> futs = new ArrayList<>();
+
+            for (int i = 0; i < SYS_THREADS * 3; i++) {
+                compute.run(new IgniteRunnable() {
+                    @IgniteInstanceResource
+                    private transient Ignite ignite;
+
+                    @Override public void run() {
+                        assertFalse(ignite.configuration().isClientMode());
+
+                        Object v = ignite.<Integer, Integer>cache(cacheName).get(key);
+
+                        if (v == null)
+                            throw new IgniteException("Failed to get value");
+                    }
+                });
+
+                futs.add(compute.future());
+            }
+
+            for (IgniteFuture<?> fut : futs)
+                fut.get();
+
+            int loadCnt = TestCacheStore.loadCnt.get();
+
+            long misses = ignite(1).cache(cacheName).metrics().getCacheMisses();
+
+            log.info("Iteration [iter=" + iter + ", loadCnt=" + loadCnt + ", misses=" + misses + ']');
+
+            assertTrue("Unexpected loadCnt: " + loadCnt, loadCnt > 0 && loadCnt <= SYS_THREADS);
+            assertTrue("Unexpected misses: " + misses, misses > 0 && misses <= SYS_THREADS);
+
+            client.destroyCache(cacheName);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestStoreFactory implements Factory<TestCacheStore> {
+        /** {@inheritDoc} */
+        @Override public TestCacheStore create() {
+            return new TestCacheStore();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestCacheStore extends CacheStoreAdapter<Integer, Integer> {
+        /** */
+        private static final AtomicInteger loadCnt = new AtomicInteger();
+
+        /** {@inheritDoc} */
+        @Override public Integer load(Integer key) throws CacheLoaderException {
+            loadCnt.incrementAndGet();
+
+            try {
+                Thread.sleep(1000);
+            }
+            catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+
+            return key;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Integer, ? extends Integer> entry) {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) {
+            // No-op.
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java
index 67ec3712e92b4..e7df3c0c6a783 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java
@@ -85,6 +85,11 @@ public class CrossCacheTxRandomOperationsTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 6 * 60 * 1000;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
@@ -170,9 +175,17 @@ protected CacheConfiguration cacheConfiguration(String name,
     }
 
     /**
+     * @param cacheMode Cache mode.
+     * @param writeSync Write synchronization mode.
+     * @param fairAff Fair affinity flag.
+     * @param ignite Node to use.
+     * @param name Cache name.
      */
-    protected void createCache(CacheMode cacheMode, CacheWriteSynchronizationMode writeSync, boolean fairAff,
-        Ignite ignite, String name) {
+    protected void createCache(CacheMode cacheMode,
+        CacheWriteSynchronizationMode writeSync,
+        boolean fairAff,
+        Ignite ignite,
+        String name) {
         ignite.createCache(cacheConfiguration(name, cacheMode, writeSync, fairAff));
     }
 
@@ -269,9 +282,18 @@ private void txOperations(TransactionConcurrency concurrency,
 
             boolean checkData = fullSync && !optimistic;
 
+            long stopTime = System.currentTimeMillis() + 10_000;
+
             for (int i = 0; i < 10_000; i++) {
-                if (i % 100 == 0)
+                if (i % 100 == 0) {
+                    if (System.currentTimeMillis() > stopTime) {
+                        log.info("Stop on timeout, iteration: " + i);
+
+                        break;
+                    }
+
                     log.info("Iteration: " + i);
+                }
 
                 boolean rollback = i % 10 == 0;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 48621afc1a8b8..b03e9c89c5b3b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -418,7 +418,26 @@ void recheckLock() {
     }
 
     /** @inheritDoc */
-    @Nullable @Override public T2<CacheObject, GridCacheVersion> innerGetVersioned(
+    @Override public void clearReserveForLoad(GridCacheVersion ver) {
+        assert false;
+    }
+
+    /** @inheritDoc */
+    @Override public EntryGetResult innerGetAndReserveForLoad(
+        boolean readSwap,
+        boolean updateMetrics,
+        boolean evt,
+        UUID subjId,
+        String taskName,
+        @Nullable IgniteCacheExpiryPolicy expiryPlc,
+        boolean keepBinary) throws IgniteCheckedException, GridCacheEntryRemovedException {
+        assert false;
+
+        return null;
+    }
+
+    /** @inheritDoc */
+    @Nullable @Override public EntryGetResult innerGetVersioned(
         @Nullable GridCacheVersion ver,
         IgniteInternalTx tx,
         boolean readSwap,
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
index f632f677ff0a0..8792ea16ac9af 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.cache.affinity.fair.FairAffinityFunctionExcludeNeighborsSelfTest;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionBackupFilterSelfTest;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionExcludeNeighborsSelfTest;
+import org.apache.ignite.internal.processors.cache.CacheConcurrentReadThroughTest;
 import org.apache.ignite.internal.processors.cache.CacheConfigurationLeakTest;
 import org.apache.ignite.internal.processors.cache.CacheDhtLocalPartitionAfterRemoveSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheEnumOperationsSingleNodeTest;
@@ -268,6 +269,7 @@ public static TestSuite suite() throws Exception {
 
         suite.addTest(new TestSuite(CacheExchangeMessageDuplicatedStateTest.class));
         suite.addTest(new TestSuite(OffheapCacheOnClientsTest.class));
+        suite.addTest(new TestSuite(CacheConcurrentReadThroughTest.class));
 
         return suite;
     }

From 74d0dcc6c56118f9e4fdaa4aa70d25d1abe7b80e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 17 Jan 2017 15:00:08 +0300
Subject: [PATCH 103/446] IGNITE-3964: SQL: add support for custom table name.
 This closes #1301.

---
 .../org/apache/ignite/cache/QueryEntity.java  |  21 ++
 .../processors/query/GridQueryProcessor.java  |  26 +-
 .../query/GridQueryTypeDescriptor.java        |   7 +
 .../processors/query/h2/IgniteH2Indexing.java |  16 +-
 .../IgniteCacheAbstractQuerySelfTest.java     | 260 +++++++++++++++++-
 .../h2/GridIndexingSpiAbstractSelfTest.java   |   5 +
 6 files changed, 325 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java b/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java
index 9758cfc0600ef..3d0247873ede0 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java
@@ -48,6 +48,9 @@ public class QueryEntity implements Serializable {
     /** Collection of query indexes. */
     private Map<String, QueryIndex> idxs = new HashMap<>();
 
+    /** Table name. */
+    private String tableName;
+
     /**
      * Creates an empty query entity.
      */
@@ -169,6 +172,24 @@ public void setIndexes(Collection<QueryIndex> idxs) {
         }
     }
 
+
+    /**
+     * Gets table name for this query entity.
+     *
+     * @return table name
+     */
+    public String getTableName() {
+        return tableName;
+    }
+
+    /**
+     * Sets table name for this query entity.
+     * @param tableName table name
+     */
+    public void setTableName(String tableName) {
+        this.tableName = tableName;
+    }
+
     /**
      * Utility method for building query entities programmatically.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 6c093ee7b541c..0f2bc9a74de8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -253,10 +253,12 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     if (keyCls == null)
                         keyCls = Object.class;
 
-                    String simpleValType = valCls == null ? typeName(qryEntity.getValueType()) : typeName(valCls);
+                    String simpleValType = ((valCls == null) ? typeName(qryEntity.getValueType()) : typeName(valCls));
 
                     desc.name(simpleValType);
 
+                    desc.tableName(qryEntity.getTableName());
+
                     if (binaryEnabled && !keyOrValMustDeserialize) {
                         // Safe to check null.
                         if (SQL_TYPES.contains(valCls))
@@ -466,7 +468,7 @@ private boolean mustDeserializeBinary(Class cls) {
      * @param desc Type descriptor.
      * @throws IgniteCheckedException If failed.
      */
-    private void addTypeByName(CacheConfiguration<?,?> ccfg, TypeDescriptor desc) throws IgniteCheckedException {
+    private void addTypeByName(CacheConfiguration<?, ?> ccfg, TypeDescriptor desc) throws IgniteCheckedException {
         if (typesByName.putIfAbsent(new TypeName(ccfg.getName(), desc.name()), desc) != null)
             throw new IgniteCheckedException("Type with name '" + desc.name() + "' already indexed " +
                 "in cache '" + ccfg.getName() + "'.");
@@ -2108,6 +2110,9 @@ private static class TypeDescriptor implements GridQueryTypeDescriptor {
         /** */
         private String name;
 
+        /** */
+        private String tblName;
+
         /** Value field names and types with preserved order. */
         @GridToStringInclude
         private final Map<String, Class<?>> fields = new LinkedHashMap<>();
@@ -2166,6 +2171,23 @@ void name(String name) {
             this.name = name;
         }
 
+        /**
+         * Gets table name for type.
+         * @return Table name.
+         */
+        public String tableName() {
+            return tblName;
+        }
+
+        /**
+         * Sets table name for type.
+         *
+         * @param tblName Table name.
+         */
+        public void tableName(String tblName) {
+            this.tblName = tblName;
+        }
+
         /** {@inheritDoc} */
         @Override public Map<String, Class<?>> fields() {
             return fields;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryTypeDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryTypeDescriptor.java
index b63684173c430..e93952510986e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryTypeDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryTypeDescriptor.java
@@ -31,6 +31,13 @@ public interface GridQueryTypeDescriptor {
      */
     public String name();
 
+    /**
+     * Gets table name for type.
+     *
+     * @return Table name.
+     */
+    public String tableName();
+
     /**
      * Gets mapping from field name to its type.
      *
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 362ddd853443c..bc5155221d878 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -741,7 +741,7 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
 
         tbl.onDrop();
 
-        tbl.schema.tbls.remove(tbl.name());
+        tbl.schema.tbls.remove(tbl.typeName());
     }
 
     /** {@inheritDoc} */
@@ -2397,7 +2397,9 @@ private class TableDescriptor implements GridH2Table.IndexesFactory {
             this.type = type;
             this.schema = schema;
 
-            fullTblName = schema.schemaName + "." + escapeName(type.name(), schema.escapeAll());
+            String tblName = escapeName(type.tableName() != null ? type.tableName() : type.name(), schema.escapeAll());
+
+            fullTblName = schema.schemaName + "." + tblName;
         }
 
         /**
@@ -2408,16 +2410,16 @@ public String schemaName() {
         }
 
         /**
-         * @return Database table name.
+         * @return Database full table name.
          */
         String fullTableName() {
             return fullTblName;
         }
 
         /**
-         * @return Database table name.
+         * @return type name.
          */
-        String name() {
+        String typeName() {
             return type.name();
         }
 
@@ -2739,8 +2741,8 @@ private Schema(String spaceName, String schemaName, GridCacheContext<?,?> cctx,
          * @param tbl Table descriptor.
          */
         public void add(TableDescriptor tbl) {
-            if (tbls.putIfAbsent(tbl.name(), tbl) != null)
-                throw new IllegalStateException("Table already registered: " + tbl.name());
+            if (tbls.putIfAbsent(tbl.typeName(), tbl) != null)
+                throw new IllegalStateException("Table already registered: " + tbl.fullTableName());
         }
 
         /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
index 7c5b4722b2dfa..ad6922cea0902 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
@@ -23,6 +23,7 @@
 import java.io.ObjectOutput;
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -45,10 +46,13 @@
 import org.apache.ignite.IgniteBinary;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.ScanQuery;
@@ -66,11 +70,11 @@
 import org.apache.ignite.events.CacheQueryExecutedEvent;
 import org.apache.ignite.events.CacheQueryReadEvent;
 import org.apache.ignite.events.Event;
-import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.query.QueryCursorEx;
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
+import org.apache.ignite.internal.util.lang.GridPlainCallable;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
@@ -180,6 +184,32 @@ protected NearCacheConfiguration nearCacheConfiguration() {
                     Long.class, EnumObject.class
                 );
 
+                List<QueryEntity> entityList = new ArrayList<>();
+
+                QueryEntity qryEntity = new QueryEntity();
+
+                qryEntity.setKeyType(Integer.class.getName());
+                qryEntity.setValueType(Type1.class.getName());
+                qryEntity.addQueryField("id", Integer.class.getName(), null);
+                qryEntity.addQueryField("name", String.class.getName(), null);
+                qryEntity.setTableName("Type2");
+                qryEntity.setIndexes(Arrays.asList(new QueryIndex("id")));
+
+                entityList.add(qryEntity);
+
+                qryEntity = new QueryEntity();
+
+                qryEntity.setKeyType(Integer.class.getName());
+                qryEntity.setValueType(Type2.class.getName());
+                qryEntity.addQueryField("id", Integer.class.getName(), null);
+                qryEntity.addQueryField("name", String.class.getName(), null);
+                qryEntity.setTableName("Type1");
+                qryEntity.setIndexes(Arrays.asList(new QueryIndex("id")));
+
+                entityList.add(qryEntity);
+
+                cc.setQueryEntities(entityList);
+
                 if (cacheMode() != CacheMode.LOCAL)
                     cc.setAffinity(new RendezvousAffinityFunction());
 
@@ -234,6 +264,7 @@ protected Ignite ignite() {
 
         stopAllGrids();
 
+
         store.reset();
     }
 
@@ -543,6 +574,113 @@ public void testSelectQuery() throws Exception {
         assert iter.next() != null;
     }
 
+    /**
+     * JUnit.
+     *
+     * @throws Exception In case of error.
+     */
+    public void testSimpleCustomTableName() throws Exception {
+        final IgniteCache<Integer, Object> cache = ignite().cache(null);
+
+        cache.put(10, new Type1(1, "Type1 record #1"));
+        cache.put(20, new Type1(2, "Type1 record #2"));
+
+        QueryCursor<Cache.Entry<Integer, Type1>> qry1 =
+            cache.query(new SqlQuery<Integer, Type1>(Type1.class, "FROM Type2"));
+
+        List<Cache.Entry<Integer, Type1>> all = qry1.getAll();
+
+        assertEquals(2, all.size());
+
+        QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery("SELECT name FROM Type2"));
+
+        assertEquals(2, qry.getAll().size());
+
+        GridTestUtils.assertThrows(log, new GridPlainCallable<Void>() {
+            @Override public Void call() throws Exception {
+                QueryCursor<Cache.Entry<Integer, Type1>> qry =
+                    cache.query(new SqlQuery<Integer, Type1>(Type1.class, "FROM Type1"));
+
+                qry.getAll();
+
+                return null;
+            }
+        }, IgniteException.class, null);
+    }
+
+    /**
+     * JUnit.
+     *
+     * @throws Exception In case of error.
+     */
+    public void testMixedCustomTableName() throws Exception {
+        final IgniteCache<Integer, Object> cache = ignite().cache(null);
+
+        cache.put(10, new Type1(1, "Type1 record #1"));
+        cache.put(20, new Type1(2, "Type1 record #2"));
+        cache.put(30, new Type2(1, "Type2 record #1"));
+        cache.put(40, new Type2(2, "Type2 record #2"));
+        cache.put(50, new Type2(3, "Type2 record #3"));
+
+        QueryCursor<Cache.Entry<Integer, Type1>> qry1 =
+            cache.query(new SqlQuery<Integer, Type1>(Type1.class, "FROM Type2"));
+
+        List<Cache.Entry<Integer, Type1>> all = qry1.getAll();
+
+        assertEquals(2, all.size());
+
+        QueryCursor<Cache.Entry<Integer, Type2>> qry2 =
+            cache.query(new SqlQuery<Integer, Type2>(Type2.class, "FROM Type1"));
+
+        assertEquals(3, qry2.getAll().size());
+
+        QueryCursor<List<?>> qry = cache.query(new SqlFieldsQuery("SELECT name FROM Type1"));
+
+        assertEquals(3, qry.getAll().size());
+
+        qry = cache.query(new SqlFieldsQuery("SELECT name FROM Type2"));
+
+        assertEquals(2, qry.getAll().size());
+
+        GridTestUtils.assertThrows(log, new GridPlainCallable<Void>() {
+            @Override public Void call() throws Exception {
+                QueryCursor<Cache.Entry<Integer, Type1>> qry1 =
+                    cache.query(new SqlQuery<Integer, Type1>(Type1.class, "FROM Type1"));
+
+                qry1.getAll().size();
+
+                return null;
+            }
+        }, IgniteException.class, null);
+    }
+
+    /**
+     * JUnit.
+     *
+     * @throws Exception In case of error.
+     */
+    public void testDistributedJoinCustomTableName() throws Exception {
+        IgniteCache<Integer, Object> cache = ignite().cache(null);
+
+        cache.put(10, new Type1(1, "Type1 record #1"));
+        cache.put(20, new Type1(2, "Type1 record #2"));
+        cache.put(30, new Type2(1, "Type2 record #1"));
+        cache.put(40, new Type2(2, "Type2 record #2"));
+        cache.put(50, new Type2(3, "Type2 record #3"));
+
+        QueryCursor<List<?>> query = cache.query(
+            new SqlFieldsQuery("SELECT t2.name, t1.name FROM Type2 as t2 LEFT JOIN Type1 as t1 ON t1.id = t2.id")
+                .setDistributedJoins(cacheMode() == PARTITIONED));
+
+        assertEquals(2, query.getAll().size());
+
+        query = cache.query(
+            new SqlFieldsQuery("SELECT t2.name, t1.name FROM Type2 as t2 RIGHT JOIN Type1 as t1 ON t1.id = t2.id")
+                .setDistributedJoins(cacheMode() == PARTITIONED));
+
+        assertEquals(3, query.getAll().size());
+    }
+
     /**
      * JUnit.
      *
@@ -1653,6 +1791,126 @@ public int salary() {
         }
     }
 
+    /**
+     *
+     */
+    public static class Type1 implements Serializable {
+        /** */
+        private int id;
+
+        /** */
+        private String name;
+
+        /**
+         * @param id ID.
+         * @param name Name.
+         */
+        Type1(int id, String name) {
+            assert name != null;
+            assert id > 0;
+
+            this.name = name;
+            this.id = id;
+        }
+
+        /**
+         * @return Name.
+         */
+        public String name() {
+            return name;
+        }
+
+        /**
+         * @return ID.
+         */
+        public int id() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return name.hashCode() + 31 * id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object obj) {
+            if (obj == this)
+                return true;
+
+            if (!(obj instanceof Type1))
+                return false;
+
+            Type1 that = (Type1)obj;
+
+            return that.name.equals(name) && that.id == id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(Type1.class, this);
+        }
+    }
+
+    /**
+     *
+     */
+    public static class Type2 implements Serializable {
+        /** */
+        private int id;
+
+        /** */
+        private String name;
+
+        /**
+         * @param id ID.
+         * @param name Name.
+         */
+        Type2(int id, String name) {
+            assert name != null;
+            assert id > 0;
+
+            this.name = name;
+            this.id = id;
+        }
+
+        /**
+         * @return Name.
+         */
+        public String name() {
+            return name;
+        }
+
+        /**
+         * @return ID.
+         */
+        public int id() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return name.hashCode() + 31 * id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object obj) {
+            if (obj == this)
+                return true;
+
+            if (!(obj instanceof Type2))
+                return false;
+
+            Type2 that = (Type2)obj;
+
+            return that.name.equals(name) && that.id == id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(Type2.class, this);
+        }
+    }
+
     /**
      * Test value object.
      */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
index bcf8f9d9b6b59..81e34d6959f90 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
@@ -539,6 +539,11 @@ private TypeDesc(String space, String name, Map<String, Class<?>> valFields, Gri
             return name;
         }
 
+        /** {@inheritDoc} */
+        @Override public String tableName() {
+            return null;
+        }
+
         /**
          * @return Space name.
          */

From 8e622e41de4acf365da7f933a08b6d31bae11124 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 17 Jan 2017 15:18:33 +0300
Subject: [PATCH 104/446] IGNITE-4247: Sql queries supports table alias. This
 closes #1297.

---
 .../apache/ignite/cache/query/SqlQuery.java   | 25 +++++++++++++
 .../processors/query/GridQueryIndexing.java   |  4 +-
 .../processors/query/GridQueryProcessor.java  |  5 ++-
 .../processors/query/h2/IgniteH2Indexing.java | 14 ++++---
 .../IgniteCacheAbstractQuerySelfTest.java     | 37 +++++++++++++++++++
 .../h2/GridIndexingSpiAbstractSelfTest.java   | 24 ++++++------
 6 files changed, 89 insertions(+), 20 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/query/SqlQuery.java b/modules/core/src/main/java/org/apache/ignite/cache/query/SqlQuery.java
index 83e171d4ab43f..5e36d209dda75 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/query/SqlQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/query/SqlQuery.java
@@ -37,6 +37,9 @@ public final class SqlQuery<K, V> extends Query<Cache.Entry<K, V>> {
     /** */
     private String type;
 
+    /** Table alias */
+    private String alias;
+
     /** SQL clause. */
     private String sql;
 
@@ -137,6 +140,27 @@ public SqlQuery<K, V> setType(String type) {
         return this;
     }
 
+    /**
+     * Sets table alias for type.
+     *
+     * @return Table alias.
+     */
+    public String getAlias() {
+        return alias;
+    }
+
+    /**
+     * Gets table alias for type.
+     *
+     * @param alias table alias for type that is used in query.
+     * @return {@code this} For chaining.
+     */
+    public SqlQuery<K, V> setAlias(String alias) {
+        this.alias = alias;
+
+        return this;
+    }
+
     /**
      * Gets the query execution timeout in milliseconds.
      *
@@ -148,6 +172,7 @@ public int getTimeout() {
 
     /**
      * Sets the query execution timeout. Query will be automatically cancelled if the execution timeout is exceeded.
+     *
      * @param timeout Timeout value. Zero value disables timeout.
      * @param timeUnit Time granularity.
      * @return {@code this} For chaining.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index 6bffa5d5d04b7..539ebc07fec79 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -95,6 +95,7 @@ public GridQueryFieldsResult queryLocalSqlFields(@Nullable String spaceName, Str
      *
      * @param spaceName Space name.
      * @param qry Query.
+     * @param alias Table alias used in Query.
      * @param params Query parameters.
      * @param type Query return type.
      * @param filter Space name and key filter.
@@ -102,7 +103,8 @@ public GridQueryFieldsResult queryLocalSqlFields(@Nullable String spaceName, Str
      * @throws IgniteCheckedException If failed.
      */
     public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable String spaceName, String qry,
-        Collection<Object> params, GridQueryTypeDescriptor type, IndexingQueryFilter filter) throws IgniteCheckedException;
+        String alias, Collection<Object> params, GridQueryTypeDescriptor type, IndexingQueryFilter filter)
+        throws IgniteCheckedException;
 
     /**
      * Executes text query.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 0f2bc9a74de8b..f4ac4ae71a546 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -796,7 +796,7 @@ public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> query(final String spac
                     if (type == null || !type.registered())
                         throw new CacheException("Failed to find SQL table for type: " + resType);
 
-                    return idx.queryLocalSql(space, clause, params, type, filters);
+                    return idx.queryLocalSql(space, clause, null, params, type, filters);
                 }
             }, false);
         }
@@ -890,7 +890,8 @@ public <K, V> Iterator<Cache.Entry<K, V>> queryLocal(
 
                         final GridCloseableIterator<IgniteBiTuple<K, V>> i = idx.queryLocalSql(
                             space,
-                            sqlQry,
+                            qry.getSql(),
+                            qry.getAlias(),
                             F.asList(params),
                             typeDesc,
                             idx.backupFilter(requestTopVer.get(), null));
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index bc5155221d878..cbf2ebde27056 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -1046,14 +1046,14 @@ public void setupConnection(Connection conn, boolean distributedJoins, boolean e
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable String spaceName,
-        final String qry, @Nullable final Collection<Object> params, GridQueryTypeDescriptor type,
+        final String qry, String alias, @Nullable final Collection<Object> params, GridQueryTypeDescriptor type,
         final IndexingQueryFilter filter) throws IgniteCheckedException {
         final TableDescriptor tbl = tableDescriptor(spaceName, type);
 
         if (tbl == null)
             throw new CacheException("Failed to find SQL table for type: " + type.name());
 
-        String sql = generateQuery(qry, tbl);
+        String sql = generateQuery(qry, alias, tbl);
 
         Connection conn = connectionForThread(tbl.schemaName());
 
@@ -1103,7 +1103,7 @@ private Iterable<List<?>> runQueryTwoStep(final GridCacheContext<?,?> cctx, fina
         String sql;
 
         try {
-            sql = generateQuery(qry.getSql(), tblDesc);
+            sql = generateQuery(qry.getSql(), qry.getAlias(), tblDesc);
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException(e);
@@ -1300,11 +1300,12 @@ public static Session session(Connection c) {
      * Prepares statement for query.
      *
      * @param qry Query string.
+     * @param tableAlias table alias.
      * @param tbl Table to use.
      * @return Prepared statement.
      * @throws IgniteCheckedException In case of error.
      */
-    private String generateQuery(String qry, TableDescriptor tbl) throws IgniteCheckedException {
+    private String generateQuery(String qry, String tableAlias, TableDescriptor tbl) throws IgniteCheckedException {
         assert tbl != null;
 
         final String qry0 = qry;
@@ -1341,10 +1342,13 @@ else if (star > 0) {
         }
 
         if (!upper.startsWith("FROM"))
-            from = " FROM " + t +
+            from = " FROM " + t + (tableAlias != null ? " as " + tableAlias : "") +
                 (upper.startsWith("WHERE") || upper.startsWith("ORDER") || upper.startsWith("LIMIT") ?
                     " " : " WHERE ");
 
+        if(tableAlias != null)
+            t = tableAlias;
+
         qry = "SELECT " + t + "." + KEY_FIELD_NAME + ", " + t + "." + VAL_FIELD_NAME + from + qry;
 
         return qry;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
index ad6922cea0902..c5a241e54ce1f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
@@ -347,6 +347,43 @@ public void testIntegerType() throws Exception {
         assertEquals(val, entry.getValue().intValue());
     }
 
+    /**
+     * Test table alias in SqlQuery.
+     *
+     * @throws Exception In case of error.
+     */
+    public void testTableAliasInSqlQuery() throws Exception {
+        IgniteCache<Integer, Integer> cache = ignite().cache(null);
+
+        int key = 898;
+
+        int val = 2;
+
+        cache.put(key, val);
+
+        SqlQuery<Integer, Integer> sqlQry = new SqlQuery<>(Integer.class, "t1._key = ? and t1._val > 1");
+
+        QueryCursor<Cache.Entry<Integer, Integer>> qry = cache.query(sqlQry.setAlias("t1").setArgs(key));
+
+        Cache.Entry<Integer, Integer> entry = F.first(qry.getAll());
+
+        assert entry != null;
+
+        assertEquals(key, entry.getKey().intValue());
+        assertEquals(val, entry.getValue().intValue());
+
+        sqlQry = new SqlQuery<>(Integer.class, "FROM Integer as t1 WHERE t1._key = ? and t1._val > 1");
+
+        qry = cache.query(sqlQry.setAlias("t1").setArgs(key));
+
+        entry = F.first(qry.getAll());
+
+        assert entry != null;
+
+        assertEquals(key, entry.getKey().intValue());
+        assertEquals(val, entry.getValue().intValue());
+    }
+
     /**
      * Tests UDFs.
      *
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
index 81e34d6959f90..ad8a7e34e82a7 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
@@ -223,15 +223,15 @@ public void testSpi() throws Exception {
         assertEquals(0, spi.size(typeAB.space(), typeAB));
         assertEquals(0, spi.size(typeBA.space(), typeBA));
 
-        assertFalse(spi.queryLocalSql(typeAA.space(), "select * from A.A", Collections.emptySet(), typeAA, null).hasNext());
-        assertFalse(spi.queryLocalSql(typeAB.space(), "select * from A.B", Collections.emptySet(), typeAB, null).hasNext());
-        assertFalse(spi.queryLocalSql(typeBA.space(), "select * from B.A", Collections.emptySet(), typeBA, null).hasNext());
+        assertFalse(spi.queryLocalSql(typeAA.space(), "select * from A.A", null, Collections.emptySet(), typeAA, null).hasNext());
+        assertFalse(spi.queryLocalSql(typeAB.space(), "select * from A.B", null, Collections.emptySet(), typeAB, null).hasNext());
+        assertFalse(spi.queryLocalSql(typeBA.space(), "select * from B.A", null, Collections.emptySet(), typeBA, null).hasNext());
 
-        assertFalse(spi.queryLocalSql(typeBA.space(), "select * from B.A, A.B, A.A",
+        assertFalse(spi.queryLocalSql(typeBA.space(), "select * from B.A, A.B, A.A", null,
             Collections.emptySet(), typeBA, null).hasNext());
 
         try {
-            spi.queryLocalSql(typeBA.space(), "select aa.*, ab.*, ba.* from A.A aa, A.B ab, B.A ba",
+            spi.queryLocalSql(typeBA.space(), "select aa.*, ab.*, ba.* from A.A aa, A.B ab, B.A ba", null,
                 Collections.emptySet(), typeBA, null).hasNext();
 
             fail("Enumerations of aliases in select block must be prohibited");
@@ -240,10 +240,10 @@ public void testSpi() throws Exception {
             // all fine
         }
 
-        assertFalse(spi.queryLocalSql(typeAB.space(), "select ab.* from A.B ab",
+        assertFalse(spi.queryLocalSql(typeAB.space(), "select ab.* from A.B ab", null,
             Collections.emptySet(), typeAB, null).hasNext());
 
-        assertFalse(spi.queryLocalSql(typeBA.space(), "select   ba.*   from B.A  as ba",
+        assertFalse(spi.queryLocalSql(typeBA.space(), "select   ba.*   from B.A  as ba", null,
             Collections.emptySet(), typeBA, null).hasNext());
 
         // Nothing to remove.
@@ -298,7 +298,7 @@ public void testSpi() throws Exception {
 
         // Query data.
         Iterator<IgniteBiTuple<Integer, Map<String, Object>>> res =
-            spi.queryLocalSql(typeAA.space(), "from a order by age", Collections.emptySet(), typeAA, null);
+            spi.queryLocalSql(typeAA.space(), "from a order by age", null, Collections.emptySet(), typeAA, null);
 
         assertTrue(res.hasNext());
         assertEquals(aa(3, "Borya", 18).value(null, false), value(res.next()));
@@ -306,7 +306,7 @@ public void testSpi() throws Exception {
         assertEquals(aa(2, "Valera", 19).value(null, false), value(res.next()));
         assertFalse(res.hasNext());
 
-        res = spi.queryLocalSql(typeAA.space(), "select aa.* from a aa order by aa.age",
+        res = spi.queryLocalSql(typeAA.space(), "select aa.* from a aa order by aa.age", null,
             Collections.emptySet(), typeAA, null);
 
         assertTrue(res.hasNext());
@@ -315,7 +315,7 @@ public void testSpi() throws Exception {
         assertEquals(aa(2, "Valera", 19).value(null, false), value(res.next()));
         assertFalse(res.hasNext());
 
-        res = spi.queryLocalSql(typeAB.space(), "from b order by name", Collections.emptySet(), typeAB, null);
+        res = spi.queryLocalSql(typeAB.space(), "from b order by name", null, Collections.emptySet(), typeAB, null);
 
         assertTrue(res.hasNext());
         assertEquals(ab(1, "Vasya", 20, "Some text about Vasya goes here.").value(null, false), value(res.next()));
@@ -323,7 +323,7 @@ public void testSpi() throws Exception {
         assertEquals(ab(4, "Vitalya", 20, "Very Good guy").value(null, false), value(res.next()));
         assertFalse(res.hasNext());
 
-        res = spi.queryLocalSql(typeAB.space(), "select bb.* from b as bb order by bb.name",
+        res = spi.queryLocalSql(typeAB.space(), "select bb.* from b as bb order by bb.name", null,
             Collections.emptySet(), typeAB, null);
 
         assertTrue(res.hasNext());
@@ -333,7 +333,7 @@ public void testSpi() throws Exception {
         assertFalse(res.hasNext());
 
 
-        res = spi.queryLocalSql(typeBA.space(), "from a", Collections.emptySet(), typeBA, null);
+        res = spi.queryLocalSql(typeBA.space(), "from a", null, Collections.emptySet(), typeBA, null);
 
         assertTrue(res.hasNext());
         assertEquals(ba(2, "Kolya", 25, true).value(null, false), value(res.next()));

From a922ac9d17f91f25aaa2bac9f0a2622dbd04c9bb Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 17 Jan 2017 15:31:04 +0300
Subject: [PATCH 105/446] IGNITE-4540: IndexingSPI can be used without have
 default H2 Indexing enabled. This closes #1423.

---
 .../cache/query/GridCacheQueryManager.java    | 83 +++++++++++++++++--
 .../processors/query/GridQueryProcessor.java  | 46 ----------
 .../cache/query/IndexingSpiQuerySelfTest.java | 66 +++++++--------
 ...ndexingSpiQueryWithH2IndexingSelfTest.java | 36 ++++++++
 4 files changed, 145 insertions(+), 86 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryWithH2IndexingSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 85c01d9937151..d64dff4b1a02e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -1,4 +1,4 @@
-/*
+ /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -45,6 +45,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.query.QueryMetrics;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -60,6 +61,7 @@
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.internal.processors.cache.CacheMetricsImpl;
 import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
@@ -167,6 +169,9 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
             }
         };
 
+    /** Default is @{code true} */
+    private final boolean isIndexingSpiAllowsBinary = !IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI);
+
     /** */
     private GridQueryProcessor qryProc;
 
@@ -204,6 +209,10 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
     /** */
     private boolean enabled;
 
+    /** */
+    private boolean qryProcEnabled;
+
+
     /** */
     private AffinityTopologyVersion qryTopVer;
 
@@ -211,9 +220,14 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
     @Override public void start0() throws IgniteCheckedException {
         CacheConfiguration ccfg = cctx.config();
 
+        qryProcEnabled = GridQueryProcessor.isEnabled(ccfg);
+
         qryProc = cctx.kernalContext().query();
+
         space = cctx.name();
 
+        enabled = qryProcEnabled || (isIndexingSpiEnabled() && !CU.isSystemCache(space));
+
         maxIterCnt = ccfg.getMaxQueryIteratorsCount();
 
         detailMetricsSz = ccfg.getQueryDetailMetricsSize();
@@ -259,8 +273,6 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
 
         cctx.events().addListener(lsnr, EVT_NODE_LEFT, EVT_NODE_FAILED);
 
-        enabled = GridQueryProcessor.isEnabled(ccfg);
-
         qryTopVer = cctx.startTopologyVersion();
 
         if (qryTopVer == null)
@@ -369,17 +381,35 @@ void processQueryRequest(UUID sndId, GridCacheQueryRequest req) {
      * @throws IgniteCheckedException If failed.
      */
     public void onSwap(CacheObject key) throws IgniteCheckedException {
+        if(!enabled)
+            return;
+
         if (!enterBusy())
             return; // Ignore index update when node is stopping.
 
         try {
-            qryProc.onSwap(space, key);
+            if (isIndexingSpiEnabled()) {
+                Object key0 = unwrapIfNeeded(key, cctx.cacheObjectContext());
+
+                cctx.kernalContext().indexing().onSwap(space, key0);
+            }
+
+            if(qryProcEnabled)
+                qryProc.onSwap(space, key);
         }
         finally {
             leaveBusy();
         }
     }
 
+    /**
+     * Checks if IndexinSPI is enabled.
+     * @return IndexingSPI enabled flag.
+     */
+    private boolean isIndexingSpiEnabled() {
+        return cctx.kernalContext().indexing().enabled();
+    }
+
     /**
      * Entry for given key unswapped.
      *
@@ -388,11 +418,25 @@ public void onSwap(CacheObject key) throws IgniteCheckedException {
      * @throws IgniteCheckedException If failed.
      */
     public void onUnswap(CacheObject key, CacheObject val) throws IgniteCheckedException {
+        if(!enabled)
+            return;
+
         if (!enterBusy())
             return; // Ignore index update when node is stopping.
 
         try {
-            qryProc.onUnswap(space, key, val);
+            if (isIndexingSpiEnabled()) {
+                CacheObjectContext coctx = cctx.cacheObjectContext();
+
+                Object key0 = unwrapIfNeeded(key, coctx);
+
+                Object val0 = unwrapIfNeeded(val, coctx);
+
+                cctx.kernalContext().indexing().onUnswap(space, key0, val0);
+            }
+
+            if(qryProcEnabled)
+                qryProc.onUnswap(space, key, val);
         }
         finally {
             leaveBusy();
@@ -429,7 +473,18 @@ public void store(CacheObject key, CacheObject val, GridCacheVersion ver, long e
             return; // Ignore index update when node is stopping.
 
         try {
-            qryProc.store(space, key, val, CU.versionToBytes(ver), expirationTime);
+            if (isIndexingSpiEnabled()) {
+                CacheObjectContext coctx = cctx.cacheObjectContext();
+
+                Object key0 = unwrapIfNeeded(key, coctx);
+
+                Object val0 = unwrapIfNeeded(val, coctx);
+
+                cctx.kernalContext().indexing().store(space, key0, val0, expirationTime);
+            }
+
+            if(qryProcEnabled)
+                qryProc.store(space, key, val, CU.versionToBytes(ver), expirationTime);
         }
         finally {
             invalidateResultCache();
@@ -454,7 +509,14 @@ public void remove(CacheObject key, CacheObject val) throws IgniteCheckedExcepti
             return; // Ignore index update when node is stopping.
 
         try {
-            qryProc.remove(space, key, val);
+            if (isIndexingSpiEnabled()) {
+                Object key0 = unwrapIfNeeded(key, cctx.cacheObjectContext());
+
+                cctx.kernalContext().indexing().remove(space, key0);
+            }
+
+            if(qryProcEnabled)
+                qryProc.remove(space, key, val);
         }
         finally {
             invalidateResultCache();
@@ -560,6 +622,13 @@ public abstract GridCloseableIterator scanQueryDistributed(GridCacheQueryAdapter
      */
     public abstract CacheQueryFuture<?> queryFieldsDistributed(GridCacheQueryBean qry, Collection<ClusterNode> nodes);
 
+    /**
+     * Unwrap CacheObject if needed.
+     */
+    private Object unwrapIfNeeded(CacheObject obj, CacheObjectContext coctx) {
+        return isIndexingSpiAllowsBinary && cctx.cacheObjects().isBinaryObject(obj) ? obj : obj.value(coctx, false);
+    }
+
     /**
      * Performs query.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index f4ac4ae71a546..48ca2b5bab69c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -160,9 +160,6 @@ public class GridQueryProcessor extends GridProcessorAdapter {
     /** */
     private static final ThreadLocal<AffinityTopologyVersion> requestTopVer = new ThreadLocal<>();
 
-    /** Default is @{true} */
-    private final boolean isIndexingSpiAllowsBinary = !IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI);
-
     /**
      * @param ctx Kernal context.
      */
@@ -682,16 +679,6 @@ public void store(final String space, final CacheObject key, final CacheObject v
 
         CacheObjectContext coctx = null;
 
-        if (ctx.indexing().enabled()) {
-            coctx = cacheObjectContext(space);
-
-            Object key0 = unwrap(key, coctx);
-
-            Object val0 = unwrap(val, coctx);
-
-            ctx.indexing().store(space, key0, val0, expirationTime);
-        }
-
         if (idx == null)
             return;
 
@@ -744,13 +731,6 @@ public void store(final String space, final CacheObject key, final CacheObject v
         }
     }
 
-    /**
-     * Unwrap CacheObject if needed.
-     */
-    private Object unwrap(CacheObject obj, CacheObjectContext coctx) {
-        return isIndexingSpiAllowsBinary && ctx.cacheObjects().isBinaryObject(obj) ? obj : obj.value(coctx, false);
-    }
-
     /**
      * @throws IgniteCheckedException If failed.
      */
@@ -1039,14 +1019,6 @@ public void remove(String space, CacheObject key, CacheObject val) throws Ignite
         if (log.isDebugEnabled())
             log.debug("Remove [space=" + space + ", key=" + key + ", val=" + val + "]");
 
-        if (ctx.indexing().enabled()) {
-            CacheObjectContext coctx = cacheObjectContext(space);
-
-            Object key0 = unwrap(key, coctx);
-
-            ctx.indexing().remove(space, key0);
-        }
-
         if (idx == null)
             return;
 
@@ -1184,14 +1156,6 @@ public void onSwap(String spaceName, CacheObject key) throws IgniteCheckedExcept
         if (log.isDebugEnabled())
             log.debug("Swap [space=" + spaceName + ", key=" + key + "]");
 
-        if (ctx.indexing().enabled()) {
-            CacheObjectContext coctx = cacheObjectContext(spaceName);
-
-            Object key0 = unwrap(key, coctx);
-
-            ctx.indexing().onSwap(spaceName, key0);
-        }
-
         if (idx == null)
             return;
 
@@ -1221,16 +1185,6 @@ public void onUnswap(String spaceName, CacheObject key, CacheObject val)
         if (log.isDebugEnabled())
             log.debug("Unswap [space=" + spaceName + ", key=" + key + ", val=" + val + "]");
 
-        if (ctx.indexing().enabled()) {
-            CacheObjectContext coctx = cacheObjectContext(spaceName);
-
-            Object key0 = unwrap(key, coctx);
-
-            Object val0 = unwrap(val, coctx);
-
-            ctx.indexing().onUnswap(spaceName, key0, val0);
-        }
-
         if (idx == null)
             return;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
index f66b99ef5b1c7..84a13df36b52f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
@@ -55,14 +55,39 @@
 import org.jetbrains.annotations.Nullable;
 
 /**
- * Indexing Spi query test
+ * Indexing Spi query only test
  */
 public class IndexingSpiQuerySelfTest extends TestCase {
+    public static final String CACHE_NAME = "test-cache";
+
     /** {@inheritDoc} */
     @Override public void tearDown() throws Exception {
         Ignition.stopAll(true);
     }
 
+    /**
+     * @return Configuration.
+     */
+    protected IgniteConfiguration configuration() {
+        IgniteConfiguration cfg = new IgniteConfiguration();
+
+        TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setMaxMissedHeartbeats(Integer.MAX_VALUE);
+
+        disco.setIpFinder(ipFinder);
+
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    /** */
+    protected <K,V> CacheConfiguration<K, V> cacheConfiguration(String cacheName) {
+        return new CacheConfiguration<>(cacheName);
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -73,9 +98,7 @@ public void testSimpleIndexingSpi() throws Exception {
 
         Ignite ignite = Ignition.start(cfg);
 
-        CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>("test-cache");
-
-        ccfg.setIndexedTypes(Integer.class, Integer.class);
+        CacheConfiguration<Integer, Integer> ccfg = cacheConfiguration(CACHE_NAME);
 
         IgniteCache<Integer, Integer> cache = ignite.createCache(ccfg);
 
@@ -98,7 +121,7 @@ public void testIndexingSpiWithDisabledQueryProcessor() throws Exception {
 
         Ignite ignite = Ignition.start(cfg);
 
-        CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>("test-cache");
+        CacheConfiguration<Integer, Integer> ccfg = cacheConfiguration(CACHE_NAME);
 
         IgniteCache<Integer, Integer> cache = ignite.createCache(ccfg);
 
@@ -121,9 +144,7 @@ public void testBinaryIndexingSpi() throws Exception {
 
         Ignite ignite = Ignition.start(cfg);
 
-        CacheConfiguration<PersonKey, Person> ccfg = new CacheConfiguration<>("test-binary-cache");
-
-        ccfg.setIndexedTypes(PersonKey.class, Person.class);
+        CacheConfiguration<PersonKey, Person> ccfg = cacheConfiguration(CACHE_NAME);
 
         IgniteCache<PersonKey, Person> cache = ignite.createCache(ccfg);
 
@@ -155,9 +176,7 @@ public void testNonBinaryIndexingSpi() throws Exception {
 
         Ignite ignite = Ignition.start(cfg);
 
-        CacheConfiguration<PersonKey, Person> ccfg = new CacheConfiguration<>("test-binary-cache");
-
-        ccfg.setIndexedTypes(PersonKey.class, Person.class);
+        CacheConfiguration<PersonKey, Person> ccfg = cacheConfiguration(CACHE_NAME);
 
         IgniteCache<PersonKey, Person> cache = ignite.createCache(ccfg);
 
@@ -187,10 +206,9 @@ public void testIndexingSpiFailure() throws Exception {
 
         Ignite ignite = Ignition.start(cfg);
 
-        CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>("test-cache");
+        CacheConfiguration<Integer, Integer> ccfg = cacheConfiguration(CACHE_NAME);
 
         ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
-        ccfg.setIndexedTypes(Integer.class, Integer.class);
 
         final IgniteCache<Integer, Integer> cache = ignite.createCache(ccfg);
 
@@ -218,24 +236,6 @@ public void testIndexingSpiFailure() throws Exception {
         }
     }
 
-    /**
-     * @return Configuration.
-     */
-    private IgniteConfiguration configuration() {
-        IgniteConfiguration cfg = new IgniteConfiguration();
-
-        TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
-        TcpDiscoverySpi disco = new TcpDiscoverySpi();
-
-        disco.setMaxMissedHeartbeats(Integer.MAX_VALUE);
-
-        disco.setIpFinder(ipFinder);
-
-        cfg.setDiscoverySpi(disco);
-
-        return cfg;
-    }
-
     /**
      * Indexing Spi implementation for test
      */
@@ -350,7 +350,7 @@ private static class MyBrokenIndexingSpi extends MyIndexingSpi {
     /**
      *
      */
-    private static class PersonKey implements Serializable, Comparable<PersonKey> {
+     static class PersonKey implements Serializable, Comparable<PersonKey> {
         /** */
         private int id;
 
@@ -385,7 +385,7 @@ public PersonKey(int id) {
     /**
      *
      */
-    private static class Person implements Serializable {
+    static class Person implements Serializable {
         /** */
         private String name;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryWithH2IndexingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryWithH2IndexingSelfTest.java
new file mode 100644
index 0000000000000..800c5a2e80701
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryWithH2IndexingSelfTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query;
+
+import org.apache.ignite.configuration.CacheConfiguration;
+
+/**
+ * Indexing Spi query with configured default indexer test
+ */
+public class IndexingSpiQueryWithH2IndexingSelfTest extends IndexingSpiQuerySelfTest {
+    /** */
+    protected <K, V> CacheConfiguration<K, V> cacheConfiguration(String cacheName) {
+        CacheConfiguration<K, V> ccfg = super.cacheConfiguration(cacheName);
+
+        ccfg.setIndexedTypes(PersonKey.class, Person.class);
+
+        ccfg.setIndexedTypes(Integer.class, Integer.class);
+
+        return ccfg;
+    }
+}
\ No newline at end of file

From d0c0bcece7d8e9d373aaf13a210f6d890e5ad48b Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 17 Jan 2017 16:19:02 +0300
Subject: [PATCH 106/446] IGNITE-3867: Fixed ScanQuery ignores pageSize
 property. This closes #1406.

---
 .../processors/cache/IgniteCacheProxy.java    |  3 +
 .../IgniteCachePartitionedQuerySelfTest.java  | 87 +++++++++++++++++++
 2 files changed, 90 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
index b9737c62dcbf3..873c8221ae921 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
@@ -491,6 +491,9 @@ private <T, R> QueryCursor<R> query(
 
         qry = ctx.queries().createScanQuery(p, transformer, scanQry.getPartition(), isKeepBinary);
 
+        if (scanQry.getPageSize() > 0)
+            qry.pageSize(scanQry.getPageSize());
+
         if (grp != null)
             qry.projection(grp);
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedQuerySelfTest.java
index 78fd914eeef20..b9f21dacdcc07 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedQuerySelfTest.java
@@ -20,15 +20,28 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractQuerySelfTest;
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryRequest;
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryResponse;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CachePeekMode.ALL;
@@ -47,6 +60,11 @@ public class IgniteCachePartitionedQuerySelfTest extends IgniteCacheAbstractQuer
         return PARTITIONED;
     }
 
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        return super.getConfiguration(gridName).setCommunicationSpi(new TestTcpCommunicationSpi());
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -135,4 +153,73 @@ private void checkResult(Iterable<Cache.Entry<UUID, Person>> entries, Person...
             assert F.asList(persons).contains(entry.getValue());
         }
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testScanQueryPagination() throws Exception {
+        final int pageSize = 5;
+
+        final AtomicInteger pages = new AtomicInteger(0);
+
+        IgniteCache<Integer, Integer> cache = ignite().cache(null);
+
+        for (int i = 0; i < 50; i++)
+            cache.put(i, i);
+
+        CommunicationSpi spi = ignite().configuration().getCommunicationSpi();
+
+        assert spi instanceof TestTcpCommunicationSpi;
+
+        TestTcpCommunicationSpi commSpi = (TestTcpCommunicationSpi)spi;
+
+        commSpi.filter = new IgniteInClosure<Message>() {
+            @Override public void apply(Message msg) {
+                if (!(msg instanceof GridIoMessage))
+                    return;
+
+                Message msg0 = ((GridIoMessage)msg).message();
+
+                if (msg0 instanceof GridCacheQueryRequest) {
+                    assertEquals(pageSize, ((GridCacheQueryRequest)msg0).pageSize());
+
+                    pages.incrementAndGet();
+                }
+                else if (msg0 instanceof GridCacheQueryResponse) {
+                    assertTrue(((GridCacheQueryResponse)msg0).data().size() <= pageSize);
+                }
+            }
+        };
+
+        try {
+            ScanQuery<Integer, Integer> qry = new ScanQuery<Integer, Integer>();
+
+            qry.setPageSize(pageSize);
+
+            List<Cache.Entry<Integer, Integer>> all = cache.query(qry).getAll();
+
+            assertTrue(pages.get() > ignite().cluster().forDataNodes(null).nodes().size());
+
+            assertEquals(50, all.size());
+        }
+        finally {
+            commSpi.filter = null;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestTcpCommunicationSpi extends TcpCommunicationSpi {
+        volatile IgniteInClosure<Message> filter;
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackClosure)
+            throws IgniteSpiException {
+            if(filter != null)
+                filter.apply(msg);
+
+            super.sendMessage(node, msg, ackClosure);
+        }
+    }
 }
\ No newline at end of file

From b54a481315a45c7a6c8f70534f655e14b25cc439 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 18 Jan 2017 12:05:22 +0300
Subject: [PATCH 107/446] ignite-4525 - Near reader is created when value is
 loaded from store.

---
 .../processors/cache/GridCacheAdapter.java    |  33 +-
 .../processors/cache/GridCacheContext.java    |   4 +-
 .../processors/cache/GridCacheEntryEx.java    |  20 +-
 .../processors/cache/GridCacheMapEntry.java   |  56 ++-
 .../processors/cache/ReaderArguments.java     |  74 +++
 .../distributed/dht/GridDhtCacheAdapter.java  |   9 +-
 .../distributed/dht/GridDhtGetFuture.java     |  85 ++--
 .../dht/GridDhtGetSingleFuture.java           |  75 ++-
 .../dht/GridPartitionedGetFuture.java         |   3 +-
 .../dht/GridPartitionedSingleGetFuture.java   |   3 +-
 .../dht/atomic/GridDhtAtomicCache.java        |   5 +-
 .../dht/colocated/GridDhtColocatedCache.java  |   3 +-
 .../distributed/near/GridNearGetFuture.java   |   6 +-
 .../local/atomic/GridLocalAtomicCache.java    |   5 +-
 .../transactions/IgniteTxLocalAdapter.java    |  35 +-
 .../cache/GridCacheTestEntryEx.java           |  11 +-
 .../near/GridNearCacheStoreUpdateTest.java    | 466 ++++++++++++++++++
 .../GridNearOffheapCacheStoreUpdateTest.java  |  35 ++
 .../testsuites/IgniteCacheTestSuite2.java     |   5 +
 19 files changed, 770 insertions(+), 163 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ReaderArguments.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheStoreUpdateTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOffheapCacheStoreUpdateTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index fd9f396926f0e..59665bb58708f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1789,6 +1789,7 @@ protected IgniteInternalFuture<Map<K, V>> getAllAsync(
         subjId = ctx.subjectIdPerCall(subjId, opCtx);
 
         return getAllAsync(keys,
+            null,
             opCtx == null || !opCtx.skipStore(),
             !skipTx,
             subjId,
@@ -1803,6 +1804,7 @@ protected IgniteInternalFuture<Map<K, V>> getAllAsync(
 
     /**
      * @param keys Keys.
+     * @param readerArgs Near cache reader will be added if not null.
      * @param readThrough Read through.
      * @param checkTx Check tx.
      * @param subjId Subj Id.
@@ -1817,6 +1819,7 @@ protected IgniteInternalFuture<Map<K, V>> getAllAsync(
      * @see GridCacheAdapter#getAllAsync(Collection)
      */
     public final IgniteInternalFuture<Map<K, V>> getAllAsync(@Nullable final Collection<? extends K> keys,
+        @Nullable final ReaderArguments readerArgs,
         boolean readThrough,
         boolean checkTx,
         @Nullable final UUID subjId,
@@ -1834,6 +1837,7 @@ public final IgniteInternalFuture<Map<K, V>> getAllAsync(@Nullable final Collect
             validateCacheKeys(keys);
 
         return getAllAsync0(ctx.cacheKeysView(keys),
+            readerArgs,
             readThrough,
             checkTx,
             subjId,
@@ -1848,6 +1852,7 @@ public final IgniteInternalFuture<Map<K, V>> getAllAsync(@Nullable final Collect
 
     /**
      * @param keys Keys.
+     * @param readerArgs Near cache reader will be added if not null.
      * @param readThrough Read-through flag.
      * @param checkTx Check local transaction flag.
      * @param subjId Subject ID.
@@ -1862,6 +1867,7 @@ public final IgniteInternalFuture<Map<K, V>> getAllAsync(@Nullable final Collect
      */
     protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
         @Nullable final Collection<KeyCacheObject> keys,
+        @Nullable final ReaderArguments readerArgs,
         final boolean readThrough,
         boolean checkTx,
         @Nullable final UUID subjId,
@@ -1932,7 +1938,8 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                     subjId,
                                     taskName,
                                     expiry,
-                                    !deserializeBinary);
+                                    !deserializeBinary,
+                                    readerArgs);
 
                                 assert res != null;
 
@@ -1957,7 +1964,8 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                     null,
                                     taskName,
                                     expiry,
-                                    !deserializeBinary);
+                                    !deserializeBinary,
+                                    readerArgs);
 
                                 if (res == null)
                                     ctx.evicts().touch(entry, topVer);
@@ -2015,29 +2023,28 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                             GridCacheEntryEx entry = entryEx(key);
 
                                             try {
-                                                GridCacheVersion verSet = entry.versionedValue(cacheVal,
+                                                T2<CacheObject, GridCacheVersion> verVal = entry.versionedValue(
+                                                    cacheVal,
                                                     res.version(),
-                                                    null);
-
-                                                boolean set = verSet != null;
+                                                    null,
+                                                    readerArgs);
 
                                                 if (log.isDebugEnabled())
                                                     log.debug("Set value loaded from store into entry [" +
-                                                        "set=" + set +
-                                                        ", curVer=" + res.version() +
-                                                        ", newVer=" + verSet + ", " +
+                                                        "oldVer=" + res.version() +
+                                                        ", newVer=" + verVal.get2() + ", " +
                                                         "entry=" + entry + ']');
 
                                                 // Don't put key-value pair into result map if value is null.
-                                                if (val != null) {
+                                                if (verVal.get1() != null) {
                                                     ctx.addResult(map,
                                                         key,
-                                                        cacheVal,
+                                                        verVal.get1(),
                                                         skipVals,
                                                         keepCacheObjects,
                                                         deserializeBinary,
-                                                        false,
-                                                        needVer ? set ? verSet : res.version() : null);
+                                                        true,
+                                                        needVer ? verVal.get2() : null);
                                                 }
 
                                                 if (tx0 == null || (!tx0.implicit() &&
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 66b71b4666e7b..424e325dd07f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -1900,9 +1900,9 @@ public <K1, V1> void addResult(Map<K1, V1> map,
         assert val != null || skipVals;
 
         if (!keepCacheObjects) {
-            Object key0 = unwrapBinaryIfNeeded(key, !deserializeBinary);
+            Object key0 = unwrapBinaryIfNeeded(key, !deserializeBinary, cpy);
 
-            Object val0 = skipVals ? true : unwrapBinaryIfNeeded(val, !deserializeBinary);
+            Object val0 = skipVals ? true : unwrapBinaryIfNeeded(val, !deserializeBinary, cpy);
 
             assert key0 != null : key;
             assert val0 != null : val;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index b1d632fd5943d..51f423a7302a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
 import org.apache.ignite.internal.processors.dr.GridDrType;
 import org.apache.ignite.internal.util.lang.GridTuple3;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -318,11 +319,12 @@ public boolean evictInternal(boolean swap, GridCacheVersion obsoleteVer,
      * @param taskName Task name.
      * @param expiryPlc Expiry policy.
      * @param keepBinary Keep binary flag.
+     * @param readerArgs Reader will be added if not null.
      * @return Cached value and entry version.
      * @throws IgniteCheckedException If loading value failed.
      * @throws GridCacheEntryRemovedException If entry was removed.
      */
-    @Nullable public EntryGetResult innerGetVersioned(
+    public EntryGetResult innerGetVersioned(
         @Nullable GridCacheVersion ver,
         IgniteInternalTx tx,
         boolean readSwap,
@@ -333,7 +335,8 @@ public boolean evictInternal(boolean swap, GridCacheVersion obsoleteVer,
         Object transformClo,
         String taskName,
         @Nullable IgniteCacheExpiryPolicy expiryPlc,
-        boolean keepBinary)
+        boolean keepBinary,
+        @Nullable ReaderArguments readerArgs)
         throws IgniteCheckedException, GridCacheEntryRemovedException;
 
     /**
@@ -344,7 +347,7 @@ public boolean evictInternal(boolean swap, GridCacheVersion obsoleteVer,
      * @param taskName Task name.
      * @param expiryPlc Expiry policy.
      * @param keepBinary Keep binary flag.
-     * @return Cached value and entry version.
+     * @param readerArgs Reader will be added if not null.
      * @throws IgniteCheckedException If loading value failed.
      * @throws GridCacheEntryRemovedException If entry was removed.
      * @return Cached value, entry version and flag indicating if entry was reserved.
@@ -355,7 +358,8 @@ public EntryGetResult innerGetAndReserveForLoad(boolean readSwap,
         UUID subjId,
         String taskName,
         @Nullable IgniteCacheExpiryPolicy expiryPlc,
-        boolean keepBinary) throws IgniteCheckedException, GridCacheEntryRemovedException;
+        boolean keepBinary,
+        @Nullable ReaderArguments readerArgs) throws IgniteCheckedException, GridCacheEntryRemovedException;
 
     /**
      * @param ver Expected entry version.
@@ -751,13 +755,15 @@ public <K, V> GridCacheVersionedEntryEx<K, V> versionedEntry(final boolean keepB
      * @param val New value.
      * @param curVer Version to match or {@code null} if match is not required.
      * @param newVer Version to set.
-     * @return Non null version if value was set.
+     * @param readerArgs Reader will be added if not null.
+     * @return Current version and value.
      * @throws IgniteCheckedException If index could not be updated.
      * @throws GridCacheEntryRemovedException If entry was removed.
      */
-    public GridCacheVersion versionedValue(CacheObject val,
+    public T2<CacheObject, GridCacheVersion> versionedValue(CacheObject val,
         @Nullable GridCacheVersion curVer,
-        @Nullable GridCacheVersion newVer)
+        @Nullable GridCacheVersion newVer,
+        @Nullable ReaderArguments readerArgs)
         throws IgniteCheckedException, GridCacheEntryRemovedException;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 9f0c2b0a47c56..59e4181df0cee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -778,7 +778,8 @@ protected final void releaseSwap() throws IgniteCheckedException {
             expirePlc,
             false,
             keepBinary,
-            false);
+            false,
+            null);
     }
 
     /** {@inheritDoc} */
@@ -788,7 +789,8 @@ protected final void releaseSwap() throws IgniteCheckedException {
         UUID subjId,
         String taskName,
         @Nullable IgniteCacheExpiryPolicy expiryPlc,
-        boolean keepBinary) throws IgniteCheckedException, GridCacheEntryRemovedException {
+        boolean keepBinary,
+        @Nullable ReaderArguments readerArgs) throws IgniteCheckedException, GridCacheEntryRemovedException {
         return (EntryGetResult)innerGet0(
             /*ver*/null,
             /*tx*/null,
@@ -803,11 +805,12 @@ protected final void releaseSwap() throws IgniteCheckedException {
             expiryPlc,
             true,
             keepBinary,
-            /*reserve*/true);
+            /*reserve*/true,
+            readerArgs);
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public EntryGetResult innerGetVersioned(
+    @Override public EntryGetResult innerGetVersioned(
         @Nullable GridCacheVersion ver,
         IgniteInternalTx tx,
         boolean readSwap,
@@ -818,7 +821,8 @@ protected final void releaseSwap() throws IgniteCheckedException {
         Object transformClo,
         String taskName,
         @Nullable IgniteCacheExpiryPolicy expiryPlc,
-        boolean keepBinary)
+        boolean keepBinary,
+        @Nullable ReaderArguments readerArgs)
         throws IgniteCheckedException, GridCacheEntryRemovedException {
         return (EntryGetResult)innerGet0(ver,
             tx,
@@ -833,7 +837,8 @@ protected final void releaseSwap() throws IgniteCheckedException {
             expiryPlc,
             true,
             keepBinary,
-            false);
+            false,
+            readerArgs);
     }
 
     /** {@inheritDoc} */
@@ -852,7 +857,8 @@ private Object innerGet0(
         @Nullable IgniteCacheExpiryPolicy expiryPlc,
         boolean retVer,
         boolean keepBinary,
-        boolean reserveForLoad
+        boolean reserveForLoad,
+        @Nullable ReaderArguments readerArgs
     ) throws IgniteCheckedException, GridCacheEntryRemovedException {
         assert !(retVer && readThrough);
         assert !(reserveForLoad && readThrough);
@@ -961,6 +967,8 @@ private Object innerGet0(
             // Cache version for optimistic check.
             startVer = ver;
 
+            addReaderIfNeed(readerArgs);
+
             if (ret != null) {
                 assert tmp || !(ret instanceof BinaryObjectOffheapImpl);
                 assert !obsolete;
@@ -1051,6 +1059,8 @@ else if (tx.dht()) {
 
                     if (cctx.deferredDelete() && deletedUnlocked() && !isInternal() && !detached())
                         deletedUnlocked(false);
+
+                    assert readerArgs == null;
                 }
 
                 if (evt && cctx.events().isRecordable(EVT_CACHE_OBJECT_READ)) {
@@ -3611,19 +3621,22 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
     }
 
     /** {@inheritDoc} */
-    @Override public synchronized GridCacheVersion versionedValue(CacheObject val,
+    @Override public synchronized T2<CacheObject, GridCacheVersion> versionedValue(CacheObject val,
         GridCacheVersion curVer,
-        GridCacheVersion newVer)
+        GridCacheVersion newVer,
+        @Nullable ReaderArguments readerArgs)
         throws IgniteCheckedException, GridCacheEntryRemovedException
     {
         checkObsolete();
 
+        addReaderIfNeed(readerArgs);
+
         if (curVer == null || curVer.equals(ver)) {
             if (val != this.val) {
                 GridCacheMvcc mvcc = mvccExtras();
 
                 if (mvcc != null && !mvcc.isEmpty())
-                    return null;
+                    return new T2<>(this.val, ver);
 
                 if (newVer == null)
                     newVer = cctx.versions().next();
@@ -3647,13 +3660,32 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
                 // Version does not change for load ops.
                 update(val, expTime, ttl, newVer, true);
 
-                return newVer;
+                return new T2<>(val, newVer);
             }
 
             assert !evictionDisabled() : this;
         }
 
-        return null;
+        return new T2<>(this.val, ver);
+    }
+
+    /**
+     * @param readerArgs Reader arguments
+     */
+    private void addReaderIfNeed(@Nullable ReaderArguments readerArgs) {
+        if (readerArgs != null) {
+            assert this instanceof GridDhtCacheEntry : this;
+            assert Thread.holdsLock(this);
+
+            try {
+                ((GridDhtCacheEntry)this).addReader(readerArgs.reader(),
+                    readerArgs.messageId(),
+                    readerArgs.topologyVersion());
+            }
+            catch (GridCacheEntryRemovedException e) {
+                assert false : this;
+            }
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ReaderArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ReaderArguments.java
new file mode 100644
index 0000000000000..b8b5e64894fda
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ReaderArguments.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.UUID;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Arguments required for adding near cache reader to entry.
+ */
+public class ReaderArguments {
+    /** */
+    private final UUID reader;
+
+    /** */
+    private final long msgId;
+
+    /** */
+    private final AffinityTopologyVersion topVer;
+
+    /**
+     * @param reader Near cache node ID.
+     * @param msgId Message ID.
+     * @param topVer Topology version.
+     */
+    public ReaderArguments(final UUID reader, final long msgId,
+        final AffinityTopologyVersion topVer) {
+        this.reader = reader;
+        this.msgId = msgId;
+        this.topVer = topVer;
+    }
+
+    /**
+     * @return Reader node ID.
+     */
+    public UUID reader() {
+        return reader;
+    }
+
+    /**
+     * @return Message ID.
+     */
+    public long messageId() {
+        return msgId;
+    }
+
+    /**
+     * @return Topology version.
+     */
+    public AffinityTopologyVersion topologyVersion() {
+        return topVer;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ReaderArguments.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index b2fb7b4b392a4..543cee1809070 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -35,8 +35,8 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.NodeStoppingException;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheOperationContext;
@@ -53,6 +53,7 @@
 import org.apache.ignite.internal.processors.cache.GridCachePreloader;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.ReaderArguments;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTtlUpdateRequest;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
@@ -623,6 +624,7 @@ else if (log.isDebugEnabled())
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         return getAllAsync(keys,
+            null,
             opCtx == null || !opCtx.skipStore(),
             /*don't check local tx. */false,
             subjId,
@@ -637,6 +639,7 @@ else if (log.isDebugEnabled())
 
     /**
      * @param keys Keys to get
+     * @param readerArgs Reader will be added if not null.
      * @param readThrough Read through flag.
      * @param subjId Subject ID.
      * @param taskName Task name.
@@ -647,6 +650,7 @@ else if (log.isDebugEnabled())
      */
     IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> getDhtAllAsync(
         Collection<KeyCacheObject> keys,
+        @Nullable final ReaderArguments readerArgs,
         boolean readThrough,
         @Nullable UUID subjId,
         String taskName,
@@ -655,6 +659,7 @@ IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> get
         boolean canRemap
     ) {
         return getAllAsync0(keys,
+            readerArgs,
             readThrough,
             /*don't check local tx. */false,
             subjId,
@@ -694,7 +699,6 @@ public GridDhtFuture<Collection<GridCacheEntryInfo>> getDhtAsync(UUID reader,
             reader,
             keys,
             readThrough,
-            /*tx*/null,
             topVer,
             subjId,
             taskNameHash,
@@ -738,7 +742,6 @@ private IgniteInternalFuture<GridCacheEntryInfo> getDhtSingleAsync(
             key,
             addRdr,
             readThrough,
-            /*tx*/null,
             topVer,
             subjId,
             taskNameHash,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
index 913580f8c02ad..3bf44895a62a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
@@ -29,7 +29,6 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -37,7 +36,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
+import org.apache.ignite.internal.processors.cache.ReaderArguments;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridCompoundIdentityFuture;
@@ -50,7 +49,6 @@
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiClosure;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -96,9 +94,6 @@ public final class GridDhtGetFuture<K, V> extends GridCompoundIdentityFuture<Col
     /** Topology version .*/
     private AffinityTopologyVersion topVer;
 
-    /** Transaction. */
-    private IgniteTxLocalEx tx;
-
     /** Retries because ownership changed. */
     private Collection<Integer> retries;
 
@@ -120,7 +115,6 @@ public final class GridDhtGetFuture<K, V> extends GridCompoundIdentityFuture<Col
      * @param reader Reader.
      * @param keys Keys.
      * @param readThrough Read through flag.
-     * @param tx Transaction.
      * @param topVer Topology version.
      * @param subjId Subject ID.
      * @param taskNameHash Task name hash code.
@@ -133,7 +127,6 @@ public GridDhtGetFuture(
         UUID reader,
         Map<KeyCacheObject, Boolean> keys,
         boolean readThrough,
-        @Nullable IgniteTxLocalEx tx,
         @NotNull AffinityTopologyVersion topVer,
         @Nullable UUID subjId,
         int taskNameHash,
@@ -150,7 +143,6 @@ public GridDhtGetFuture(
         this.msgId = msgId;
         this.keys = keys;
         this.readThrough = readThrough;
-        this.tx = tx;
         this.topVer = topVer;
         this.subjId = subjId;
         this.taskNameHash = taskNameHash;
@@ -159,7 +151,7 @@ public GridDhtGetFuture(
 
         futId = IgniteUuid.randomUuid();
 
-        ver = tx == null ? cctx.versions().next() : tx.xidVersion();
+        ver = cctx.versions().next();
 
         if (log == null)
             log = U.logger(cctx.kernalContext(), logRef, GridDhtGetFuture.class);
@@ -340,6 +332,8 @@ private IgniteInternalFuture<Collection<GridCacheEntryInfo>> getAsync(
 
         ClusterNode readerNode = cctx.discovery().node(reader);
 
+        ReaderArguments readerArgs = null;
+
         if (readerNode != null && !readerNode.isLocal() && cctx.discovery().cacheNearNode(readerNode, cctx.name())) {
             for (Map.Entry<KeyCacheObject, Boolean> k : keys.entrySet()) {
                 while (true) {
@@ -351,12 +345,19 @@ private IgniteInternalFuture<Collection<GridCacheEntryInfo>> getAsync(
 
                         boolean addReader = (!e.deleted() && k.getValue() && !skipVals);
 
-                        if (addReader)
+                        if (addReader) {
                             e.unswap(false);
 
+                            // Entry will be removed on touch() if no data in cache,
+                            // but they could be loaded from store,
+                            // we have to add reader again later.
+                            if (readerArgs == null)
+                                readerArgs = new ReaderArguments(reader, msgId, topVer);
+                        }
+
                         // Register reader. If there are active transactions for this entry,
                         // then will wait for their completion before proceeding.
-                        // TODO: GG-4003:
+                        // TODO: IGNITE-3498:
                         // TODO: What if any transaction we wait for actually removes this entry?
                         // TODO: In this case seems like we will be stuck with untracked near entry.
                         // TODO: To fix, check that reader is contained in the list of readers once
@@ -392,28 +393,19 @@ private IgniteInternalFuture<Collection<GridCacheEntryInfo>> getAsync(
         IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> fut;
 
         if (txFut == null || txFut.isDone()) {
-            if (tx == null) {
-                fut = cache().getDhtAllAsync(
-                    keys.keySet(),
-                    readThrough,
-                    subjId,
-                    taskName,
-                    expiryPlc,
-                    skipVals,
-                    /*can remap*/true);
-            }
-            else {
-                fut = tx.getAllAsync(cctx,
-                    null,
-                    keys.keySet(),
-                    /*deserialize binary*/false,
-                    skipVals,
-                    /*keep cache objects*/true,
-                    /*skip store*/!readThrough,
-                    false);
-            }
+            fut = cache().getDhtAllAsync(
+                keys.keySet(),
+                readerArgs,
+                readThrough,
+                subjId,
+                taskName,
+                expiryPlc,
+                skipVals,
+                /*can remap*/true);
         }
         else {
+            final ReaderArguments args = readerArgs;
+
             // If we are here, then there were active transactions for some entries
             // when we were adding the reader. In that case we must wait for those
             // transactions to complete.
@@ -424,26 +416,15 @@ private IgniteInternalFuture<Collection<GridCacheEntryInfo>> getAsync(
                         if (e != null)
                             throw new GridClosureException(e);
 
-                        if (tx == null) {
-                            return cache().getDhtAllAsync(
-                                keys.keySet(),
-                                readThrough,
-                                subjId,
-                                taskName,
-                                expiryPlc,
-                                skipVals,
-                                /*can remap*/true);
-                        }
-                        else {
-                            return tx.getAllAsync(cctx,
-                                null,
-                                keys.keySet(),
-                                /*deserialize binary*/false,
-                                skipVals,
-                                /*keep cache objects*/true,
-                                /*skip store*/!readThrough,
-                                false);
-                        }
+                        return cache().getDhtAllAsync(
+                            keys.keySet(),
+                            args,
+                            readThrough,
+                            subjId,
+                            taskName,
+                            expiryPlc,
+                            skipVals,
+                            /*can remap*/true);
                     }
                 }
             );
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
index 93949377e12db..49bebd6a74b45 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
@@ -35,7 +35,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
+import org.apache.ignite.internal.processors.cache.ReaderArguments;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.F;
@@ -90,9 +90,6 @@ public final class GridDhtGetSingleFuture<K, V> extends GridFutureAdapter<GridCa
     /** Topology version .*/
     private AffinityTopologyVersion topVer;
 
-    /** Transaction. */
-    private IgniteTxLocalEx tx;
-
     /** Retries because ownership changed. */
     private Collection<Integer> retries;
 
@@ -115,7 +112,6 @@ public final class GridDhtGetSingleFuture<K, V> extends GridFutureAdapter<GridCa
      * @param key Key.
      * @param addRdr Add reader flag.
      * @param readThrough Read through flag.
-     * @param tx Transaction.
      * @param topVer Topology version.
      * @param subjId Subject ID.
      * @param taskNameHash Task name hash code.
@@ -129,7 +125,6 @@ public GridDhtGetSingleFuture(
         KeyCacheObject key,
         Boolean addRdr,
         boolean readThrough,
-        @Nullable IgniteTxLocalEx tx,
         @NotNull AffinityTopologyVersion topVer,
         @Nullable UUID subjId,
         int taskNameHash,
@@ -145,7 +140,6 @@ public GridDhtGetSingleFuture(
         this.key = key;
         this.addRdr = addRdr;
         this.readThrough = readThrough;
-        this.tx = tx;
         this.topVer = topVer;
         this.subjId = subjId;
         this.taskNameHash = taskNameHash;
@@ -154,7 +148,7 @@ public GridDhtGetSingleFuture(
 
         futId = IgniteUuid.randomUuid();
 
-        ver = tx == null ? cctx.versions().next() : tx.xidVersion();
+        ver = cctx.versions().next();
 
         if (log == null)
             log = U.logger(cctx.kernalContext(), logRef, GridDhtGetSingleFuture.class);
@@ -306,6 +300,8 @@ private void getAsync() {
 
         ClusterNode readerNode = cctx.discovery().node(reader);
 
+        ReaderArguments readerArgs = null;
+
         if (readerNode != null && !readerNode.isLocal() && cctx.discovery().cacheNearNode(readerNode, cctx.name())) {
             while (true) {
                 GridDhtCacheEntry e = cache().entryExx(key, topVer);
@@ -314,14 +310,21 @@ private void getAsync() {
                     if (e.obsolete())
                         continue;
 
-                    boolean addReader = (!e.deleted() && addRdr && !skipVals);
+                    boolean addReader = (!e.deleted() && this.addRdr && !skipVals);
 
-                    if (addReader)
+                    if (addReader) {
                         e.unswap(false);
 
+                        // Entry will be removed on touch() if no data in cache,
+                        // but they could be loaded from store,
+                        // we have to add reader again later.
+                        if (readerArgs == null)
+                            readerArgs = new ReaderArguments(reader, msgId, topVer);
+                    }
+
                     // Register reader. If there are active transactions for this entry,
                     // then will wait for their completion before proceeding.
-                    // TODO: GG-4003:
+                    // TODO: IGNITE-3498:
                     // TODO: What if any transaction we wait for actually removes this entry?
                     // TODO: In this case seems like we will be stuck with untracked near entry.
                     // TODO: To fix, check that reader is contained in the list of readers once
@@ -348,28 +351,19 @@ private void getAsync() {
         IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> fut;
 
         if (rdrFut == null || rdrFut.isDone()) {
-            if (tx == null) {
-                fut = cache().getDhtAllAsync(
-                    Collections.singleton(key),
-                    readThrough,
-                    subjId,
-                    taskName,
-                    expiryPlc,
-                    skipVals,
-                    /*can remap*/true);
-            }
-            else {
-                fut = tx.getAllAsync(cctx,
-                    null,
-                    Collections.singleton(key),
-                    /*deserialize binary*/false,
-                    skipVals,
-                    /*keep cache objects*/true,
-                    /*skip store*/!readThrough,
-                    false);
-            }
+            fut = cache().getDhtAllAsync(
+                Collections.singleton(key),
+                readerArgs,
+                readThrough,
+                subjId,
+                taskName,
+                expiryPlc,
+                skipVals,
+                /*can remap*/true);
         }
         else {
+            final ReaderArguments args = readerArgs;
+
             rdrFut.listen(
                 new IgniteInClosure<IgniteInternalFuture<Boolean>>() {
                     @Override public void apply(IgniteInternalFuture<Boolean> fut) {
@@ -381,29 +375,16 @@ private void getAsync() {
                             return;
                         }
 
-                        IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> fut0;
-
-                        if (tx == null) {
-                            fut0 = cache().getDhtAllAsync(
+                        IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> fut0 =
+                            cache().getDhtAllAsync(
                                 Collections.singleton(key),
+                                args,
                                 readThrough,
                                 subjId,
                                 taskName,
                                 expiryPlc,
                                 skipVals,
                                 /*can remap*/true);
-                        }
-                        else {
-                            fut0 = tx.getAllAsync(cctx,
-                                null,
-                                Collections.singleton(key),
-                                /*deserialize binary*/false,
-                                skipVals,
-                                /*keep cache objects*/true,
-                                /*skip store*/!readThrough,
-                                false
-                            );
-                        }
 
                         fut0.listen(createGetFutureListener());
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index d0f209df94ca0..c8e2cf328bcd3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -461,7 +461,8 @@ private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
                             null,
                             taskName,
                             expiryPlc,
-                            !deserializeBinary);
+                            !deserializeBinary,
+                            null);
 
                         if (res != null) {
                             v = res.value();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index e188a3523a3a5..e369bfa83a8be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -385,7 +385,8 @@ private boolean localGet(AffinityTopologyVersion topVer, int part) {
                             null,
                             taskName,
                             expiryPlc,
-                            true);
+                            true,
+                            null);
 
                         if (res != null) {
                             v = res.value();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 94a049ea86c6e..f601e0a9ddb7e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1505,7 +1505,8 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
                                     null,
                                     taskName,
                                     expiry,
-                                    true);
+                                    true,
+                                    null);
 
                                 if (res != null) {
                                     v = res.value();
@@ -2292,7 +2293,7 @@ private void reloadIfNeeded(final List<GridDhtCacheEntry> entries) throws Ignite
                         try {
                             GridCacheVersion ver = entry.version();
 
-                            entry.versionedValue(ctx.toCacheObject(v), null, ver);
+                            entry.versionedValue(ctx.toCacheObject(v), null, ver, null);
                         }
                         catch (GridCacheEntryRemovedException e) {
                             assert false : "Entry should not get obsolete while holding lock [entry=" + entry +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 56af95e6b65c1..29f0607e5c33c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -496,7 +496,8 @@ public final IgniteInternalFuture<Map<K, V>> loadAsync(
                                     null,
                                     taskName,
                                     expiryPlc,
-                                    !deserializeBinary);
+                                    !deserializeBinary,
+                                    null);
 
                                 if (res != null) {
                                     v = res.value();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index ab0e88cbf375a..8bc513e4dfda2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -449,7 +449,8 @@ private Map<KeyCacheObject, GridNearCacheEntry> map(
                             null,
                             taskName,
                             expiryPlc,
-                            !deserializeBinary);
+                            !deserializeBinary,
+                            null);
 
                         if (res != null) {
                             v = res.value();
@@ -589,7 +590,8 @@ private boolean localDhtGet(KeyCacheObject key,
                             null,
                             taskName,
                             expiryPlc,
-                            !deserializeBinary);
+                            !deserializeBinary,
+                            null);
 
                         if (res != null) {
                             v = res.value();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index d1acada062cb7..ad818a63d8c21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -59,7 +59,6 @@
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.resource.GridResourceIoc;
-import org.apache.ignite.internal.processors.resource.GridResourceProcessor;
 import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
@@ -528,7 +527,8 @@ private Map<K, V> getAllInternal(@Nullable Collection<? extends K> keys,
                                 null,
                                 taskName,
                                 expiry,
-                                !deserializeBinary);
+                                !deserializeBinary,
+                                null);
 
                             if (res != null) {
                                 v = res.value();
@@ -602,6 +602,7 @@ private Map<K, V> getAllInternal(@Nullable Collection<? extends K> keys,
 
         return getAllAsync(
             keys,
+            null,
             opCtx == null || !opCtx.skipStore(),
             false,
             subjId,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 91c9c92075eab..f05d90df6cf10 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -438,7 +438,8 @@ protected boolean commitAfterLock() {
                             null,
                             resolveTaskName(),
                             expiryPlc,
-                            txEntry == null ? keepBinary : txEntry.keepBinary());
+                            txEntry == null ? keepBinary : txEntry.keepBinary(),
+                            null);
 
                         if (res == null) {
                             if (misses == null)
@@ -477,17 +478,19 @@ protected boolean commitAfterLock() {
                                 GridCacheEntryEx entry = cacheCtx.cache().entryEx(key);
 
                                 try {
-                                    GridCacheVersion setVer = entry.versionedValue(cacheVal, ver, null);
-
-                                    boolean set = setVer != null;
+                                    T2<CacheObject, GridCacheVersion> verVal = entry.versionedValue(cacheVal,
+                                        ver,
+                                        null,
+                                        null);
 
-                                    if (set)
-                                        ver = setVer;
+                                    if (log.isDebugEnabled()) {
+                                        log.debug("Set value loaded from store into entry [" +
+                                            "oldVer=" + ver +
+                                            ", newVer=" + verVal.get2() +
+                                            ", entry=" + entry + ']');
+                                    }
 
-                                    if (log.isDebugEnabled())
-                                        log.debug("Set value loaded from store into entry [set=" + set +
-                                            ", curVer=" + ver + ", newVer=" + setVer + ", " +
-                                            "entry=" + entry + ']');
+                                    ver = verVal.get2();
 
                                     break;
                                 }
@@ -1232,7 +1235,8 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                     transformClo,
                                     resolveTaskName(),
                                     null,
-                                    txEntry.keepBinary());
+                                    txEntry.keepBinary(),
+                                    null);
 
                                 if (res != null) {
                                     val = res.value();
@@ -1316,7 +1320,8 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                         null,
                                         resolveTaskName(),
                                         accessPlc,
-                                        !deserializeBinary) : null;
+                                        !deserializeBinary,
+                                        null) : null;
 
                                 if (res != null) {
                                     val = res.value();
@@ -1666,7 +1671,8 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                                             transformClo,
                                             resolveTaskName(),
                                             null,
-                                            txEntry.keepBinary());
+                                            txEntry.keepBinary(),
+                                            null);
 
                                         if (res != null) {
                                             val = res.value();
@@ -2390,7 +2396,8 @@ private boolean enlistWriteEntry(GridCacheContext cacheCtx,
                                         entryProcessor,
                                         resolveTaskName(),
                                         null,
-                                        keepBinary) : null;
+                                        keepBinary,
+                                        null) : null;
 
                                 if (res != null) {
                                     old = res.value();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index b03e9c89c5b3b..8db68b4ef50fb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -430,7 +430,8 @@ void recheckLock() {
         UUID subjId,
         String taskName,
         @Nullable IgniteCacheExpiryPolicy expiryPlc,
-        boolean keepBinary) throws IgniteCheckedException, GridCacheEntryRemovedException {
+        boolean keepBinary,
+        @Nullable ReaderArguments args) throws IgniteCheckedException, GridCacheEntryRemovedException {
         assert false;
 
         return null;
@@ -448,7 +449,8 @@ void recheckLock() {
         Object transformClo,
         String taskName,
         @Nullable IgniteCacheExpiryPolicy expiryPlc,
-        boolean keepBinary) {
+        boolean keepBinary,
+        @Nullable ReaderArguments readerArgs) {
         assert false;
 
         return null;
@@ -684,9 +686,10 @@ void recheckLock() {
     }
 
     /** @inheritDoc */
-    @Override public GridCacheVersion versionedValue(CacheObject val,
+    @Override public T2<CacheObject, GridCacheVersion> versionedValue(CacheObject val,
         GridCacheVersion curVer,
-        GridCacheVersion newVer) {
+        GridCacheVersion newVer,
+        @Nullable ReaderArguments readerArgs) {
         assert false;
 
         return null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheStoreUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheStoreUpdateTest.java
new file mode 100644
index 0000000000000..183b9caab4d74
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheStoreUpdateTest.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+/**
+ * Check that near cache is updated when entry loaded from store.
+ */
+public class GridNearCacheStoreUpdateTest extends GridCommonAbstractTest {
+    /** */
+    private static final String CACHE_NAME = "cache";
+
+    /** */
+    private Ignite srv;
+
+    /** */
+    private Ignite client;
+
+    /** */
+    private IgniteCache<String, String> cache;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("client"))
+            cfg.setClientMode(true);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        srv = startGrid("server");
+        client = startGrid("client");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If fail.
+     */
+    public void testAtomicUpdateNear() throws Exception {
+        cache = client.createCache(cacheConfiguration(), new NearCacheConfiguration<String, String>());
+
+        checkNear(null, null);
+    }
+
+    /**
+     * @throws Exception If fail.
+     */
+    public void testTransactionAtomicUpdateNear() throws Exception {
+        cache = client.createCache(cacheConfiguration(), new NearCacheConfiguration<String, String>());
+
+        checkNear(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ);
+    }
+
+    /**
+     * @throws Exception If fail.
+     */
+    public void testPessimisticRepeatableReadUpdateNear() throws Exception {
+        cache = client.createCache(cacheConfiguration().setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL),
+            new NearCacheConfiguration<String, String>());
+
+        checkNear(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ);
+    }
+
+    /**
+     * @throws Exception If fail.
+     */
+    public void testPessimisticReadCommittedUpdateNear() throws Exception {
+        cache = client.createCache(cacheConfiguration().setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL),
+            new NearCacheConfiguration<String, String>());
+
+        checkNear(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.READ_COMMITTED);
+    }
+
+    /**
+     * @throws Exception If fail.
+     */
+    public void testOptimisticSerializableUpdateNear() throws Exception {
+        cache = client.createCache(cacheConfiguration().setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL),
+            new NearCacheConfiguration<String, String>());
+
+        checkNear(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.SERIALIZABLE);
+    }
+
+    /**
+     * @param txConc Transaction concurrency.
+     * @param txIsolation Transaction isolation.
+     * @throws Exception If fail.
+     */
+    private void checkNear(TransactionConcurrency txConc, TransactionIsolation txIsolation) throws Exception {
+        checkNearSingle(txConc, txIsolation);
+        checkNearSingleConcurrent(txConc, txIsolation);
+        checkNearBatch(txConc, txIsolation);
+        checkNearBatchConcurrent(txConc, txIsolation);
+    }
+
+    /**
+     * @param txConc Transaction concurrency.
+     * @param txIsolation Transaction isolation.
+     * @throws Exception If fail.
+     */
+    private void checkNearSingle(TransactionConcurrency txConc, TransactionIsolation txIsolation) throws Exception {
+        final String key = "key";
+
+        boolean tx = txConc != null && txIsolation != null;
+
+        final IgniteCache<String, String> clientCache = this.cache;
+        final IgniteCache<String, String> srvCache = srv.<String, String>cache(CACHE_NAME);
+
+        if (tx) {
+            doInTransaction(client, txConc, txIsolation, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    // Read from store.
+                    assertEquals(key, clientCache.get(key));
+
+                    return null;
+                }
+            });
+        }
+        else
+            assertEquals(key, clientCache.get(key));
+
+        final String updatedVal = "key_updated";
+
+        if (tx) {
+            doInTransaction(srv, txConc, txIsolation, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    // Update value.
+                    srvCache.put(key, updatedVal);
+
+                    return null;
+                }
+            });
+        }
+        else
+            srvCache.put(key, updatedVal);
+
+        if (tx) {
+            doInTransaction(client, txConc, txIsolation, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    assertEquals(updatedVal, clientCache.get(key));
+
+                    return null;
+                }
+            });
+        }
+        else
+            assertEquals(updatedVal, clientCache.get(key));
+    }
+
+    /**
+     * @param txConc Transaction concurrency.
+     * @param txIsolation Transaction isolation.
+     * @throws Exception If fail.
+     */
+    private void checkNearSingleConcurrent(final TransactionConcurrency txConc, final TransactionIsolation txIsolation) throws Exception {
+        for (int i = 0; i < 10; i++) {
+            final String key = String.valueOf(-((new Random().nextInt(99) + 1)));
+
+            boolean tx = txConc != null && txIsolation != null;
+
+            final IgniteCache<String, String> clientCache = this.cache;
+            final IgniteCache<String, String> srvCache = srv.cache(CACHE_NAME);
+
+            final CountDownLatch storeLatch = new CountDownLatch(1);
+
+            final IgniteInternalFuture<Object> fut1 = GridTestUtils.runAsync(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    storeLatch.await();
+
+                    clientCache.get(key);
+
+                    return null;
+                }
+            });
+
+
+//            IgniteInternalFuture<Object> fut2 = null;
+
+            // TODO Sometimes Near cache becomes inconsistent
+//            if (!tx) {
+//                // TODO: IGNITE-3498
+//                // TODO: Doesn't work on transactional cache.
+//                fut2 = GridTestUtils.runAsync(new Callable<Object>() {
+//                    @Override public Object call() throws Exception {
+//                        storeLatch.await();
+//
+//                        srvCache.remove(key);
+//
+//                        return null;
+//                    }
+//                });
+//            }
+
+            final IgniteInternalFuture<Object> fut3 = GridTestUtils.runAsync(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    storeLatch.await();
+
+                    srvCache.put(key, "other");
+
+                    return null;
+                }
+            });
+
+            storeLatch.countDown();
+
+            fut1.get();
+
+//            if (!tx)
+//                fut2.get();
+
+            fut3.get();
+
+            final String srvVal = srvCache.get(key);
+            final String clientVal = clientCache.get(key);
+
+            assertEquals(srvVal, clientVal);
+        }
+    }
+
+    /**
+     * @param txConc Transaction concurrency.
+     * @param txIsolation Transaction isolation.
+     * @throws Exception If fail.
+     */
+    private void checkNearBatch(TransactionConcurrency txConc, TransactionIsolation txIsolation) throws Exception {
+        final Map<String, String> data1 = new HashMap<>();
+        final Map<String, String> data2 = new HashMap<>();
+
+        for (int i = 0; i < 10; i++) {
+            data1.put(String.valueOf(i), String.valueOf(i));
+            data2.put(String.valueOf(i), "other");
+        }
+
+        final IgniteCache<String, String> clientCache = this.cache;
+        final IgniteCache<String, String> srvCache = srv.cache(CACHE_NAME);
+
+        boolean tx = txConc != null && txIsolation != null;
+
+        // Read from store.
+        if (tx) {
+            doInTransaction(client, txConc, txIsolation, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    assertEquals(data1, clientCache.getAll(data1.keySet()));
+
+                    return null;
+                }
+            });
+        }
+        else
+            assertEquals(data1, clientCache.getAll(data1.keySet()));
+
+        // Update value.
+        if (tx) {
+            doInTransaction(srv, txConc, txIsolation, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    srvCache.putAll(data2);
+
+                    return null;
+                }
+            });
+        }
+        else
+            srvCache.putAll(data2);
+
+        if (tx) {
+            doInTransaction(client, txConc, txIsolation, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    assertEquals(data2, clientCache.getAll(data2.keySet()));
+
+                    return null;
+                }
+            });
+        }
+        else
+            assertEquals(data2, clientCache.getAll(data2.keySet()));
+    }
+
+    /**
+     * @param txConc Transaction concurrency.
+     * @param txIsolation Transaction isolation.
+     * @throws Exception If fail.
+     */
+    private void checkNearBatchConcurrent(TransactionConcurrency txConc, TransactionIsolation txIsolation)
+        throws Exception {
+        final Map<String, String> data1 = new HashMap<>();
+        final Map<String, String> data2 = new HashMap<>();
+
+        for (int j = 0; j < 10; j++) {
+            data1.clear();
+            data2.clear();
+
+            for (int i = j * 10; i < j * 10 + 10; i++) {
+                data1.put(String.valueOf(i), String.valueOf(i));
+                data2.put(String.valueOf(i), "other");
+            }
+
+            final IgniteCache<String, String> clientCache = this.cache;
+            final IgniteCache<String, String> srvCache = srv.cache(CACHE_NAME);
+
+            boolean tx = txConc != null && txIsolation != null;
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            final IgniteInternalFuture<Object> fut1 = GridTestUtils.runAsync(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    latch.await();
+
+                    clientCache.getAll(data1.keySet());
+
+                    return null;
+                }
+            });
+
+            IgniteInternalFuture<Object> fut2 =  GridTestUtils.runAsync(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    latch.await();
+
+                    srvCache.putAll(data2);
+
+                    return null;
+                }
+            });
+
+//            IgniteInternalFuture<Object> fut3 = null;
+//
+//            // TODO Sometimes Near cache becomes inconsistent
+//            if (!tx) {
+//                // TODO: IGNITE-3498
+//                // TODO: Doesn't work on transactional cache.
+//                fut3 = GridTestUtils.runAsync(new Callable<Object>() {
+//                    @Override public Object call() throws Exception {
+//                        latch.await();
+//
+//                        srvCache.removeAll(data1.keySet());
+//
+//                        return null;
+//                    }
+//                });
+//            }
+
+            latch.countDown();
+
+//            if (!tx)
+//                fut3.get();
+
+            fut1.get();
+            fut2.get();
+
+            final Map<String, String> srvVals = srvCache.getAll(data1.keySet());
+            final Map<String, String> clientVals = clientCache.getAll(data1.keySet());
+
+            assertEquals(srvVals, clientVals);
+        }
+    }
+
+    /**
+     * @return Cache configuration.
+     */
+    protected CacheConfiguration<String, String> cacheConfiguration() {
+        CacheConfiguration<String, String> cfg = new CacheConfiguration<>(CACHE_NAME);
+
+        cfg.setCacheStoreFactory(new StoreFactory());
+
+        cfg.setReadThrough(true);
+        cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+        return cfg;
+    }
+
+    /**
+     *
+     */
+    private static class StoreFactory implements Factory<CacheStore<? super String, ? super String>> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public CacheStore<? super String, ? super String> create() {
+            return new TestStore();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestStore extends CacheStoreAdapter<String, String> implements Serializable {
+        /** */
+        private final ConcurrentHashMap<String, String> map = new ConcurrentHashMap<>();
+
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         *
+         */
+        public TestStore() {
+            for (int i = -100; i < 1000; i++)
+                map.put(String.valueOf(i), String.valueOf(i));
+
+            map.put("key", "key");
+        }
+
+        /** {@inheritDoc} */
+        @Override public String load(String key) throws CacheLoaderException {
+            return map.get(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends String, ? extends String> entry) throws CacheWriterException {
+            map.put(entry.getKey(), entry.getValue());
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("SuspiciousMethodCalls")
+        @Override public void delete(Object key) throws CacheWriterException {
+            map.remove(key);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOffheapCacheStoreUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOffheapCacheStoreUpdateTest.java
new file mode 100644
index 0000000000000..ae3f695d0a443
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOffheapCacheStoreUpdateTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+
+/**
+ * Check that near cache is updated when entry loaded from store.
+ */
+public class GridNearOffheapCacheStoreUpdateTest extends GridNearCacheStoreUpdateTest {
+    /** {@inheritDoc} */
+    @Override protected CacheConfiguration<String, String> cacheConfiguration() {
+        final CacheConfiguration<String, String> ccfg = super.cacheConfiguration();
+
+        ccfg.setMemoryMode(CacheMemoryMode.OFFHEAP_TIERED);
+
+        return ccfg;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
index 8792ea16ac9af..af46c5783cfc7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
@@ -114,8 +114,10 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheRendezvousAffinityClientSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridPartitionedBackupLoadSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheStoreUpdateTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.NearCacheSyncUpdateTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.NoneRebalanceModeSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearOffheapCacheStoreUpdateTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedEvictionSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedJobExecutionTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicBasicStoreSelfTest;
@@ -271,6 +273,9 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(OffheapCacheOnClientsTest.class));
         suite.addTest(new TestSuite(CacheConcurrentReadThroughTest.class));
 
+        suite.addTest(new TestSuite(GridNearCacheStoreUpdateTest.class));
+        suite.addTest(new TestSuite(GridNearOffheapCacheStoreUpdateTest.class));
+
         return suite;
     }
 }

From 2eb24cad277e14322cf42155697cae78e0f80e13 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 18 Jan 2017 13:00:25 +0300
Subject: [PATCH 108/446] ignite-4147 - Fail if joining node has different of
 cluster SSL configuration.

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 20 +++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  9 ++
 .../TcpDiscoverySslSecuredUnsecuredTest.java  | 93 +++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |  4 +-
 4 files changed, 124 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 0f5f74181516e..9a1261c52a6fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -20,6 +20,7 @@
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StreamCorruptedException;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
@@ -44,6 +45,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.atomic.AtomicReference;
+import javax.net.ssl.SSLException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
@@ -655,6 +657,14 @@ else if (addrs.isEmpty()) {
 
                 errs.add(e);
 
+                if (X.hasCause(e, SSLException.class))
+                    throw new IgniteSpiException("Unable to establish secure connection. " +
+                        "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
+                if (X.hasCause(e, StreamCorruptedException.class))
+                    throw new IgniteSpiException("Unable to establish plain connection. " +
+                        "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
@@ -1527,7 +1537,15 @@ private void tryJoin() throws InterruptedException {
 
             joinCnt++;
 
-            T2<SocketStream, Boolean> joinRes = joinTopology(false, spi.joinTimeout);
+            T2<SocketStream, Boolean> joinRes;
+            try {
+                joinRes = joinTopology(false, spi.joinTimeout);
+            }
+            catch (IgniteSpiException e) {
+                joinError(e);
+
+                return;
+            }
 
             if (joinRes == null) {
                 if (join)
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index c79133379c267..40da281cbc916 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -24,6 +24,7 @@
 import java.io.ObjectStreamException;
 import java.io.OutputStream;
 import java.io.Serializable;
+import java.io.StreamCorruptedException;
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -1218,6 +1219,14 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
                 errs.add(e);
 
+                if (X.hasCause(e, SSLException.class))
+                    throw new IgniteException("Unable to establish secure connection. " +
+                        "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
+                if (X.hasCause(e, StreamCorruptedException.class))
+                    throw new IgniteException("Unable to establish plain connection. " +
+                        "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java
new file mode 100644
index 0000000000000..229616512cdf8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests cases when node connects to cluster with different SSL configuration.
+ * Exception with meaningful message should be thrown.
+ */
+public class TcpDiscoverySslSecuredUnsecuredTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(gridName.contains("client"));
+
+        if (gridName.contains("ssl"))
+            cfg.setSslContextFactory(GridTestUtils.sslFactory());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSecuredUnsecuredServerConnection() throws Exception {
+        checkConnection("plain-server", "ssl-server");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testUnsecuredSecuredServerConnection() throws Exception {
+        checkConnection("ssl-server", "plain-server");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSecuredClientUnsecuredServerConnection() throws Exception {
+        checkConnection("plain-server", "ssl-client");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testUnsecuredClientSecuredServerConnection() throws Exception {
+        checkConnection("ssl-server", "plain-client");
+    }
+
+    /**
+     * @param name1 First grid name.
+     * @param name2 Second grid name.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    private void checkConnection(final String name1, final String name2) throws Exception {
+        startGrid(name1);
+
+        GridTestUtils.assertThrows(null, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                startGrid(name2);
+
+                return null;
+            }
+        }, IgniteCheckedException.class, null);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index af7eb7e8e2403..98bf6daa06c03 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiStartStopSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSecuredUnsecuredTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSelfTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.jdbc.TcpDiscoveryJdbcIpFinderSelfTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinderSelfTest;
@@ -86,7 +87,8 @@ public static TestSuite suite() throws Exception {
 
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
+        suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));
 
         return suite;
     }
-}
\ No newline at end of file
+}

From 2305e38345d8a7ca812d265d00eaca5bb7d6adb1 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 18 Jan 2017 14:57:53 +0300
Subject: [PATCH 109/446] Minor fixes in tests.

---
 .../processors/cache/IgniteCacheAbstractQuerySelfTest.java    | 4 ++--
 ...iteCacheDistributedQueryStopOnCancelOrTimeoutSelfTest.java | 3 ++-
 ...acheQueryStopOnCancelOrTimeoutDistributedJoinSelfTest.java | 3 ++-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
index c5a241e54ce1f..9f56877769313 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
@@ -642,7 +642,7 @@ public void testSimpleCustomTableName() throws Exception {
 
                 return null;
             }
-        }, IgniteException.class, null);
+        }, CacheException.class, null);
     }
 
     /**
@@ -688,7 +688,7 @@ public void testMixedCustomTableName() throws Exception {
 
                 return null;
             }
-        }, IgniteException.class, null);
+        }, CacheException.class, null);
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryStopOnCancelOrTimeoutSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryStopOnCancelOrTimeoutSelfTest.java
index 80c4a086a09d2..50fb03449b050 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryStopOnCancelOrTimeoutSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryStopOnCancelOrTimeoutSelfTest.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.cache.query.QueryCancelledException;
+import org.apache.ignite.internal.processors.GridProcessor;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -244,7 +245,7 @@ private void checkCleanState() {
             IgniteEx grid = grid(i);
 
             // Validate everything was cleaned up.
-            ConcurrentMap<UUID, ?> map = U.field(((IgniteH2Indexing)U.field(U.field(
+            ConcurrentMap<UUID, ?> map = U.field(((IgniteH2Indexing)U.field((GridProcessor)U.field(
                 grid.context(), "qryProc"), "idx")).mapQueryExecutor(), "qryRess");
 
             String msg = "Map executor state is not cleared";
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryStopOnCancelOrTimeoutDistributedJoinSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryStopOnCancelOrTimeoutDistributedJoinSelfTest.java
index 4baaf8f650d81..3263b4155f55c 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryStopOnCancelOrTimeoutDistributedJoinSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryStopOnCancelOrTimeoutDistributedJoinSelfTest.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.GridProcessor;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -122,7 +123,7 @@ private void checkCleanState() {
             IgniteEx grid = grid(i);
 
             // Validate everything was cleaned up.
-            ConcurrentMap<UUID, ?> map = U.field(((IgniteH2Indexing) U.field(U.field(
+            ConcurrentMap<UUID, ?> map = U.field(((IgniteH2Indexing)U.field((GridProcessor)U.field(
                     grid.context(), "qryProc"), "idx")).mapQueryExecutor(), "qryRess");
 
             String msg = "Map executor state is not cleared";

From ecf4b8b5bd05a5c1120e08d9951cddd26d0e924c Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 18 Jan 2017 17:22:14 +0300
Subject: [PATCH 110/446] Minor fix test for Ignite-4247.

---
 .../cache/IgniteCacheAbstractQuerySelfTest.java           | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
index 9f56877769313..7e0d20b76bdb6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
@@ -633,7 +633,7 @@ public void testSimpleCustomTableName() throws Exception {
 
         assertEquals(2, qry.getAll().size());
 
-        GridTestUtils.assertThrows(log, new GridPlainCallable<Void>() {
+        Throwable throwable = GridTestUtils.assertThrowsInherited(log, new GridPlainCallable<Void>() {
             @Override public Void call() throws Exception {
                 QueryCursor<Cache.Entry<Integer, Type1>> qry =
                     cache.query(new SqlQuery<Integer, Type1>(Type1.class, "FROM Type1"));
@@ -642,7 +642,11 @@ public void testSimpleCustomTableName() throws Exception {
 
                 return null;
             }
-        }, CacheException.class, null);
+        }, RuntimeException.class, null);
+
+        assertNotNull(throwable);
+
+        assertTrue(throwable instanceof IgniteException || throwable instanceof CacheException);
     }
 
     /**

From f9aaf0353cea54afefea4caac74b1583eb17969b Mon Sep 17 00:00:00 2001
From: agura <agura>
Date: Wed, 18 Jan 2017 18:04:45 +0300
Subject: [PATCH 111/446] ignite-4499 Drop node from topology in case when
 connection creation is impossible

---
 .../apache/ignite/IgniteSystemProperties.java |   3 +
 .../tcp/TcpCommunicationSpi.java              |  16 +
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  88 ++++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  61 ++--
 .../messages/TcpDiscoveryAbstractMessage.java |  21 ++
 .../tcp/TcpCommunicationSpiDropNodesTest.java | 322 ++++++++++++++++++
 .../TcpCommunicationSpiFaultyClientTest.java  | 270 +++++++++++++++
 .../ignite/testframework/GridTestNode.java    |   1 +
 .../junits/GridAbstractTest.java              |   2 +
 .../IgniteSpiCommunicationSelfTestSuite.java  |   5 +
 .../IgniteCacheAbstractQuerySelfTest.java     |   8 +-
 11 files changed, 758 insertions(+), 39 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 0da0f4990912a..d77b2fb2838d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -65,6 +65,9 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_NO_DISCO_ORDER = "IGNITE_NO_DISCO_ORDER";
 
+    /** Defines reconnect delay in milliseconds for client node that was failed forcible. */
+    public static final String IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY = "IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY";
+
     /**
      * If this system property is set to {@code false} - no checks for new versions will
      * be performed by Ignite. By default, Ignite periodically checks for the new
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 1fe437cc710c0..94b7efe078e84 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -94,6 +94,7 @@
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -2550,6 +2551,21 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     "operating system firewall is disabled on local and remote hosts) " +
                     "[addrs=" + addrs + ']');
 
+            if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
+                X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
+                    IgniteSpiOperationTimeoutException.class)) {
+                LT.warn(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
+                    "cluster [" +
+                    "rmtNode=" + node +
+                    ", err=" + errs +
+                    ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
+
+                getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
+                    "rmtNode=" + node +
+                    ", errs=" + errs +
+                    ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
+            }
+
             throw errs;
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 9a1261c52a6fa..35f0908a20b34 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -50,6 +50,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheMetrics;
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
@@ -100,6 +101,7 @@
 import org.jsr166.ConcurrentHashMap8;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY;
 import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
 import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
@@ -166,6 +168,9 @@ class ClientImpl extends TcpDiscoveryImpl {
     /** */
     protected MessageWorker msgWorker;
 
+    /** Force fail message for local node. */
+    private TcpDiscoveryNodeFailedMessage forceFailMsg;
+
     /** */
     @GridToStringExclude
     private int joinCnt;
@@ -450,6 +455,8 @@ else if (state == DISCONNECTED) {
 
             msg.warning(warning);
 
+            msg.force(true);
+
             msgWorker.addMessage(msg);
         }
     }
@@ -1396,6 +1403,14 @@ else if (msg == SPI_STOP) {
                         else
                             leaveLatch.countDown();
                     }
+                    else if (msg instanceof TcpDiscoveryNodeFailedMessage &&
+                        ((TcpDiscoveryNodeFailedMessage)msg).failedNodeId().equals(locNode.id())) {
+                        TcpDiscoveryNodeFailedMessage msg0 = (TcpDiscoveryNodeFailedMessage)msg;
+
+                        assert msg0.force() : msg0;
+
+                        forceFailMsg = msg0;
+                    }
                     else if (msg instanceof SocketClosedMessage) {
                         if (((SocketClosedMessage)msg).sock == currSock) {
                             currSock = null;
@@ -1412,25 +1427,45 @@ else if (msg instanceof SocketClosedMessage) {
                                 }
                             }
                             else {
-                                if (log.isDebugEnabled())
-                                    log.debug("Connection closed, will try to restore connection.");
+                                if (forceFailMsg != null) {
+                                    if (log.isDebugEnabled()) {
+                                        log.debug("Connection closed, local node received force fail message, " +
+                                            "will not try to restore connection");
+                                    }
+
+                                    queue.addFirst(SPI_RECONNECT_FAILED);
+                                }
+                                else {
+                                    if (log.isDebugEnabled())
+                                        log.debug("Connection closed, will try to restore connection.");
 
-                                assert reconnector == null;
+                                    assert reconnector == null;
 
-                                final Reconnector reconnector = new Reconnector(join);
-                                this.reconnector = reconnector;
-                                reconnector.start();
+                                    final Reconnector reconnector = new Reconnector(join);
+                                    this.reconnector = reconnector;
+                                    reconnector.start();
+                                }
                             }
                         }
                     }
                     else if (msg == SPI_RECONNECT_FAILED) {
-                        reconnector.cancel();
-                        reconnector.join();
+                        if (reconnector != null) {
+                            reconnector.cancel();
+                            reconnector.join();
 
-                        reconnector = null;
+                            reconnector = null;
+                        }
+                        else
+                            assert forceFailMsg != null;
 
                         if (spi.isClientReconnectDisabled()) {
                             if (state != SEGMENTED && state != STOPPED) {
+                                if (forceFailMsg != null) {
+                                    U.quietAndWarn(log, "Local node was dropped from cluster due to network problems " +
+                                        "[nodeInitiatedFail=" + forceFailMsg.creatorNodeId() +
+                                        ", msg=" + forceFailMsg.warning() + ']');
+                                }
+
                                 if (log.isDebugEnabled()) {
                                     log.debug("Failed to restore closed connection, reconnect disabled, " +
                                         "local node segmented [networkTimeout=" + spi.netTimeout + ']');
@@ -1445,7 +1480,9 @@ else if (msg == SPI_RECONNECT_FAILED) {
                             if (state == STARTING || state == CONNECTED) {
                                 if (log.isDebugEnabled()) {
                                     log.debug("Failed to restore closed connection, will try to reconnect " +
-                                        "[networkTimeout=" + spi.netTimeout + ", joinTimeout=" + spi.joinTimeout + ']');
+                                        "[networkTimeout=" + spi.netTimeout +
+                                        ", joinTimeout=" + spi.joinTimeout +
+                                        ", failMsg=" + forceFailMsg + ']');
                                 }
 
                                 state = DISCONNECTED;
@@ -1468,7 +1505,36 @@ else if (msg == SPI_RECONNECT_FAILED) {
 
                             UUID newId = UUID.randomUUID();
 
-                            if (log.isInfoEnabled()) {
+                            if (forceFailMsg != null) {
+                                long delay = IgniteSystemProperties.getLong(IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY,
+                                    10_000);
+
+                                if (delay > 0) {
+                                    U.quietAndWarn(log, "Local node was dropped from cluster due to network problems, " +
+                                        "will try to reconnect with new id after " + delay + "ms (reconnect delay " +
+                                        "can be changed using IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY system " +
+                                        "property) [" +
+                                        "newId=" + newId +
+                                        ", prevId=" + locNode.id() +
+                                        ", locNode=" + locNode +
+                                        ", nodeInitiatedFail=" + forceFailMsg.creatorNodeId() +
+                                        ", msg=" + forceFailMsg.warning() + ']');
+
+                                    Thread.sleep(delay);
+                                }
+                                else {
+                                    U.quietAndWarn(log, "Local node was dropped from cluster due to network problems, " +
+                                        "will try to reconnect with new id [" +
+                                        "newId=" + newId +
+                                        ", prevId=" + locNode.id() +
+                                        ", locNode=" + locNode +
+                                        ", nodeInitiatedFail=" + forceFailMsg.creatorNodeId() +
+                                        ", msg=" + forceFailMsg.warning() + ']');
+                                }
+
+                                forceFailMsg = null;
+                            }
+                            else if (log.isInfoEnabled()) {
                                 log.info("Client node disconnected from cluster, will try to reconnect with new id " +
                                     "[newId=" + newId + ", prevId=" + locNode.id() + ", locNode=" + locNode + ']');
                             }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 40da281cbc916..f33566cc01a5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -775,6 +775,8 @@ private void interruptPing(TcpDiscoveryNode node) {
 
             msg.warning(warning);
 
+            msg.force(true);
+
             msgWorker.addMessage(msg);
         }
     }
@@ -4610,8 +4612,12 @@ private void processNodeFailedMessage(TcpDiscoveryNodeFailedMessage msg) {
                 else {
                     boolean contains;
 
+                    UUID creatorId = msg.creatorNodeId();
+
+                    assert creatorId != null : msg;
+
                     synchronized (mux) {
-                        contains = failedNodes.containsKey(sndNode);
+                        contains = failedNodes.containsKey(sndNode) || ring.node(creatorId) == null;
                     }
 
                     if (contains) {
@@ -4623,25 +4629,29 @@ private void processNodeFailedMessage(TcpDiscoveryNodeFailedMessage msg) {
                 }
             }
 
-            UUID nodeId = msg.failedNodeId();
+            UUID failedNodeId = msg.failedNodeId();
             long order = msg.order();
 
-            TcpDiscoveryNode node = ring.node(nodeId);
+            TcpDiscoveryNode failedNode = ring.node(failedNodeId);
 
-            if (node != null && node.internalOrder() != order) {
+            if (failedNode != null && failedNode.internalOrder() != order) {
                 if (log.isDebugEnabled())
                     log.debug("Ignoring node failed message since node internal order does not match " +
-                        "[msg=" + msg + ", node=" + node + ']');
+                        "[msg=" + msg + ", node=" + failedNode + ']');
 
                 return;
             }
 
-            if (node != null) {
-                assert !node.isLocal() || !msg.verified() : msg;
+            if (failedNode != null) {
+                assert !failedNode.isLocal() || !msg.verified() : msg;
 
-                synchronized (mux) {
-                    if (!failedNodes.containsKey(node))
-                        failedNodes.put(node, msg.senderNodeId() != null ? msg.senderNodeId() : getLocalNodeId());
+                boolean skipUpdateFailedNodes = msg.force() && !msg.verified();
+
+                if (!skipUpdateFailedNodes) {
+                    synchronized (mux) {
+                        if (!failedNodes.containsKey(failedNode))
+                            failedNodes.put(failedNode, msg.senderNodeId() != null ? msg.senderNodeId() : getLocalNodeId());
+                    }
                 }
             }
             else {
@@ -4668,11 +4678,11 @@ private void processNodeFailedMessage(TcpDiscoveryNodeFailedMessage msg) {
             }
 
             if (msg.verified()) {
-                node = ring.removeNode(nodeId);
+                failedNode = ring.removeNode(failedNodeId);
 
-                interruptPing(node);
+                interruptPing(failedNode);
 
-                assert node != null;
+                assert failedNode != null;
 
                 long topVer;
 
@@ -4698,16 +4708,18 @@ private void processNodeFailedMessage(TcpDiscoveryNodeFailedMessage msg) {
                 }
 
                 synchronized (mux) {
-                    failedNodes.remove(node);
+                    failedNodes.remove(failedNode);
 
-                    leavingNodes.remove(node);
+                    leavingNodes.remove(failedNode);
 
-                    failedNodesMsgSent.remove(node.id());
+                    failedNodesMsgSent.remove(failedNode.id());
 
-                    ClientMessageWorker worker = clientMsgWorkers.remove(node.id());
+                    if (!msg.force()) { // ClientMessageWorker will stop after sending force fail message.
+                        ClientMessageWorker worker = clientMsgWorkers.remove(failedNode.id());
 
-                    if (worker != null)
-                        worker.interrupt();
+                        if (worker != null)
+                            worker.interrupt();
+                    }
                 }
 
                 if (msg.warning() != null && !msg.creatorNodeId().equals(getLocalNodeId())) {
@@ -4719,10 +4731,10 @@ private void processNodeFailedMessage(TcpDiscoveryNodeFailedMessage msg) {
                 }
 
                 synchronized (mux) {
-                    joiningNodes.remove(node.id());
+                    joiningNodes.remove(failedNode.id());
                 }
 
-                notifyDiscovery(EVT_NODE_FAILED, topVer, node);
+                notifyDiscovery(EVT_NODE_FAILED, topVer, failedNode);
 
                 spi.stats.onNodeFailed();
             }
@@ -6317,7 +6329,12 @@ else if (msgLog.isDebugEnabled())
                         spi.failureDetectionTimeout() : spi.getSocketTimeout());
                 }
 
-                success = true;
+                boolean clientFailed = msg instanceof TcpDiscoveryNodeFailedMessage &&
+                    ((TcpDiscoveryNodeFailedMessage)msg).failedNodeId().equals(clientNodeId);
+
+                assert !clientFailed || msg.force() : msg;
+
+                success = !clientFailed;
             }
             catch (IgniteCheckedException | IOException e) {
                 if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryAbstractMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryAbstractMessage.java
index 783a113ea1ff6..e982b2f32ea31 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryAbstractMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryAbstractMessage.java
@@ -48,6 +48,9 @@ public abstract class TcpDiscoveryAbstractMessage implements Serializable {
     /** */
     protected static final int CLIENT_ACK_FLAG_POS = 4;
 
+    /** */
+    protected static final int FORCE_FAIL_FLAG_POS = 8;
+
     /** Sender of the message (transient). */
     private transient UUID sndNodeId;
 
@@ -204,6 +207,24 @@ public void client(boolean client) {
         setFlag(CLIENT_FLAG_POS, client);
     }
 
+    /**
+     * Get force fail node flag.
+     *
+     * @return Force fail node flag.
+     */
+    public boolean force() {
+        return getFlag(FORCE_FAIL_FLAG_POS);
+    }
+
+    /**
+     * Sets force fail node flag.
+     *
+     * @param force Force fail node flag.
+     */
+    public void force(boolean force) {
+        setFlag(FORCE_FAIL_FLAG_POS, force);
+    }
+
     /**
      * @return Pending message index.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
new file mode 100644
index 0000000000000..d29231e449b0b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+
+/**
+ *
+ */
+public class TcpCommunicationSpiDropNodesTest extends GridCommonAbstractTest {
+    /** IP finder. */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** Nodes count. */
+    private static final int NODES_CNT = 4;
+
+    /** Block. */
+    private static volatile boolean block;
+
+    /** Predicate. */
+    private static IgniteBiPredicate<ClusterNode, ClusterNode> pred;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClockSyncFrequency(300000);
+        cfg.setFailureDetectionTimeout(1000);
+
+        TestCommunicationSpi spi = new TestCommunicationSpi();
+
+        spi.setIdleConnectionTimeout(100);
+        spi.setSharedMemoryPort(-1);
+
+        TcpDiscoverySpi discoSpi = (TcpDiscoverySpi) cfg.getDiscoverySpi();
+        discoSpi.setIpFinder(IP_FINDER);
+
+        cfg.setCommunicationSpi(spi);
+        cfg.setDiscoverySpi(discoSpi);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        block = false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneNode() throws Exception {
+        pred = new IgniteBiPredicate<ClusterNode, ClusterNode>() {
+            @Override public boolean apply(ClusterNode locNode, ClusterNode rmtNode) {
+                return block && rmtNode.order() == 3;
+            }
+        };
+
+        startGrids(NODES_CNT);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        grid(0).events().localListen(new IgnitePredicate<Event>() {
+            @Override
+            public boolean apply(Event event) {
+                latch.countDown();
+
+                return true;
+            }
+        }, EVT_NODE_FAILED);
+
+        U.sleep(1000); // Wait for write timeout and closing idle connections.
+
+        block = true;
+
+        grid(0).compute().broadcast(new IgniteRunnable() {
+            @Override public void run() {
+                // No-op.
+            }
+        });
+
+        assertTrue(latch.await(15, TimeUnit.SECONDS));
+
+        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return grid(3).cluster().topologyVersion() == NODES_CNT + 1;
+            }
+        }, 5000));
+
+        for (int i = 0; i < 10; i++) {
+            U.sleep(1000);
+
+            assertEquals(NODES_CNT - 1, grid(0).cluster().nodes().size());
+
+            int liveNodesCnt = 0;
+
+            for (int j = 0; j < NODES_CNT; j++) {
+                IgniteEx ignite;
+
+                try {
+                    ignite = grid(j);
+
+                    log.info("Checking topology for grid(" + j + "): " + ignite.cluster().nodes());
+
+                    ClusterNode locNode = ignite.localNode();
+
+                    if (locNode.order() != 3) {
+                        assertEquals(NODES_CNT - 1, ignite.cluster().nodes().size());
+
+                        for (ClusterNode node : ignite.cluster().nodes())
+                            assertTrue(node.order() != 3);
+
+                        liveNodesCnt++;
+                    }
+                }
+                catch (Exception e) {
+                    log.info("Checking topology for grid(" + j + "): no grid in topology.");
+                }
+            }
+
+            assertEquals(NODES_CNT - 1, liveNodesCnt);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTwoNodesEachOther() throws Exception {
+        pred = new IgniteBiPredicate<ClusterNode, ClusterNode>() {
+            @Override public boolean apply(ClusterNode locNode, ClusterNode rmtNode) {
+                return block && (locNode.order() == 2 || locNode.order() == 4) &&
+                    (rmtNode.order() == 2 || rmtNode.order() == 4);
+            }
+        };
+
+        startGrids(NODES_CNT);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        grid(0).events().localListen(new IgnitePredicate<Event>() {
+            @Override
+            public boolean apply(Event event) {
+                latch.countDown();
+
+                return true;
+            }
+        }, EVT_NODE_FAILED);
+
+        U.sleep(1000); // Wait for write timeout and closing idle connections.
+
+        block = true;
+
+        final CyclicBarrier barrier = new CyclicBarrier(2);
+
+        IgniteInternalFuture<Void> fut1 = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                barrier.await();
+
+                grid(1).compute().withNoFailover().broadcast(new IgniteRunnable() {
+                    @Override public void run() {
+                        // No-op.
+                    }
+                });
+
+                return null;
+            }
+        });
+
+        IgniteInternalFuture<Void> fut2 = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                barrier.await();
+
+                grid(3).compute().withNoFailover().broadcast(new IgniteRunnable() {
+                    @Override public void run() {
+                        // No-op.
+                    }
+                });
+
+                return null;
+            }
+        });
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+
+        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return grid(2).cluster().nodes().size() == NODES_CNT - 1;
+            }
+        }, 5000);
+
+        try {
+            fut1.get();
+        }
+        catch (IgniteCheckedException e) {
+            // No-op.
+        }
+
+        try {
+            fut2.get();
+        }
+        catch (IgniteCheckedException e) {
+            // No-op.
+        }
+
+        long failedNodeOrder = 1 + 2 + 3 + 4;
+
+        for (ClusterNode node : grid(0).cluster().nodes())
+            failedNodeOrder -= node.order();
+
+        for (int i = 0; i < 10; i++) {
+            U.sleep(1000);
+
+            assertEquals(NODES_CNT - 1, grid(0).cluster().nodes().size());
+
+            int liveNodesCnt = 0;
+
+            for (int j = 0; j < NODES_CNT; j++) {
+                IgniteEx ignite;
+
+                try {
+                    ignite = grid(j);
+
+                    log.info("Checking topology for grid(" + j + "): " + ignite.cluster().nodes());
+
+                    ClusterNode locNode = ignite.localNode();
+
+                    if (locNode.order() != failedNodeOrder) {
+                        assertEquals(NODES_CNT - 1, ignite.cluster().nodes().size());
+
+                        for (ClusterNode node : ignite.cluster().nodes())
+                            assertTrue(node.order() != failedNodeOrder);
+
+                        liveNodesCnt++;
+                    }
+                }
+                catch (Exception e) {
+                    log.info("Checking topology for grid(" + j + "): no grid in topology.");
+                }
+            }
+
+            assertEquals(NODES_CNT - 1, liveNodesCnt);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override protected GridCommunicationClient createTcpClient(ClusterNode node) throws IgniteCheckedException {
+            if (pred.apply(getLocalNode(), node)) {
+                Map<String, Object> attrs = new HashMap<>(node.attributes());
+
+                attrs.put(createAttributeName(ATTR_ADDRS), Collections.singleton("127.0.0.1"));
+                attrs.put(createAttributeName(ATTR_PORT), 47200);
+                attrs.put(createAttributeName(ATTR_EXT_ADDRS), Collections.emptyList());
+                attrs.put(createAttributeName(ATTR_HOST_NAMES), Collections.emptyList());
+
+                ((TcpDiscoveryNode)node).setAttributes(attrs);
+            }
+
+            return super.createTcpClient(node);
+        }
+
+        /**
+         * @param name Name.
+         */
+        private String createAttributeName(String name) {
+            return getClass().getSimpleName() + '.' + name;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
new file mode 100644
index 0000000000000..6e99487bd46ae
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+
+/**
+ * Tests that faulty client will be failed if connection can't be established.
+ */
+public class TcpCommunicationSpiFaultyClientTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** Predicate. */
+    private static final IgnitePredicate<ClusterNode> PRED = new IgnitePredicate<ClusterNode>() {
+        @Override public boolean apply(ClusterNode node) {
+            return block && node.order() == 3;
+        }
+    };
+
+    /** Client mode. */
+    private static boolean clientMode;
+
+    /** Block. */
+    private static volatile boolean block;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClockSyncFrequency(300000);
+        cfg.setFailureDetectionTimeout(1000);
+        cfg.setClientMode(clientMode);
+
+        TestCommunicationSpi spi = new TestCommunicationSpi();
+
+        spi.setIdleConnectionTimeout(100);
+        spi.setSharedMemoryPort(-1);
+
+        TcpDiscoverySpi discoSpi = (TcpDiscoverySpi) cfg.getDiscoverySpi();
+
+        discoSpi.setIpFinder(IP_FINDER);
+        discoSpi.setClientReconnectDisabled(true);
+
+        cfg.setCommunicationSpi(spi);
+        cfg.setDiscoverySpi(discoSpi);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        block = false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoServerOnHost() throws Exception {
+        testFailClient(null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNotAcceptedConnection() throws Exception {
+        testFailClient(new FakeServer());
+    }
+
+    /**
+     * @param srv Server.
+     * @throws Exception If failed.
+     */
+    private void testFailClient(FakeServer srv) throws Exception {
+        IgniteInternalFuture<Long> fut = null;
+
+        try {
+            if (srv != null)
+                fut = GridTestUtils.runMultiThreadedAsync(srv, 1, "fake-server");
+
+            clientMode = false;
+
+            startGrids(2);
+
+            clientMode = true;
+
+            startGrid(2);
+            startGrid(3);
+
+            U.sleep(1000); // Wait for write timeout and closing idle connections.
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            grid(0).events().localListen(new IgnitePredicate<Event>() {
+                @Override
+                public boolean apply(Event event) {
+                    latch.countDown();
+
+                    return true;
+                }
+            }, EVT_NODE_FAILED);
+
+            block = true;
+
+            try {
+                grid(0).compute(grid(0).cluster().forClients()).withNoFailover().broadcast(new IgniteRunnable() {
+                    @Override public void run() {
+                        // No-op.
+                    }
+                });
+            }
+            catch (IgniteException e) {
+                // No-op.
+            }
+
+            assertTrue(latch.await(3, TimeUnit.SECONDS));
+
+            assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return grid(0).cluster().forClients().nodes().size() == 1;
+                }
+            }, 5000));
+
+            for (int i = 0; i < 5; i++) {
+                U.sleep(1000);
+
+                log.info("Check topology (" + (i + 1) + "): " + grid(0).cluster().nodes());
+
+                assertEquals(1, grid(0).cluster().forClients().nodes().size());
+            }
+        }
+        finally {
+            if (srv != null) {
+                srv.stop();
+
+                assert fut != null;
+
+                fut.get();
+            }
+
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * Server that emulates connection troubles.
+     */
+    private static class FakeServer implements Runnable {
+        /** Server. */
+        private final ServerSocket srv;
+
+        /** Stop. */
+        private volatile boolean stop;
+
+        /**
+         * Default constructor.
+         */
+        FakeServer() throws IOException {
+            this.srv = new ServerSocket(47200, 50, InetAddress.getByName("127.0.0.1"));
+        }
+
+        /**
+         *
+         */
+        public void stop() {
+            stop = true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            try {
+                while (!stop) {
+                    try {
+                        U.sleep(10);
+                    }
+                    catch (IgniteInterruptedCheckedException e) {
+                        // No-op.
+                    }
+                }
+            }
+            finally {
+                U.closeQuiet(srv);
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override protected GridCommunicationClient createTcpClient(ClusterNode node) throws IgniteCheckedException {
+            if (PRED.apply(node)) {
+                Map<String, Object> attrs = new HashMap<>(node.attributes());
+
+                attrs.put(createAttributeName(ATTR_ADDRS), Collections.singleton("127.0.0.1"));
+                attrs.put(createAttributeName(ATTR_PORT), 47200);
+                attrs.put(createAttributeName(ATTR_EXT_ADDRS), Collections.emptyList());
+                attrs.put(createAttributeName(ATTR_HOST_NAMES), Collections.emptyList());
+
+                ((TcpDiscoveryNode)node).setAttributes(attrs);
+            }
+
+            return super.createTcpClient(node);
+        }
+
+        /**
+         * @param name Name.
+         */
+        private String createAttributeName(String name) {
+            return getClass().getSimpleName() + '.' + name;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestNode.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestNode.java
index e0e8eba0219fa..6365443542ac6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestNode.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestNode.java
@@ -82,6 +82,7 @@ public GridTestNode(UUID id) {
     private void initAttributes() {
         attrs.put(IgniteNodeAttributes.ATTR_BUILD_VER, "10");
         attrs.put(IgniteNodeAttributes.ATTR_GRID_NAME, "null");
+        attrs.put(IgniteNodeAttributes.ATTR_CLIENT_MODE, false);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 9f507e64ca688..ffaec4fbc89af 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -102,6 +102,7 @@
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.FileSystemXmlApplicationContext;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY;
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.PRIMARY;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
@@ -173,6 +174,7 @@ public abstract class GridAbstractTest extends TestCase {
     static {
         System.setProperty(IgniteSystemProperties.IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE, "10000");
         System.setProperty(IgniteSystemProperties.IGNITE_UPDATE_NOTIFIER, "false");
+        System.setProperty(IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY, "1");
 
         if (BINARY_MARSHALLER)
             GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, BinaryMarshaller.class.getName());
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
index c557fbbcad791..ddc255126e1bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
@@ -35,6 +35,8 @@
 import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiTcpNoDelayOffSelfTest;
 import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiTcpSelfTest;
 import org.apache.ignite.spi.communication.tcp.IgniteTcpCommunicationRecoveryAckClosureSelfTest;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiDropNodesTest;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiFaultyClientTest;
 
 /**
  * Test suite for all communication SPIs.
@@ -72,6 +74,9 @@ public static TestSuite suite() throws Exception {
 
         suite.addTest(new TestSuite(GridTcpCommunicationSpiConfigSelfTest.class));
 
+        suite.addTest(new TestSuite(TcpCommunicationSpiFaultyClientTest.class));
+        suite.addTest(new TestSuite(TcpCommunicationSpiDropNodesTest.class));
+
         return suite;
     }
 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
index 7e0d20b76bdb6..9f56877769313 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
@@ -633,7 +633,7 @@ public void testSimpleCustomTableName() throws Exception {
 
         assertEquals(2, qry.getAll().size());
 
-        Throwable throwable = GridTestUtils.assertThrowsInherited(log, new GridPlainCallable<Void>() {
+        GridTestUtils.assertThrows(log, new GridPlainCallable<Void>() {
             @Override public Void call() throws Exception {
                 QueryCursor<Cache.Entry<Integer, Type1>> qry =
                     cache.query(new SqlQuery<Integer, Type1>(Type1.class, "FROM Type1"));
@@ -642,11 +642,7 @@ public void testSimpleCustomTableName() throws Exception {
 
                 return null;
             }
-        }, RuntimeException.class, null);
-
-        assertNotNull(throwable);
-
-        assertTrue(throwable instanceof IgniteException || throwable instanceof CacheException);
+        }, CacheException.class, null);
     }
 
     /**

From d396398c1b4660b3bca24d2650a10f6c0677b4df Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 2 Dec 2016 10:36:41 +0300
Subject: [PATCH 112/446] ignite-4314 cache.clear should not destroy offheap
 map

(cherry picked from commit 88c06ec)
---
 .../cache/GridCacheClearAllRunnable.java      | 58 ++++++++-----------
 .../cache/GridCacheSwapManager.java           |  8 ---
 .../cache/transactions/IgniteTxHandler.java   |  2 +-
 3 files changed, 26 insertions(+), 42 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheClearAllRunnable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheClearAllRunnable.java
index 4f97e7ba18e61..9e7f32948d377 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheClearAllRunnable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheClearAllRunnable.java
@@ -87,48 +87,40 @@ public GridCacheClearAllRunnable(GridCacheAdapter<K, V> cache, GridCacheVersion
         // Clear swapped entries.
         if (!ctx.isNear()) {
             if (ctx.swap().offHeapEnabled()) {
-                if (GridQueryProcessor.isEnabled(ctx.config())) {
-                    for (Iterator<KeyCacheObject> it =
-                        ctx.swap().offHeapKeyIterator(true, true, AffinityTopologyVersion.NONE); it.hasNext();) {
-                        KeyCacheObject key = it.next();
-
-                        if (owns(key))
-                            clearEntry(cache.entryEx(key));
+                for (Iterator<KeyCacheObject> it = ctx.swap().offHeapKeyIterator(true, true, AffinityTopologyVersion.NONE); it.hasNext();) {
+                    KeyCacheObject key = it.next();
 
-                    }
+                    if (owns(key))
+                        clearEntry(cache.entryEx(key));
                 }
-                else if (id == 0)
-                    ctx.swap().clearOffHeap();
             }
 
-            if (ctx.isSwapOrOffheapEnabled()) {
-                if (ctx.swap().swapEnabled()) {
-                    if (GridQueryProcessor.isEnabled(ctx.config())) {
-                        Iterator<KeyCacheObject> it = null;
+            if (ctx.swap().swapEnabled()) {
+                if (GridQueryProcessor.isEnabled(ctx.config())) {
+                    Iterator<KeyCacheObject> it = null;
 
-                        try {
-                            it = ctx.swap().swapKeyIterator(true, true, AffinityTopologyVersion.NONE);
-                        }
-                        catch (IgniteCheckedException e) {
-                            U.error(log, "Failed to get iterator over swap.", e);
-                        }
+                    try {
+                        it = ctx.swap().swapKeyIterator(true, true, AffinityTopologyVersion.NONE);
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to get iterator over swap.", e);
+                    }
 
-                        if (it != null) {
-                            while (it.hasNext()) {
-                                KeyCacheObject key = it.next();
+                    if (it != null) {
+                        while (it.hasNext()) {
+                            KeyCacheObject key = it.next();
 
-                                if (owns(key))
-                                    clearEntry(cache.entryEx(key));
-                            }
+                            if (owns(key))
+                                clearEntry(cache.entryEx(key));
                         }
                     }
-                    else if (id == 0) {
-                        try {
-                            ctx.swap().clearSwap();
-                        }
-                        catch (IgniteCheckedException e) {
-                            U.error(log, "Failed to clearLocally entries from swap storage.", e);
-                        }
+                }
+                else if (id == 0) {
+                    try {
+                        ctx.swap().clearSwap();
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to clearLocally entries from swap storage.", e);
                     }
                 }
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java
index fd0b4718151c0..d4499b3c84060 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java
@@ -1415,14 +1415,6 @@ public void writeToSwap(int part, KeyCacheObject key, byte[] entry) throws Ignit
                 EVT_CACHE_OBJECT_SWAPPED, null, false, null, true, null, null, null, false);
     }
 
-    /**
-     * Clears off-heap.
-     */
-    public void clearOffHeap() {
-        if (offheapEnabled)
-            initOffHeap();
-    }
-
     /**
      * Clears swap.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index f784ba2942452..d56415652bc73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -1507,7 +1507,7 @@ protected void sendReply(UUID nodeId, GridDhtTxFinishRequest req, boolean commit
                                     if (log.isDebugEnabled())
                                         log.debug("Got entry removed exception, will retry: " + entry.txKey());
 
-                                    entry.cached(null);
+                                    entry.cached(cacheCtx.cache().entryEx(entry.key(), req.topologyVersion()));
                                 }
                             }
                         }

From 51e1f874624c428cc93e9c16407ec5a8b4cf8420 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 19 Jan 2017 10:52:42 +0300
Subject: [PATCH 113/446] Throw CacheException from queries API.

---
 .../processors/query/GridQueryProcessor.java         |  2 +-
 .../cache/query/IndexingSpiQuerySelfTest.java        |  3 +--
 .../cache/IgniteCacheAbstractQuerySelfTest.java      |  2 --
 .../near/IgniteCachePartitionedQuerySelfTest.java    | 12 +++++-------
 4 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 48ca2b5bab69c..c2e5717fbddf8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -905,7 +905,7 @@ public <K, V> Iterator<Cache.Entry<K, V>> queryLocal(
                 }, true);
         }
         catch (IgniteCheckedException e) {
-            throw new IgniteException(e);
+            throw new CacheException(e);
         }
         finally {
             busyLock.leaveBusy();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
index 84a13df36b52f..e780fdc5656c2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
@@ -328,8 +328,7 @@ private static class MyBinaryIndexingSpi extends MyIndexingSpi {
         }
 
         /** {@inheritDoc} */
-        @Override
-        public void onUnswap(@Nullable String spaceName, Object key, Object val) throws IgniteSpiException {
+        @Override public void onUnswap(@Nullable String spaceName, Object key, Object val) throws IgniteSpiException {
             assertTrue(key instanceof BinaryObject);
 
             assertTrue(val instanceof BinaryObject);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
index 9f56877769313..751d954f53f22 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
@@ -46,7 +46,6 @@
 import org.apache.ignite.IgniteBinary;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
@@ -264,7 +263,6 @@ protected Ignite ignite() {
 
         stopAllGrids();
 
-
         store.reset();
     }
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedQuerySelfTest.java
index b9f21dacdcc07..d5a9faf5ff8c6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedQuerySelfTest.java
@@ -39,7 +39,6 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.CommunicationSpi;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 
@@ -185,9 +184,8 @@ public void testScanQueryPagination() throws Exception {
 
                     pages.incrementAndGet();
                 }
-                else if (msg0 instanceof GridCacheQueryResponse) {
+                else if (msg0 instanceof GridCacheQueryResponse)
                     assertTrue(((GridCacheQueryResponse)msg0).data().size() <= pageSize);
-                }
             }
         };
 
@@ -211,15 +209,15 @@ else if (msg0 instanceof GridCacheQueryResponse) {
      *
      */
     private static class TestTcpCommunicationSpi extends TcpCommunicationSpi {
+        /** */
         volatile IgniteInClosure<Message> filter;
 
         /** {@inheritDoc} */
-        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackClosure)
-            throws IgniteSpiException {
-            if(filter != null)
+        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+            if (filter != null)
                 filter.apply(msg);
 
-            super.sendMessage(node, msg, ackClosure);
+            super.sendMessage(node, msg, ackC);
         }
     }
 }
\ No newline at end of file

From 9c9175d4a84194a224a4020e6185d1e2aee0a5aa Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 19 Jan 2017 11:19:18 +0300
Subject: [PATCH 114/446] ignite-4147 - Rollback due to test failing on many
 restarts, should be improved to be more durable

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 20 +---
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  9 --
 .../TcpDiscoverySslSecuredUnsecuredTest.java  | 93 -------------------
 .../IgniteSpiDiscoverySelfTestSuite.java      |  2 -
 4 files changed, 1 insertion(+), 123 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 35f0908a20b34..39c539c30fb89 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -20,7 +20,6 @@
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.StreamCorruptedException;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
@@ -45,7 +44,6 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.atomic.AtomicReference;
-import javax.net.ssl.SSLException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
@@ -664,14 +662,6 @@ else if (addrs.isEmpty()) {
 
                 errs.add(e);
 
-                if (X.hasCause(e, SSLException.class))
-                    throw new IgniteSpiException("Unable to establish secure connection. " +
-                        "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
-
-                if (X.hasCause(e, StreamCorruptedException.class))
-                    throw new IgniteSpiException("Unable to establish plain connection. " +
-                        "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
-
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
@@ -1603,15 +1593,7 @@ private void tryJoin() throws InterruptedException {
 
             joinCnt++;
 
-            T2<SocketStream, Boolean> joinRes;
-            try {
-                joinRes = joinTopology(false, spi.joinTimeout);
-            }
-            catch (IgniteSpiException e) {
-                joinError(e);
-
-                return;
-            }
+            T2<SocketStream, Boolean> joinRes = joinTopology(false, spi.joinTimeout);
 
             if (joinRes == null) {
                 if (join)
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index f33566cc01a5a..d462ac2638fd2 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -24,7 +24,6 @@
 import java.io.ObjectStreamException;
 import java.io.OutputStream;
 import java.io.Serializable;
-import java.io.StreamCorruptedException;
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -1221,14 +1220,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
                 errs.add(e);
 
-                if (X.hasCause(e, SSLException.class))
-                    throw new IgniteException("Unable to establish secure connection. " +
-                        "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
-
-                if (X.hasCause(e, StreamCorruptedException.class))
-                    throw new IgniteException("Unable to establish plain connection. " +
-                        "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
-
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java
deleted file mode 100644
index 229616512cdf8..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.tcp;
-
-import java.util.concurrent.Callable;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Tests cases when node connects to cluster with different SSL configuration.
- * Exception with meaningful message should be thrown.
- */
-public class TcpDiscoverySslSecuredUnsecuredTest extends GridCommonAbstractTest {
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
-        final IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        cfg.setClientMode(gridName.contains("client"));
-
-        if (gridName.contains("ssl"))
-            cfg.setSslContextFactory(GridTestUtils.sslFactory());
-
-        return cfg;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        stopAllGrids();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testSecuredUnsecuredServerConnection() throws Exception {
-        checkConnection("plain-server", "ssl-server");
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testUnsecuredSecuredServerConnection() throws Exception {
-        checkConnection("ssl-server", "plain-server");
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testSecuredClientUnsecuredServerConnection() throws Exception {
-        checkConnection("plain-server", "ssl-client");
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testUnsecuredClientSecuredServerConnection() throws Exception {
-        checkConnection("ssl-server", "plain-client");
-    }
-
-    /**
-     * @param name1 First grid name.
-     * @param name2 Second grid name.
-     * @throws Exception If failed.
-     */
-    @SuppressWarnings("ThrowableNotThrown")
-    private void checkConnection(final String name1, final String name2) throws Exception {
-        startGrid(name1);
-
-        GridTestUtils.assertThrows(null, new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                startGrid(name2);
-
-                return null;
-            }
-        }, IgniteCheckedException.class, null);
-    }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 98bf6daa06c03..5f870a474578b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -34,7 +34,6 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiStartStopSelfTest;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSecuredUnsecuredTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSelfTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.jdbc.TcpDiscoveryJdbcIpFinderSelfTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinderSelfTest;
@@ -87,7 +86,6 @@ public static TestSuite suite() throws Exception {
 
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
-        suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));
 
         return suite;
     }

From f35057833900ba028b3a9a8bd547df61f42a45ed Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 19 Jan 2017 12:44:57 +0300
Subject: [PATCH 115/446] CacheScanPartitionQueryFallbackSelfTest fixed to use
 default page size.

---
 .../cache/CacheScanPartitionQueryFallbackSelfTest.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java
index 02b213e4bee7e..efa9ce515a6f5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java
@@ -248,7 +248,7 @@ private void scanFallbackOnRebalancing(final boolean cur) throws Exception {
                                 info("Running query [node=" + nodeId + ", part=" + part + ']');
 
                             try (QueryCursor<Cache.Entry<Integer, Integer>> cur0 =
-                                     cache.query(new ScanQuery<Integer, Integer>(part).setPageSize(5))) {
+                                     cache.query(new ScanQuery<Integer, Integer>(part))) {
 
                                 if (cur)
                                     doTestScanQueryCursor(cur0, part);

From 71a76c81530d1aa38525fd78a90b065b6e19a6bb Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 19 Jan 2017 13:29:55 +0300
Subject: [PATCH 116/446] IGNITE-4550: Move service deployment to certain test.
 This closes #1437.

---
 .../GridCacheAbstractFullApiSelfTest.java     | 20 ++++++++-----------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
index b328960cd6a57..1cfb330da8cd6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
@@ -64,6 +64,7 @@
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -294,18 +295,6 @@ protected CacheMemoryMode memoryMode() {
             cacheCfgMap = null;
         }
 
-        // We won't deploy service unless non-client node is configured.
-        for (int i = 0; i < gridCount(); i++) {
-            Boolean clientMode = grid(i).configuration().isClientMode();
-
-            if (clientMode != null && clientMode) // Can be null in multi jvm tests.
-                continue;
-
-            grid(0).services(grid(0).cluster()).deployNodeSingleton(SERVICE_NAME1, new DummyServiceImpl());
-
-            break;
-        }
-
         for (int i = 0; i < gridCount(); i++)
             info("Grid " + i + ": " + grid(i).localNode().id());
     }
@@ -5521,6 +5510,13 @@ public void testLockInsideTransaction() throws Exception {
      * @throws Exception If failed.
      */
     public void testTransformResourceInjection() throws Exception {
+        ClusterGroup servers = grid(0).cluster().forServers();
+
+        if(F.isEmpty(servers.nodes()))
+            return;
+
+        grid(0).services( grid(0).cluster()).deployNodeSingleton(SERVICE_NAME1, new DummyServiceImpl());
+
         IgniteCache<String, Integer> cache = jcache();
         Ignite ignite = ignite(0);
 

From 476b089b1dd4b4c5d3b6ae21e1b3b2c010c086ac Mon Sep 17 00:00:00 2001
From: tledkov-gridgain <tledkov@gridgain.com>
Date: Fri, 20 Jan 2017 17:33:34 +0300
Subject: [PATCH 117/446] IGNITE-4507: Hadoop: added direct output support for
 combiner. This closes #1434.

---
 .../processors/hadoop/HadoopTaskContext.java  | 10 +++
 .../hadoop/impl/v1/HadoopV1MapTask.java       | 89 +++++++++++--------
 .../hadoop/impl/v1/HadoopV1ReduceTask.java    | 69 ++++++++------
 .../hadoop/impl/v2/HadoopV2Context.java       | 10 ---
 .../hadoop/impl/v2/HadoopV2MapTask.java       | 18 ++--
 .../hadoop/impl/v2/HadoopV2ReduceTask.java    | 14 +++
 .../hadoop/impl/v2/HadoopV2TaskContext.java   |  1 +
 .../hadoop/shuffle/HadoopShuffleJob.java      |  7 --
 .../shuffle/direct/HadoopDirectDataInput.java |  2 +-
 .../taskexecutor/HadoopRunnableTask.java      | 12 ++-
 .../impl/HadoopAbstractMapReduceTest.java     |  2 +
 .../impl/HadoopMapReduceEmbeddedSelfTest.java |  6 +-
 12 files changed, 145 insertions(+), 95 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
index dddd0176e32fe..d6e93946b59a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
@@ -207,4 +207,14 @@ public void output(HadoopTaskOutput out) {
      * @throws IgniteCheckedException On any error in callable.
      */
     public abstract <T> T runAsJobOwner(Callable<T> c) throws IgniteCheckedException;
+
+    /**
+     * Callback invoked from mapper thread when map is finished.
+     *
+     * @throws IgniteCheckedException If failed.
+     */
+    public void onMapperFinished() throws IgniteCheckedException {
+        if (output instanceof HadoopMapperAwareTaskOutput)
+            ((HadoopMapperAwareTaskOutput)output).onMapperFinished();
+    }
 }
\ No newline at end of file
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v1/HadoopV1MapTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v1/HadoopV1MapTask.java
index 65ff280df7659..2aa4292135c10 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v1/HadoopV1MapTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v1/HadoopV1MapTask.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
 import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
 import org.apache.ignite.internal.processors.hadoop.HadoopJob;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperUtils;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskCancelledException;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
@@ -45,7 +46,7 @@ public class HadoopV1MapTask extends HadoopV1Task {
     /**
      * Constructor.
      *
-     * @param taskInfo 
+     * @param taskInfo Taks info.
      */
     public HadoopV1MapTask(HadoopTaskInfo taskInfo) {
         super(taskInfo);
@@ -56,67 +57,79 @@ public HadoopV1MapTask(HadoopTaskInfo taskInfo) {
     @Override public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
         HadoopJob job = taskCtx.job();
 
-        HadoopV2TaskContext ctx = (HadoopV2TaskContext)taskCtx;
+        HadoopV2TaskContext taskCtx0 = (HadoopV2TaskContext)taskCtx;
 
-        JobConf jobConf = ctx.jobConf();
+        if (taskCtx.taskInfo().hasMapperIndex())
+            HadoopMapperUtils.mapperIndex(taskCtx.taskInfo().mapperIndex());
+        else
+            HadoopMapperUtils.clearMapperIndex();
 
-        InputFormat inFormat = jobConf.getInputFormat();
+        try {
+            JobConf jobConf = taskCtx0.jobConf();
 
-        HadoopInputSplit split = info().inputSplit();
+            InputFormat inFormat = jobConf.getInputFormat();
 
-        InputSplit nativeSplit;
+            HadoopInputSplit split = info().inputSplit();
 
-        if (split instanceof HadoopFileBlock) {
-            HadoopFileBlock block = (HadoopFileBlock)split;
+            InputSplit nativeSplit;
 
-            nativeSplit = new FileSplit(new Path(block.file().toString()), block.start(), block.length(), EMPTY_HOSTS);
-        }
-        else
-            nativeSplit = (InputSplit)ctx.getNativeSplit(split);
+            if (split instanceof HadoopFileBlock) {
+                HadoopFileBlock block = (HadoopFileBlock)split;
 
-        assert nativeSplit != null;
+                nativeSplit = new FileSplit(new Path(block.file().toString()), block.start(), block.length(), EMPTY_HOSTS);
+            }
+            else
+                nativeSplit = (InputSplit)taskCtx0.getNativeSplit(split);
 
-        Reporter reporter = new HadoopV1Reporter(taskCtx);
+            assert nativeSplit != null;
 
-        HadoopV1OutputCollector collector = null;
+            Reporter reporter = new HadoopV1Reporter(taskCtx);
 
-        try {
-            collector = collector(jobConf, ctx, !job.info().hasCombiner() && !job.info().hasReducer(),
-                fileName(), ctx.attemptId());
+            HadoopV1OutputCollector collector = null;
 
-            RecordReader reader = inFormat.getRecordReader(nativeSplit, jobConf, reporter);
+            try {
+                collector = collector(jobConf, taskCtx0, !job.info().hasCombiner() && !job.info().hasReducer(),
+                    fileName(), taskCtx0.attemptId());
 
-            Mapper mapper = ReflectionUtils.newInstance(jobConf.getMapperClass(), jobConf);
+                RecordReader reader = inFormat.getRecordReader(nativeSplit, jobConf, reporter);
 
-            Object key = reader.createKey();
-            Object val = reader.createValue();
+                Mapper mapper = ReflectionUtils.newInstance(jobConf.getMapperClass(), jobConf);
 
-            assert mapper != null;
+                Object key = reader.createKey();
+                Object val = reader.createValue();
+
+                assert mapper != null;
 
-            try {
                 try {
-                    while (reader.next(key, val)) {
-                        if (isCancelled())
-                            throw new HadoopTaskCancelledException("Map task cancelled.");
+                    try {
+                        while (reader.next(key, val)) {
+                            if (isCancelled())
+                                throw new HadoopTaskCancelledException("Map task cancelled.");
+
+                            mapper.map(key, val, collector, reporter);
+                        }
 
-                        mapper.map(key, val, collector, reporter);
+                        taskCtx.onMapperFinished();
+                    }
+                    finally {
+                        mapper.close();
                     }
                 }
                 finally {
-                    mapper.close();
+                    collector.closeWriter();
                 }
+
+                collector.commit();
             }
-            finally {
-                collector.closeWriter();
-            }
+            catch (Exception e) {
+                if (collector != null)
+                    collector.abort();
 
-            collector.commit();
+                throw new IgniteCheckedException(e);
+            }
         }
-        catch (Exception e) {
-            if (collector != null)
-                collector.abort();
-
-            throw new IgniteCheckedException(e);
+        finally {
+            HadoopMapperUtils.clearMapperIndex();
         }
     }
 }
\ No newline at end of file
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v1/HadoopV1ReduceTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v1/HadoopV1ReduceTask.java
index 92c024ec06924..5c1dd15de2d4d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v1/HadoopV1ReduceTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v1/HadoopV1ReduceTask.java
@@ -23,6 +23,7 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.processors.hadoop.HadoopJob;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperUtils;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskCancelledException;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
@@ -53,49 +54,63 @@ public HadoopV1ReduceTask(HadoopTaskInfo taskInfo, boolean reduce) {
     @Override public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
         HadoopJob job = taskCtx.job();
 
-        HadoopV2TaskContext ctx = (HadoopV2TaskContext)taskCtx;
+        HadoopV2TaskContext taskCtx0 = (HadoopV2TaskContext)taskCtx;
 
-        JobConf jobConf = ctx.jobConf();
-
-        HadoopTaskInput input = taskCtx.input();
-
-        HadoopV1OutputCollector collector = null;
+        if (!reduce && taskCtx.taskInfo().hasMapperIndex())
+            HadoopMapperUtils.mapperIndex(taskCtx.taskInfo().mapperIndex());
+        else
+            HadoopMapperUtils.clearMapperIndex();
 
         try {
-            collector = collector(jobConf, ctx, reduce || !job.info().hasReducer(), fileName(), ctx.attemptId());
+            JobConf jobConf = taskCtx0.jobConf();
 
-            Reducer reducer;
-            if (reduce) reducer = ReflectionUtils.newInstance(jobConf.getReducerClass(),
-                jobConf);
-            else reducer = ReflectionUtils.newInstance(jobConf.getCombinerClass(),
-                jobConf);
+            HadoopTaskInput input = taskCtx.input();
 
-            assert reducer != null;
+            HadoopV1OutputCollector collector = null;
 
             try {
+                collector = collector(jobConf, taskCtx0, reduce || !job.info().hasReducer(), fileName(), taskCtx0.attemptId());
+
+                Reducer reducer;
+                if (reduce) reducer = ReflectionUtils.newInstance(jobConf.getReducerClass(),
+                    jobConf);
+                else reducer = ReflectionUtils.newInstance(jobConf.getCombinerClass(),
+                    jobConf);
+
+                assert reducer != null;
+
                 try {
-                    while (input.next()) {
-                        if (isCancelled())
-                            throw new HadoopTaskCancelledException("Reduce task cancelled.");
+                    try {
+                        while (input.next()) {
+                            if (isCancelled())
+                                throw new HadoopTaskCancelledException("Reduce task cancelled.");
+
+                            reducer.reduce(input.key(), input.values(), collector, Reporter.NULL);
+                        }
 
-                        reducer.reduce(input.key(), input.values(), collector, Reporter.NULL);
+                        if (!reduce)
+                            taskCtx.onMapperFinished();
+                    }
+                    finally {
+                        reducer.close();
                     }
                 }
                 finally {
-                    reducer.close();
+                    collector.closeWriter();
                 }
+
+                collector.commit();
             }
-            finally {
-                collector.closeWriter();
-            }
+            catch (Exception e) {
+                if (collector != null)
+                    collector.abort();
 
-            collector.commit();
+                throw new IgniteCheckedException(e);
+            }
         }
-        catch (Exception e) {
-            if (collector != null)
-                collector.abort();
-
-            throw new IgniteCheckedException(e);
+        finally {
+            if (!reduce)
+                HadoopMapperUtils.clearMapperIndex();
         }
     }
 }
\ No newline at end of file
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java
index eec0636ce7853..1f4e67501ae56 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java
@@ -154,16 +154,6 @@ public HadoopV2Context(HadoopV2TaskContext ctx) {
         }
     }
 
-    /**
-     * Callback invoked from mapper thread when map is finished.
-     *
-     * @throws IgniteCheckedException If failed.
-     */
-    public void onMapperFinished() throws IgniteCheckedException {
-        if (output instanceof HadoopMapperAwareTaskOutput)
-            ((HadoopMapperAwareTaskOutput)output).onMapperFinished();
-    }
-
     /** {@inheritDoc} */
     @Override public OutputCommitter getOutputCommitter() {
         throw new UnsupportedOperationException();
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java
index eb3b935bd3d8b..1519199cce76b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java
@@ -56,30 +56,32 @@ public HadoopV2MapTask(HadoopTaskInfo taskInfo) {
             HadoopMapperUtils.clearMapperIndex();
 
         try {
-            InputSplit nativeSplit = hadoopContext().getInputSplit();
+            HadoopV2Context hadoopCtx = hadoopContext();
+
+            InputSplit nativeSplit = hadoopCtx.getInputSplit();
 
             if (nativeSplit == null)
                 throw new IgniteCheckedException("Input split cannot be null.");
 
             InputFormat inFormat = ReflectionUtils.newInstance(jobCtx.getInputFormatClass(),
-                hadoopContext().getConfiguration());
+                hadoopCtx.getConfiguration());
 
-            RecordReader reader = inFormat.createRecordReader(nativeSplit, hadoopContext());
+            RecordReader reader = inFormat.createRecordReader(nativeSplit, hadoopCtx);
 
-            reader.initialize(nativeSplit, hadoopContext());
+            reader.initialize(nativeSplit, hadoopCtx);
 
-            hadoopContext().reader(reader);
+            hadoopCtx.reader(reader);
 
             HadoopJobInfo jobInfo = taskCtx.job().info();
 
             outputFormat = jobInfo.hasCombiner() || jobInfo.hasReducer() ? null : prepareWriter(jobCtx);
 
-            Mapper mapper = ReflectionUtils.newInstance(jobCtx.getMapperClass(), hadoopContext().getConfiguration());
+            Mapper mapper = ReflectionUtils.newInstance(jobCtx.getMapperClass(), hadoopCtx.getConfiguration());
 
             try {
-                mapper.run(new WrappedMapper().getMapContext(hadoopContext()));
+                mapper.run(new WrappedMapper().getMapContext(hadoopCtx));
 
-                hadoopContext().onMapperFinished();
+                taskCtx.onMapperFinished();
             }
             finally {
                 closeWriter();
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2ReduceTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2ReduceTask.java
index 930ec1d49746e..09e063437d070 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2ReduceTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2ReduceTask.java
@@ -24,6 +24,7 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperUtils;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
 
 /**
@@ -53,10 +54,17 @@ public HadoopV2ReduceTask(HadoopTaskInfo taskInfo, boolean reduce) {
 
         JobContextImpl jobCtx = taskCtx.jobContext();
 
+        // Set mapper index for combiner tasks
+        if (!reduce && taskCtx.taskInfo().hasMapperIndex())
+            HadoopMapperUtils.mapperIndex(taskCtx.taskInfo().mapperIndex());
+        else
+            HadoopMapperUtils.clearMapperIndex();
+
         try {
             outputFormat = reduce || !taskCtx.job().info().hasReducer() ? prepareWriter(jobCtx) : null;
 
             Reducer reducer;
+
             if (reduce) reducer = ReflectionUtils.newInstance(jobCtx.getReducerClass(),
                 jobCtx.getConfiguration());
             else reducer = ReflectionUtils.newInstance(jobCtx.getCombinerClass(),
@@ -64,6 +72,9 @@ public HadoopV2ReduceTask(HadoopTaskInfo taskInfo, boolean reduce) {
 
             try {
                 reducer.run(new WrappedReducer().getReducerContext(hadoopContext()));
+
+                if (!reduce)
+                    taskCtx.onMapperFinished();
             }
             finally {
                 closeWriter();
@@ -84,6 +95,9 @@ public HadoopV2ReduceTask(HadoopTaskInfo taskInfo, boolean reduce) {
             throw new IgniteCheckedException(e);
         }
         finally {
+            if (!reduce)
+                HadoopMapperUtils.clearMapperIndex();
+
             if (err != null)
                 abort(outputFormat);
         }
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
index d328550a7a2f7..475e43db50eb0 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
@@ -49,6 +49,7 @@
 import org.apache.ignite.internal.processors.hadoop.HadoopJob;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperAwareTaskOutput;
 import org.apache.ignite.internal.processors.hadoop.HadoopPartitioner;
 import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
 import org.apache.ignite.internal.processors.hadoop.HadoopSplitWrapper;
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 318ead36f9376..4bcc398af763a 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -182,13 +182,6 @@ public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUn
         boolean stripeMappers0 = get(job.info(), SHUFFLE_MAPPER_STRIPED_OUTPUT, true);
 
         if (stripeMappers0) {
-            if (job.info().hasCombiner()) {
-                log.info("Striped mapper output is disabled because it cannot be used together with combiner [jobId=" +
-                    job.id() + ']');
-
-                stripeMappers0 = false;
-            }
-
             if (!embedded) {
                 log.info("Striped mapper output is disabled becuase it cannot be used in external mode [jobId=" +
                     job.id() + ']');
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java
index e3a713aa5fe8b..ef2905b72046a 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java
@@ -48,7 +48,7 @@ public HadoopDirectDataInput(byte[] buf) {
 
     /** {@inheritDoc} */
     @Override public int read() throws IOException {
-        return readByte();
+        return (int)readByte() & 0xFF;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopRunnableTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopRunnableTask.java
index a57efe6f5dcd9..339bf5bedf0c7 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopRunnableTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/HadoopRunnableTask.java
@@ -122,7 +122,7 @@ public long executionTime() {
 
     /**
      * Implements actual task running.
-     * @throws IgniteCheckedException
+     * @throws IgniteCheckedException On error.
      */
     void call0() throws IgniteCheckedException {
         execStartTs = U.currentTimeMillis();
@@ -144,7 +144,15 @@ void call0() throws IgniteCheckedException {
             runTask(perfCntr);
 
             if (info.type() == MAP && job.info().hasCombiner()) {
-                ctx.taskInfo(new HadoopTaskInfo(COMBINE, info.jobId(), info.taskNumber(), info.attempt(), null));
+                // Switch to combiner.
+                HadoopTaskInfo combineTaskInfo = new HadoopTaskInfo(COMBINE, info.jobId(), info.taskNumber(),
+                    info.attempt(), null);
+
+                // Mapper and combiner share the same index.
+                if (ctx.taskInfo().hasMapperIndex())
+                    combineTaskInfo.mapperIndex(ctx.taskInfo().mapperIndex());
+
+                ctx.taskInfo(combineTaskInfo);
 
                 try {
                     runTask(perfCntr);
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractMapReduceTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractMapReduceTest.java
index 89005f60b44e4..cd997a47f921c 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractMapReduceTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractMapReduceTest.java
@@ -172,6 +172,8 @@ private void checkOwner(IgfsPath p) {
      */
     protected final void doTest(IgfsPath inFile, boolean useNewMapper, boolean useNewCombiner, boolean useNewReducer)
         throws Exception {
+        log.info("useNewMapper=" + useNewMapper + ", useNewCombiner=" + useNewCombiner + ", useNewReducer=" + useNewReducer);
+
         igfs.delete(new IgfsPath(PATH_OUTPUT), true);
 
         JobConf jobConf = new JobConf();
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
index 8897a38148a68..bce67f67d7199 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
@@ -55,14 +55,14 @@ public class HadoopMapReduceEmbeddedSelfTest extends HadoopMapReduceTest {
         return cfg;
     }
 
-    /*
+    /**
      * @throws Exception If fails.
      */
     public void testMultiReducerWholeMapReduceExecution() throws Exception {
         checkMultiReducerWholeMapReduceExecution(false);
     }
 
-    /*
+    /**
      * @throws Exception If fails.
      */
     public void testMultiReducerWholeMapReduceExecutionStriped() throws Exception {
@@ -100,6 +100,8 @@ public void checkMultiReducerWholeMapReduceExecution(boolean striped) throws Exc
 
             if (striped)
                 jobConf.set(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), "true");
+            else
+                jobConf.set(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), "false");
 
             jobConf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, CustomSerialization.class.getName());
 

From db84b9b6e39ac37791b35479bccb8bbd75bb1676 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Mon, 23 Jan 2017 11:26:18 +0300
Subject: [PATCH 118/446] Merge ignite-4499.

---
 .../communication/tcp/TcpCommunicationSpiDropNodesTest.java  | 5 +++--
 .../tcp/TcpCommunicationSpiFaultyClientTest.java             | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
index d29231e449b0b..2b49d5319e63f 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
@@ -297,7 +297,8 @@ public boolean apply(Event event) {
      */
     private static class TestCommunicationSpi extends TcpCommunicationSpi {
         /** {@inheritDoc} */
-        @Override protected GridCommunicationClient createTcpClient(ClusterNode node) throws IgniteCheckedException {
+        @Override protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
+            throws IgniteCheckedException {
             if (pred.apply(getLocalNode(), node)) {
                 Map<String, Object> attrs = new HashMap<>(node.attributes());
 
@@ -309,7 +310,7 @@ private static class TestCommunicationSpi extends TcpCommunicationSpi {
                 ((TcpDiscoveryNode)node).setAttributes(attrs);
             }
 
-            return super.createTcpClient(node);
+            return super.createTcpClient(node, connIdx);
         }
 
         /**
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
index 6e99487bd46ae..f9756e9eb1539 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
@@ -245,7 +245,8 @@ public void stop() {
      */
     private static class TestCommunicationSpi extends TcpCommunicationSpi {
         /** {@inheritDoc} */
-        @Override protected GridCommunicationClient createTcpClient(ClusterNode node) throws IgniteCheckedException {
+        @Override protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
+            throws IgniteCheckedException {
             if (PRED.apply(node)) {
                 Map<String, Object> attrs = new HashMap<>(node.attributes());
 
@@ -257,7 +258,7 @@ private static class TestCommunicationSpi extends TcpCommunicationSpi {
                 ((TcpDiscoveryNode)node).setAttributes(attrs);
             }
 
-            return super.createTcpClient(node);
+            return super.createTcpClient(node, connIdx);
         }
 
         /**

From 7f0af43519f9e27f84ce8106d98b90aff00ee936 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Mon, 23 Jan 2017 14:14:22 +0300
Subject: [PATCH 119/446] Added benchmark.

---
 .../IgnitePutRandomValueSizeBenchmark.java    | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutRandomValueSizeBenchmark.java

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutRandomValueSizeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutRandomValueSizeBenchmark.java
new file mode 100644
index 0000000000000..be85b9fead0e9
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutRandomValueSizeBenchmark.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+
+/**
+ *
+ */
+public class IgnitePutRandomValueSizeBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        int key = nextRandom(args.range());
+        int size = 64 + nextRandom(64);
+
+        cache.put(key, new byte[size]);
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteCache<Integer, Object> cache() {
+        return ignite().cache("atomic-offheap");
+    }
+}

From e832ef9ce363fad34097aa78293a57f4aefcbcc0 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Tue, 24 Jan 2017 14:44:33 +0300
Subject: [PATCH 120/446] IGNITE-3699 (Backported from master)
 CreatedExpiryPolicy doesn't work if entry is loaded from store

---
 .../processors/cache/GridCacheAdapter.java    |  35 +++---
 .../processors/cache/GridCacheEntryEx.java    |   4 +-
 .../processors/cache/GridCacheMapEntry.java   |  17 ++-
 .../GridDistributedCacheAdapter.java          |   6 +-
 .../distributed/dht/GridDhtCacheAdapter.java  |   6 +-
 .../distributed/dht/GridDhtLockFuture.java    |  21 +++-
 .../dht/GridDhtTransactionalCacheAdapter.java |   7 ++
 .../dht/GridDhtTxLocalAdapter.java            |   8 +-
 .../dht/GridPartitionedGetFuture.java         |   1 +
 .../dht/GridPartitionedSingleGetFuture.java   |   2 +
 .../dht/atomic/GridDhtAtomicCache.java        |   3 +-
 .../dht/colocated/GridDhtColocatedCache.java  |  10 ++
 .../colocated/GridDhtColocatedLockFuture.java |  10 +-
 .../distributed/near/GridNearAtomicCache.java |   1 +
 .../distributed/near/GridNearGetFuture.java   |   1 +
 .../distributed/near/GridNearGetRequest.java  |  77 ++++++++-----
 .../distributed/near/GridNearLockFuture.java  |   7 ++
 .../distributed/near/GridNearLockRequest.java |  81 +++++++++-----
 .../near/GridNearSingleGetRequest.java        |  57 +++++++---
 .../near/GridNearTransactionalCache.java      |   2 +
 .../distributed/near/GridNearTxLocal.java     |  17 ++-
 .../cache/local/GridLocalCache.java           |   1 +
 .../local/atomic/GridLocalAtomicCache.java    |   1 +
 .../transactions/IgniteTxLocalAdapter.java    |  37 +++++--
 .../cache/transactions/IgniteTxLocalEx.java   |   3 +
 .../cache/GridCacheTestEntryEx.java           |   3 +-
 ...acheExpiryPolicyWithStoreAbstractTest.java | 102 ++++++++++++++++++
 ...gniteCacheTxExpiryPolicyWithStoreTest.java |  21 ++++
 28 files changed, 432 insertions(+), 109 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 59665bb58708f..dc8f0308a0099 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -528,6 +528,7 @@ public boolean isDht() {
      * @param retval Flag to return value.
      * @param isolation Transaction isolation.
      * @param invalidate Invalidate flag.
+     * @param createTtl TTL for create operation.
      * @param accessTtl TTL for read operation.
      * @return Locks future.
      */
@@ -539,6 +540,7 @@ public abstract IgniteInternalFuture<Boolean> txLockAsync(
         boolean retval,
         TransactionIsolation isolation,
         boolean invalidate,
+        long createTtl,
         long accessTtl);
 
     /**
@@ -1873,7 +1875,7 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
         @Nullable final UUID subjId,
         final String taskName,
         final boolean deserializeBinary,
-        @Nullable IgniteCacheExpiryPolicy expiry,
+        @Nullable final IgniteCacheExpiryPolicy expiry,
         final boolean skipVals,
         final boolean keepCacheObjects,
         boolean canRemap,
@@ -2027,7 +2029,8 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                                     cacheVal,
                                                     res.version(),
                                                     null,
-                                                    readerArgs);
+                                                    readerArgs,
+                                                    expiry);
 
                                                 if (log.isDebugEnabled())
                                                     log.debug("Set value loaded from store into entry [" +
@@ -5936,28 +5939,28 @@ protected abstract static class CacheExpiryPolicy implements IgniteCacheExpiryPo
         }
 
         /**
-         * @param ttl Access TTL.
+         * @param createTtl Create TTL.
+         * @param accessTtl Access TTL.
          * @return Access expire policy.
          */
-        @Nullable public static CacheExpiryPolicy forAccess(final long ttl) {
-            if (ttl == CU.TTL_NOT_CHANGED)
+        @Nullable public static CacheExpiryPolicy fromRemote(final long createTtl, final long accessTtl) {
+            if (createTtl == CU.TTL_NOT_CHANGED && accessTtl == CU.TTL_NOT_CHANGED)
                 return null;
 
             return new CacheExpiryPolicy() {
-                @Override public long forAccess() {
-                    return ttl;
+                @Override public long forCreate() {
+                    return createTtl;
                 }
-            };
-        }
 
-        /** {@inheritDoc} */
-        @Override public long forCreate() {
-            return CU.TTL_NOT_CHANGED;
-        }
+                @Override public long forAccess() {
+                    return accessTtl;
+                }
 
-        /** {@inheritDoc} */
-        @Override public long forUpdate() {
-            return CU.TTL_NOT_CHANGED;
+                /** {@inheritDoc} */
+                @Override public long forUpdate() {
+                    return CU.TTL_NOT_CHANGED;
+                }
+            };
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index 51f423a7302a0..3c42d5360a4b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -756,6 +756,7 @@ public <K, V> GridCacheVersionedEntryEx<K, V> versionedEntry(final boolean keepB
      * @param curVer Version to match or {@code null} if match is not required.
      * @param newVer Version to set.
      * @param readerArgs Reader will be added if not null.
+     * @param loadExpiryPlc Expiry policy if entry is loaded from store.
      * @return Current version and value.
      * @throws IgniteCheckedException If index could not be updated.
      * @throws GridCacheEntryRemovedException If entry was removed.
@@ -763,7 +764,8 @@ public <K, V> GridCacheVersionedEntryEx<K, V> versionedEntry(final boolean keepB
     public T2<CacheObject, GridCacheVersion> versionedValue(CacheObject val,
         @Nullable GridCacheVersion curVer,
         @Nullable GridCacheVersion newVer,
-        @Nullable ReaderArguments readerArgs)
+        @Nullable ReaderArguments readerArgs,
+        @Nullable IgniteCacheExpiryPolicy loadExpiryPlc)
         throws IgniteCheckedException, GridCacheEntryRemovedException;
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 59e4181df0cee..7e26719d99076 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -3624,7 +3624,8 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
     @Override public synchronized T2<CacheObject, GridCacheVersion> versionedValue(CacheObject val,
         GridCacheVersion curVer,
         GridCacheVersion newVer,
-        @Nullable ReaderArguments readerArgs)
+        @Nullable ReaderArguments readerArgs,
+        @Nullable IgniteCacheExpiryPolicy loadExpiryPlc)
         throws IgniteCheckedException, GridCacheEntryRemovedException
     {
         checkObsolete();
@@ -3643,9 +3644,19 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
 
                 CacheObject old = rawGetOrUnmarshalUnlocked(false);
 
-                long ttl = ttlExtras();
+                long ttl;
+                long expTime;
 
-                long expTime = CU.toExpireTime(ttl);
+                if (loadExpiryPlc != null) {
+                    IgniteBiTuple<Long, Long> initTtlAndExpireTime = initialTtlAndExpireTime(loadExpiryPlc);
+
+                    ttl = initTtlAndExpireTime.get1();
+                    expTime = initTtlAndExpireTime.get2();
+                }
+                else {
+                    ttl = ttlExtras();
+                    expTime = expireTimeExtras();
+                }
 
                 // Detach value before index update.
                 val = cctx.kernalContext().cacheObjects().prepareForCache(val, cctx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheAdapter.java
index 03f647432a7c9..d89a468404371 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheAdapter.java
@@ -102,11 +102,12 @@ protected GridDistributedCacheAdapter(GridCacheContext<K, V> ctx, GridCacheConcu
         boolean retval,
         TransactionIsolation isolation,
         boolean isInvalidate,
+        long createTtl,
         long accessTtl
     ) {
         assert tx != null;
 
-        return lockAllAsync(keys, timeout, tx, isInvalidate, isRead, retval, isolation, accessTtl);
+        return lockAllAsync(keys, timeout, tx, isInvalidate, isRead, retval, isolation, createTtl, accessTtl);
     }
 
     /** {@inheritDoc} */
@@ -121,6 +122,7 @@ protected GridDistributedCacheAdapter(GridCacheContext<K, V> ctx, GridCacheConcu
             false,
             /*retval*/true,
             null,
+            -1L,
             -1L);
     }
 
@@ -132,6 +134,7 @@ protected GridDistributedCacheAdapter(GridCacheContext<K, V> ctx, GridCacheConcu
      * @param isRead Indicates whether value is read or written.
      * @param retval Flag to return value.
      * @param isolation Transaction isolation.
+     * @param createTtl TTL for create operation.
      * @param accessTtl TTL for read operation.
      * @return Future for locks.
      */
@@ -142,6 +145,7 @@ protected abstract IgniteInternalFuture<Boolean> lockAllAsync(Collection<KeyCach
         boolean isRead,
         boolean retval,
         @Nullable TransactionIsolation isolation,
+        long createTtl,
         long accessTtl);
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 543cee1809070..bc34df781c0a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -760,7 +760,7 @@ private IgniteInternalFuture<GridCacheEntryInfo> getDhtSingleAsync(
     protected void processNearSingleGetRequest(final UUID nodeId, final GridNearSingleGetRequest req) {
         assert ctx.affinityNode();
 
-        final CacheExpiryPolicy expiryPlc = CacheExpiryPolicy.forAccess(req.accessTtl());
+        final CacheExpiryPolicy expiryPlc = CacheExpiryPolicy.fromRemote(req.createTtl(), req.accessTtl());
 
         IgniteInternalFuture<GridCacheEntryInfo> fut =
             getDhtSingleAsync(
@@ -860,9 +860,7 @@ protected void processNearGetRequest(final UUID nodeId, final GridNearGetRequest
         assert ctx.affinityNode();
         assert !req.reload() : req;
 
-        long ttl = req.accessTtl();
-
-        final CacheExpiryPolicy expiryPlc = CacheExpiryPolicy.forAccess(ttl);
+        final CacheExpiryPolicy expiryPlc = CacheExpiryPolicy.fromRemote(req.createTtl(), req.accessTtl());
 
         IgniteInternalFuture<Collection<GridCacheEntryInfo>> fut =
             getDhtAsync(nodeId,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
index d77933e9f5d47..686a4c6bed872 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
@@ -156,6 +156,9 @@ public final class GridDhtLockFuture extends GridCompoundIdentityFuture<Boolean>
     /** Pending locks. */
     private final Collection<KeyCacheObject> pendingLocks;
 
+    /** TTL for create operation. */
+    private long createTtl;
+
     /** TTL for read operation. */
     private long accessTtl;
 
@@ -194,6 +197,7 @@ public GridDhtLockFuture(
         long timeout,
         GridDhtTxLocalAdapter tx,
         long threadId,
+        long createTtl,
         long accessTtl,
         CacheEntryPredicate[] filter,
         boolean skipStore,
@@ -214,6 +218,7 @@ public GridDhtLockFuture(
         this.timeout = timeout;
         this.filter = filter;
         this.tx = tx;
+        this.createTtl = createTtl;
         this.accessTtl = accessTtl;
         this.skipStore = skipStore;
         this.keepBinary = keepBinary;
@@ -1059,10 +1064,22 @@ private void loadMissingFromStore() {
                             try {
                                 CacheObject val0 = cctx.toCacheObject(val);
 
+                                long ttl = createTtl;
+                                long expireTime;
+
+                                if (ttl == CU.TTL_ZERO)
+                                    expireTime = CU.expireTimeInPast();
+                                else {
+                                    if (ttl == CU.TTL_NOT_CHANGED)
+                                        ttl = CU.TTL_ETERNAL;
+
+                                    expireTime = CU.toExpireTime(ttl);
+                                }
+
                                 entry0.initialValue(val0,
                                     ver,
-                                    0,
-                                    0,
+                                    ttl,
+                                    expireTime,
                                     false,
                                     topVer,
                                     GridDrType.DR_LOAD,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index 01bc4e028955f..a9e3bc496721c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -677,6 +677,7 @@ else if (txLockMsgLog.isDebugEnabled()) {
         boolean isRead,
         boolean retval,
         TransactionIsolation isolation,
+        long createTtl,
         long accessTtl) {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
@@ -688,6 +689,7 @@ else if (txLockMsgLog.isDebugEnabled()) {
             isRead,
             retval,
             isolation,
+            createTtl,
             accessTtl,
             CU.empty0(),
             opCtx != null && opCtx.skipStore(),
@@ -704,6 +706,7 @@ else if (txLockMsgLog.isDebugEnabled()) {
      * @param isRead Read flag.
      * @param retval Return value flag.
      * @param isolation Transaction isolation.
+     * @param createTtl TTL for create operation.
      * @param accessTtl TTL for read operation.
      * @param filter Optional filter.
      * @param skipStore Skip store flag.
@@ -716,6 +719,7 @@ public GridDhtFuture<Boolean> lockAllAsyncInternal(@Nullable Collection<KeyCache
         boolean isRead,
         boolean retval,
         TransactionIsolation isolation,
+        long createTtl,
         long accessTtl,
         CacheEntryPredicate[] filter,
         boolean skipStore,
@@ -738,6 +742,7 @@ public GridDhtFuture<Boolean> lockAllAsyncInternal(@Nullable Collection<KeyCache
             timeout,
             tx,
             tx.threadId(),
+            createTtl,
             accessTtl,
             filter,
             skipStore,
@@ -859,6 +864,7 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                         req.timeout(),
                         tx,
                         req.threadId(),
+                        req.createTtl(),
                         req.accessTtl(),
                         filter,
                         req.skipStore(),
@@ -1007,6 +1013,7 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                     req.messageId(),
                     req.txRead(),
                     req.needReturnValue(),
+                    req.createTtl(),
                     req.accessTtl(),
                     req.skipStore(),
                     req.keepBinary());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
index 35dfb62c5fc98..12a45d8fdca25 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
@@ -148,7 +148,7 @@ protected GridDhtTxLocalAdapter(
             storeEnabled,
             onePhaseCommit,
             txSize,
-            subjId, 
+            subjId,
             taskNameHash
         );
 
@@ -534,6 +534,7 @@ private void addMapping(
      * @param entries Entries to lock.
      * @param msgId Message ID.
      * @param read Read flag.
+     * @param createTtl TTL for create operation.
      * @param accessTtl TTL for read operation.
      * @param needRetVal Return value flag.
      * @param skipStore Skip store flag.
@@ -546,6 +547,7 @@ IgniteInternalFuture<GridCacheReturn> lockAllAsync(
         long msgId,
         final boolean read,
         final boolean needRetVal,
+        long createTtl,
         long accessTtl,
         boolean skipStore,
         boolean keepBinary
@@ -652,6 +654,7 @@ IgniteInternalFuture<GridCacheReturn> lockAllAsync(
                 passedKeys,
                 read,
                 needRetVal,
+                createTtl,
                 accessTtl,
                 null,
                 skipStore,
@@ -670,6 +673,7 @@ IgniteInternalFuture<GridCacheReturn> lockAllAsync(
      * @param passedKeys Passed keys.
      * @param read {@code True} if read.
      * @param needRetVal Return value flag.
+     * @param createTtl TTL for create operation.
      * @param accessTtl TTL for read operation.
      * @param filter Entry write filter.
      * @param skipStore Skip store flag.
@@ -681,6 +685,7 @@ private IgniteInternalFuture<GridCacheReturn> obtainLockAsync(
         final Collection<KeyCacheObject> passedKeys,
         final boolean read,
         final boolean needRetVal,
+        final long createTtl,
         final long accessTtl,
         @Nullable final CacheEntryPredicate[] filter,
         boolean skipStore,
@@ -706,6 +711,7 @@ private IgniteInternalFuture<GridCacheReturn> obtainLockAsync(
             read,
             needRetVal,
             isolation,
+            createTtl,
             accessTtl,
             CU.empty0(),
             skipStore,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index c8e2cf328bcd3..5892b374951f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -341,6 +341,7 @@ private void map(
                     topVer,
                     subjId,
                     taskName == null ? 0 : taskName.hashCode(),
+                    expiryPlc != null ? expiryPlc.forCreate() : -1L,
                     expiryPlc != null ? expiryPlc.forAccess() : -1L,
                     skipVals,
                     cctx.deploymentEnabled());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index e369bfa83a8be..7c14f35b34cf4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -281,6 +281,7 @@ private void map(AffinityTopologyVersion topVer) {
                     topVer,
                     subjId,
                     taskName == null ? 0 : taskName.hashCode(),
+                    expiryPlc != null ? expiryPlc.forCreate() : -1L,
                     expiryPlc != null ? expiryPlc.forAccess() : -1L,
                     skipVals,
                     /**add reader*/false,
@@ -300,6 +301,7 @@ private void map(AffinityTopologyVersion topVer) {
                     topVer,
                     subjId,
                     taskName == null ? 0 : taskName.hashCode(),
+                    expiryPlc != null ? expiryPlc.forCreate() : -1L,
                     expiryPlc != null ? expiryPlc.forAccess() : -1L,
                     skipVals,
                     cctx.deploymentEnabled());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index f601e0a9ddb7e..2f97bccf88a71 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -863,6 +863,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
         boolean isRead,
         boolean retval,
         @Nullable TransactionIsolation isolation,
+        long createTtl,
         long accessTtl) {
         return new FinishedLockFuture(new UnsupportedOperationException("Locks are not supported for " +
             "CacheAtomicityMode.ATOMIC mode (use CacheAtomicityMode.TRANSACTIONAL instead)"));
@@ -2293,7 +2294,7 @@ private void reloadIfNeeded(final List<GridDhtCacheEntry> entries) throws Ignite
                         try {
                             GridCacheVersion ver = entry.version();
 
-                            entry.versionedValue(ctx.toCacheObject(v), null, ver, null);
+                            entry.versionedValue(ctx.toCacheObject(v), null, ver, null, null);
                         }
                         catch (GridCacheEntryRemovedException e) {
                             assert false : "Entry should not get obsolete while holding lock [entry=" + entry +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 29f0607e5c33c..5ed30dbcadcba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -615,6 +615,7 @@ else if (!skipVals && ctx.config().isStatisticsEnabled())
         boolean isRead,
         boolean retval,
         @Nullable TransactionIsolation isolation,
+        long createTtl,
         long accessTtl
     ) {
         assert tx == null || tx instanceof GridNearTxLocal : tx;
@@ -629,6 +630,7 @@ else if (!skipVals && ctx.config().isStatisticsEnabled())
             isRead,
             retval,
             timeout,
+            createTtl,
             accessTtl,
             CU.empty0(),
             opCtx != null && opCtx.skipStore(),
@@ -876,6 +878,7 @@ public void removeLocks(long threadId, GridCacheVersion ver, Collection<KeyCache
      * @param txRead Tx read.
      * @param retval Return value flag.
      * @param timeout Lock timeout.
+     * @param createTtl TTL for create operation.
      * @param accessTtl TTL for read operation.
      * @param filter filter Optional filter.
      * @param skipStore Skip store flag.
@@ -891,6 +894,7 @@ IgniteInternalFuture<Exception> lockAllAsync(
         final boolean txRead,
         final boolean retval,
         final long timeout,
+        final long createTtl,
         final long accessTtl,
         @Nullable final CacheEntryPredicate[] filter,
         final boolean skipStore,
@@ -915,6 +919,7 @@ IgniteInternalFuture<Exception> lockAllAsync(
                 txRead,
                 retval,
                 timeout,
+                createTtl,
                 accessTtl,
                 filter,
                 skipStore,
@@ -936,6 +941,7 @@ IgniteInternalFuture<Exception> lockAllAsync(
                             txRead,
                             retval,
                             timeout,
+                            createTtl,
                             accessTtl,
                             filter,
                             skipStore,
@@ -956,6 +962,7 @@ IgniteInternalFuture<Exception> lockAllAsync(
      * @param txRead Tx read.
      * @param retval Return value flag.
      * @param timeout Lock timeout.
+     * @param createTtl TTL for create operation.
      * @param accessTtl TTL for read operation.
      * @param filter filter Optional filter.
      * @param skipStore Skip store flag.
@@ -971,6 +978,7 @@ private IgniteInternalFuture<Exception> lockAllAsync0(
         final boolean txRead,
         boolean retval,
         final long timeout,
+        final long createTtl,
         final long accessTtl,
         @Nullable final CacheEntryPredicate[] filter,
         boolean skipStore,
@@ -988,6 +996,7 @@ private IgniteInternalFuture<Exception> lockAllAsync0(
                 timeout,
                 tx,
                 threadId,
+                createTtl,
                 accessTtl,
                 filter,
                 skipStore,
@@ -1056,6 +1065,7 @@ else if (!b)
                 keys,
                 retval,
                 txRead,
+                createTtl,
                 accessTtl,
                 skipStore,
                 keepBinary);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index 5557d3423e201..40e87ee2cfd7c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -145,6 +145,9 @@ public final class GridDhtColocatedLockFuture extends GridCompoundIdentityFuture
     /** Trackable flag (here may be non-volatile). */
     private boolean trackable;
 
+    /** TTL for create operation. */
+    private final long createTtl;
+
     /** TTL for read operation. */
     private final long accessTtl;
 
@@ -164,6 +167,7 @@ public final class GridDhtColocatedLockFuture extends GridCompoundIdentityFuture
      * @param read Read flag.
      * @param retval Flag to return value or not.
      * @param timeout Lock acquisition timeout.
+     * @param createTtl TTL for create operation.
      * @param accessTtl TTL for read operation.
      * @param filter Filter.
      * @param skipStore Skip store flag.
@@ -175,6 +179,7 @@ public GridDhtColocatedLockFuture(
         boolean read,
         boolean retval,
         long timeout,
+        long createTtl,
         long accessTtl,
         CacheEntryPredicate[] filter,
         boolean skipStore,
@@ -189,6 +194,7 @@ public GridDhtColocatedLockFuture(
         this.read = read;
         this.retval = retval;
         this.timeout = timeout;
+        this.createTtl = createTtl;
         this.accessTtl = accessTtl;
         this.filter = filter;
         this.skipStore = skipStore;
@@ -926,6 +932,7 @@ private synchronized void map0(
                                         inTx() && tx.syncMode() == FULL_SYNC,
                                         inTx() ? tx.subjectId() : null,
                                         inTx() ? tx.taskNameHash() : 0,
+                                        read ? createTtl : -1L,
                                         read ? accessTtl : -1L,
                                         skipStore,
                                         keepBinary,
@@ -1102,7 +1109,7 @@ private void proceedMapping0()
 
     /**
      * Locks given keys directly through dht cache.
-     *  @param keys Collection of keys.
+     * @param keys Collection of keys.
      * @param topVer Topology version to lock on.
      */
     private void lockLocally(
@@ -1121,6 +1128,7 @@ private void lockLocally(
             read,
             retval,
             timeout,
+            createTtl,
             accessTtl,
             filter,
             skipStore,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index a8219b08cefdc..d1056fdff4284 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -641,6 +641,7 @@ public void processDhtAtomicUpdateRequest(
         boolean isRead,
         boolean retval,
         @Nullable TransactionIsolation isolation,
+        long createTtl,
         long accessTtl) {
         return dht.lockAllAsync(null, timeout);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index 8bc513e4dfda2..8c64e3e9603bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -375,6 +375,7 @@ private void map(
                     topVer,
                     subjId,
                     taskName == null ? 0 : taskName.hashCode(),
+                    expiryPlc != null ? expiryPlc.forCreate() : -1L,
                     expiryPlc != null ? expiryPlc.forAccess() : -1L,
                     skipVals,
                     cctx.deploymentEnabled());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
index fa7f367cc4460..e02658cecb7c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
@@ -99,6 +99,9 @@ public class GridNearGetRequest extends GridCacheMessage implements GridCacheDep
     /** Task name hash. */
     private int taskNameHash;
 
+    /** TTL for read operation. */
+    private long createTtl;
+
     /** TTL for read operation. */
     private long accessTtl;
 
@@ -121,6 +124,7 @@ public GridNearGetRequest() {
      * @param topVer Topology version.
      * @param subjId Subject ID.
      * @param taskNameHash Task name hash.
+     * @param createTtl New TTL to set after entry is created, -1 to leave unchanged.
      * @param accessTtl New TTL to set after entry is accessed, -1 to leave unchanged.
      * @param addDepInfo Deployment info.
      */
@@ -134,6 +138,7 @@ public GridNearGetRequest(
         @NotNull AffinityTopologyVersion topVer,
         UUID subjId,
         int taskNameHash,
+        long createTtl,
         long accessTtl,
         boolean skipVals,
         boolean addDepInfo
@@ -161,6 +166,7 @@ public GridNearGetRequest(
         this.topVer = topVer;
         this.subjId = subjId;
         this.taskNameHash = taskNameHash;
+        this.createTtl = createTtl;
         this.accessTtl = accessTtl;
         this.skipVals = skipVals;
         this.addDepInfo = addDepInfo;
@@ -237,6 +243,13 @@ public boolean skipValues() {
         return topVer;
     }
 
+    /**
+     * @return New TTL to set after entry is created, -1 to leave unchanged.
+     */
+    public long createTtl() {
+        return createTtl;
+    }
+
     /**
      * @return New TTL to set after entry is accessed, -1 to leave unchanged.
      */
@@ -320,73 +333,79 @@ public long accessTtl() {
                 writer.incrementState();
 
             case 4:
-                if (!writer.writeCollection("flags", flags, MessageCollectionItemType.BOOLEAN))
+                if (!writer.writeLong("createTtl", createTtl))
                     return false;
 
                 writer.incrementState();
 
             case 5:
-                if (!writer.writeIgniteUuid("futId", futId))
+                if (!writer.writeCollection("flags", flags, MessageCollectionItemType.BOOLEAN))
                     return false;
 
                 writer.incrementState();
 
             case 6:
-                if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG))
+                if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
             case 7:
-                if (!writer.writeIgniteUuid("miniId", miniId))
+                if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
             case 8:
-                if (!writer.writeBoolean("readThrough", readThrough))
+                if (!writer.writeIgniteUuid("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
             case 9:
-                if (!writer.writeBoolean("reload", reload))
+                if (!writer.writeCollection("partIds", partIds, MessageCollectionItemType.INT))
                     return false;
 
                 writer.incrementState();
 
             case 10:
-                if (!writer.writeBoolean("skipVals", skipVals))
+                if (!writer.writeBoolean("readThrough", readThrough))
                     return false;
 
                 writer.incrementState();
 
             case 11:
-                if (!writer.writeUuid("subjId", subjId))
+                if (!writer.writeBoolean("reload", reload))
                     return false;
 
                 writer.incrementState();
 
             case 12:
-                if (!writer.writeInt("taskNameHash", taskNameHash))
+                if (!writer.writeBoolean("skipVals", skipVals))
                     return false;
 
                 writer.incrementState();
 
             case 13:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
             case 14:
-                if (!writer.writeMessage("ver", ver))
+                if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
             case 15:
-                if (!writer.writeCollection("partIds", partIds, MessageCollectionItemType.INT))
+                if (!writer.writeMessage("topVer", topVer))
+                    return false;
+
+                writer.incrementState();
+
+            case 16:
+                if (!writer.writeMessage("ver", ver))
                     return false;
 
                 writer.incrementState();
@@ -416,7 +435,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 4:
-                flags = reader.readCollection("flags", MessageCollectionItemType.BOOLEAN);
+                createTtl = reader.readLong("createTtl");
 
                 if (!reader.isLastRead())
                     return false;
@@ -424,7 +443,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 5:
-                futId = reader.readIgniteUuid("futId");
+                flags = reader.readCollection("flags", MessageCollectionItemType.BOOLEAN);
 
                 if (!reader.isLastRead())
                     return false;
@@ -432,7 +451,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 6:
-                keys = reader.readCollection("keys", MessageCollectionItemType.MSG);
+                futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
                     return false;
@@ -440,7 +459,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 7:
-                miniId = reader.readIgniteUuid("miniId");
+                keys = reader.readCollection("keys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
                     return false;
@@ -448,7 +467,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 8:
-                readThrough = reader.readBoolean("readThrough");
+                miniId = reader.readIgniteUuid("miniId");
 
                 if (!reader.isLastRead())
                     return false;
@@ -456,7 +475,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 9:
-                reload = reader.readBoolean("reload");
+                partIds = reader.readCollection("partIds", MessageCollectionItemType.INT);
 
                 if (!reader.isLastRead())
                     return false;
@@ -464,7 +483,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 10:
-                skipVals = reader.readBoolean("skipVals");
+                readThrough = reader.readBoolean("readThrough");
 
                 if (!reader.isLastRead())
                     return false;
@@ -472,7 +491,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 11:
-                subjId = reader.readUuid("subjId");
+                reload = reader.readBoolean("reload");
 
                 if (!reader.isLastRead())
                     return false;
@@ -480,7 +499,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 12:
-                taskNameHash = reader.readInt("taskNameHash");
+                skipVals = reader.readBoolean("skipVals");
 
                 if (!reader.isLastRead())
                     return false;
@@ -488,7 +507,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 13:
-                topVer = reader.readMessage("topVer");
+                subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
                     return false;
@@ -496,7 +515,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 14:
-                ver = reader.readMessage("ver");
+                taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
                     return false;
@@ -504,7 +523,15 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 15:
-                partIds = reader.readCollection("partIds", MessageCollectionItemType.INT);
+                topVer = reader.readMessage("topVer");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 16:
+                ver = reader.readMessage("ver");
 
                 if (!reader.isLastRead())
                     return false;
@@ -523,7 +550,7 @@ public long accessTtl() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 16;
+        return 17;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index 7c9860290eea2..491b4ece0ea13 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -148,6 +148,9 @@ public final class GridNearLockFuture extends GridCompoundIdentityFuture<Boolean
     @GridToStringExclude
     private List<GridDistributedCacheEntry> entries;
 
+    /** TTL for create operation. */
+    private long createTtl;
+
     /** TTL for read operation. */
     private long accessTtl;
 
@@ -168,6 +171,7 @@ public final class GridNearLockFuture extends GridCompoundIdentityFuture<Boolean
      * @param read Read flag.
      * @param retval Flag to return value or not.
      * @param timeout Lock acquisition timeout.
+     * @param createTtl TTL for create operation.
      * @param accessTtl TTL for read operation.
      * @param filter Filter.
      * @param skipStore skipStore
@@ -180,6 +184,7 @@ public GridNearLockFuture(
         boolean read,
         boolean retval,
         long timeout,
+        long createTtl,
         long accessTtl,
         CacheEntryPredicate[] filter,
         boolean skipStore,
@@ -195,6 +200,7 @@ public GridNearLockFuture(
         this.read = read;
         this.retval = retval;
         this.timeout = timeout;
+        this.createTtl = createTtl;
         this.accessTtl = accessTtl;
         this.filter = filter;
         this.skipStore = skipStore;
@@ -1054,6 +1060,7 @@ private void map(Iterable<KeyCacheObject> keys, boolean remap, boolean topLocked
                                                 inTx() && tx.syncMode() == FULL_SYNC,
                                                 inTx() ? tx.subjectId() : null,
                                                 inTx() ? tx.taskNameHash() : 0,
+                                                read ? createTtl : -1L,
                                                 read ? accessTtl : -1L,
                                                 skipStore,
                                                 keepBinary,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java
index 2e8cd6e852759..9e12153ea10f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java
@@ -80,6 +80,9 @@ public class GridNearLockRequest extends GridDistributedLockRequest {
     /** Sync commit flag. */
     private boolean syncCommit;
 
+    /** TTL for create operation. */
+    private long createTtl;
+
     /** TTL for read operation. */
     private long accessTtl;
 
@@ -116,6 +119,7 @@ public GridNearLockRequest() {
      * @param syncCommit Synchronous commit flag.
      * @param subjId Subject ID.
      * @param taskNameHash Task name hash code.
+     * @param createTtl TTL for create operation.
      * @param accessTtl TTL for read operation.
      * @param skipStore Skip store flag.
      * @param firstClientReq {@code True} if first lock request for lock operation sent from client node.
@@ -141,6 +145,7 @@ public GridNearLockRequest(
         boolean syncCommit,
         @Nullable UUID subjId,
         int taskNameHash,
+        long createTtl,
         long accessTtl,
         boolean skipStore,
         boolean keepBinary,
@@ -174,6 +179,7 @@ public GridNearLockRequest(
         this.syncCommit = syncCommit;
         this.subjId = subjId;
         this.taskNameHash = taskNameHash;
+        this.createTtl = createTtl;
         this.accessTtl = accessTtl;
         this.retVal = retVal;
         this.firstClientReq = firstClientReq;
@@ -311,6 +317,13 @@ public GridCacheVersion dhtVersion(int idx) {
         return dhtVers[idx];
     }
 
+    /**
+     * @return New TTL to set after entry is created, -1 to leave unchanged.
+     */
+    public long createTtl() {
+        return createTtl;
+    }
+
     /**
      * @return TTL for read operation.
      */
@@ -368,84 +381,90 @@ public long accessTtl() {
                 writer.incrementState();
 
             case 21:
-                if (!writer.writeObjectArray("dhtVers", dhtVers, MessageCollectionItemType.MSG))
+                if (!writer.writeLong("createTtl", createTtl))
                     return false;
 
                 writer.incrementState();
 
             case 22:
-                if (!writer.writeObjectArray("filter", filter, MessageCollectionItemType.MSG))
+                if (!writer.writeObjectArray("dhtVers", dhtVers, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
             case 23:
-                if (!writer.writeBoolean("firstClientReq", firstClientReq))
+                if (!writer.writeObjectArray("filter", filter, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
             case 24:
-                if (!writer.writeBoolean("hasTransforms", hasTransforms))
+                if (!writer.writeBoolean("firstClientReq", firstClientReq))
                     return false;
 
                 writer.incrementState();
 
             case 25:
-                if (!writer.writeBoolean("implicitSingleTx", implicitSingleTx))
+                if (!writer.writeBoolean("hasTransforms", hasTransforms))
                     return false;
 
                 writer.incrementState();
 
             case 26:
-                if (!writer.writeBoolean("implicitTx", implicitTx))
+                if (!writer.writeBoolean("implicitSingleTx", implicitSingleTx))
                     return false;
 
                 writer.incrementState();
 
             case 27:
-                if (!writer.writeIgniteUuid("miniId", miniId))
+                if (!writer.writeBoolean("implicitTx", implicitTx))
                     return false;
 
                 writer.incrementState();
 
             case 28:
-                if (!writer.writeBoolean("onePhaseCommit", onePhaseCommit))
+                if (!writer.writeIgniteUuid("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
             case 29:
-                if (!writer.writeBoolean("retVal", retVal))
+                if (!writer.writeBoolean("onePhaseCommit", onePhaseCommit))
                     return false;
 
                 writer.incrementState();
 
             case 30:
-                if (!writer.writeUuid("subjId", subjId))
+                if (!writer.writeBoolean("retVal", retVal))
                     return false;
 
                 writer.incrementState();
 
             case 31:
-                if (!writer.writeBoolean("syncCommit", syncCommit))
+                if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
             case 32:
-                if (!writer.writeInt("taskNameHash", taskNameHash))
+                if (!writer.writeBoolean("syncCommit", syncCommit))
                     return false;
 
                 writer.incrementState();
 
             case 33:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
             case 34:
+                if (!writer.writeMessage("topVer", topVer))
+                    return false;
+
+                writer.incrementState();
+
+            case 35:
                 if (!writer.writeCollection("partIds", partIds, MessageCollectionItemType.INT))
                     return false;
 
@@ -476,7 +495,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 21:
-                dhtVers = reader.readObjectArray("dhtVers", MessageCollectionItemType.MSG, GridCacheVersion.class);
+                createTtl = reader.readLong("createTtl");
 
                 if (!reader.isLastRead())
                     return false;
@@ -484,7 +503,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 22:
-                filter = reader.readObjectArray("filter", MessageCollectionItemType.MSG, CacheEntryPredicate.class);
+                dhtVers = reader.readObjectArray("dhtVers", MessageCollectionItemType.MSG, GridCacheVersion.class);
 
                 if (!reader.isLastRead())
                     return false;
@@ -492,7 +511,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 23:
-                firstClientReq = reader.readBoolean("firstClientReq");
+                filter = reader.readObjectArray("filter", MessageCollectionItemType.MSG, CacheEntryPredicate.class);
 
                 if (!reader.isLastRead())
                     return false;
@@ -500,7 +519,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 24:
-                hasTransforms = reader.readBoolean("hasTransforms");
+                firstClientReq = reader.readBoolean("firstClientReq");
 
                 if (!reader.isLastRead())
                     return false;
@@ -508,7 +527,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 25:
-                implicitSingleTx = reader.readBoolean("implicitSingleTx");
+                hasTransforms = reader.readBoolean("hasTransforms");
 
                 if (!reader.isLastRead())
                     return false;
@@ -516,7 +535,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 26:
-                implicitTx = reader.readBoolean("implicitTx");
+                implicitSingleTx = reader.readBoolean("implicitSingleTx");
 
                 if (!reader.isLastRead())
                     return false;
@@ -524,7 +543,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 27:
-                miniId = reader.readIgniteUuid("miniId");
+                implicitTx = reader.readBoolean("implicitTx");
 
                 if (!reader.isLastRead())
                     return false;
@@ -532,7 +551,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 28:
-                onePhaseCommit = reader.readBoolean("onePhaseCommit");
+                miniId = reader.readIgniteUuid("miniId");
 
                 if (!reader.isLastRead())
                     return false;
@@ -540,7 +559,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 29:
-                retVal = reader.readBoolean("retVal");
+                onePhaseCommit = reader.readBoolean("onePhaseCommit");
 
                 if (!reader.isLastRead())
                     return false;
@@ -548,7 +567,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 30:
-                subjId = reader.readUuid("subjId");
+                retVal = reader.readBoolean("retVal");
 
                 if (!reader.isLastRead())
                     return false;
@@ -556,7 +575,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 31:
-                syncCommit = reader.readBoolean("syncCommit");
+                subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
                     return false;
@@ -564,7 +583,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 32:
-                taskNameHash = reader.readInt("taskNameHash");
+                syncCommit = reader.readBoolean("syncCommit");
 
                 if (!reader.isLastRead())
                     return false;
@@ -572,7 +591,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 33:
-                topVer = reader.readMessage("topVer");
+                taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
                     return false;
@@ -580,6 +599,14 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 34:
+                topVer = reader.readMessage("topVer");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 35:
                 partIds = reader.readCollection("partIds", MessageCollectionItemType.INT);
 
                 if (!reader.isLastRead())
@@ -599,7 +626,7 @@ public long accessTtl() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 35;
+        return 36;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
index 7fc2b1e056837..8fe33d8326628 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
@@ -79,6 +79,9 @@ public class GridNearSingleGetRequest extends GridCacheMessage implements GridCa
     /** Task name hash. */
     private int taskNameHash;
 
+    /** TTL for read operation. */
+    private long createTtl;
+
     /** TTL for read operation. */
     private long accessTtl;
 
@@ -99,6 +102,7 @@ public GridNearSingleGetRequest() {
      * @param topVer Topology version.
      * @param subjId Subject ID.
      * @param taskNameHash Task name hash.
+     * @param createTtl New TTL to set after entry is created, -1 to leave unchanged.
      * @param accessTtl New TTL to set after entry is accessed, -1 to leave unchanged.
      * @param addReader Add reader flag.
      * @param needVer {@code True} if entry version is needed.
@@ -112,6 +116,7 @@ public GridNearSingleGetRequest(
         @NotNull AffinityTopologyVersion topVer,
         UUID subjId,
         int taskNameHash,
+        long createTtl,
         long accessTtl,
         boolean skipVals,
         boolean addReader,
@@ -127,6 +132,7 @@ public GridNearSingleGetRequest(
         this.topVer = topVer;
         this.subjId = subjId;
         this.taskNameHash = taskNameHash;
+        this.createTtl = createTtl;
         this.accessTtl = accessTtl;
         this.addDepInfo = addDepInfo;
 
@@ -180,6 +186,13 @@ public int taskNameHash() {
         return topVer;
     }
 
+    /**
+     * @return New TTL to set after entry is created, -1 to leave unchanged.
+     */
+    public long createTtl() {
+        return createTtl;
+    }
+
     /**
      * @return New TTL to set after entry is accessed, -1 to leave unchanged.
      */
@@ -266,7 +279,7 @@ public boolean needEntryInfo() {
                 reader.incrementState();
 
             case 4:
-                flags = reader.readByte("flags");
+                createTtl = reader.readLong("createTtl");
 
                 if (!reader.isLastRead())
                     return false;
@@ -274,7 +287,7 @@ public boolean needEntryInfo() {
                 reader.incrementState();
 
             case 5:
-                futId = reader.readLong("futId");
+                flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
                     return false;
@@ -282,7 +295,7 @@ public boolean needEntryInfo() {
                 reader.incrementState();
 
             case 6:
-                key = reader.readMessage("key");
+                futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
                     return false;
@@ -290,7 +303,7 @@ public boolean needEntryInfo() {
                 reader.incrementState();
 
             case 7:
-                subjId = reader.readUuid("subjId");
+                key = reader.readMessage("key");
 
                 if (!reader.isLastRead())
                     return false;
@@ -298,7 +311,7 @@ public boolean needEntryInfo() {
                 reader.incrementState();
 
             case 8:
-                taskNameHash = reader.readInt("taskNameHash");
+                partId = reader.readInt("partId", -1);
 
                 if (!reader.isLastRead())
                     return false;
@@ -306,7 +319,7 @@ public boolean needEntryInfo() {
                 reader.incrementState();
 
             case 9:
-                topVer = reader.readMessage("topVer");
+                subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
                     return false;
@@ -314,7 +327,15 @@ public boolean needEntryInfo() {
                 reader.incrementState();
 
             case 10:
-                partId = reader.readInt("partId", -1);
+                taskNameHash = reader.readInt("taskNameHash");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 11:
+                topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
                     return false;
@@ -348,43 +369,49 @@ public boolean needEntryInfo() {
                 writer.incrementState();
 
             case 4:
-                if (!writer.writeByte("flags", flags))
+                if (!writer.writeLong("createTtl", createTtl))
                     return false;
 
                 writer.incrementState();
 
             case 5:
-                if (!writer.writeLong("futId", futId))
+                if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
             case 6:
-                if (!writer.writeMessage("key", key))
+                if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
             case 7:
-                if (!writer.writeUuid("subjId", subjId))
+                if (!writer.writeMessage("key", key))
                     return false;
 
                 writer.incrementState();
 
             case 8:
-                if (!writer.writeInt("taskNameHash", taskNameHash))
+                if (!writer.writeInt("partId", partId))
                     return false;
 
                 writer.incrementState();
 
             case 9:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
             case 10:
-                if (!writer.writeInt("partId", partId))
+                if (!writer.writeInt("taskNameHash", taskNameHash))
+                    return false;
+
+                writer.incrementState();
+
+            case 11:
+                if (!writer.writeMessage("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -406,7 +433,7 @@ public boolean needEntryInfo() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 11;
+        return 12;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
index 7ac3295e6cf8e..b3eb7551f05a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
@@ -445,6 +445,7 @@ private void processLockResponse(UUID nodeId, GridNearLockResponse res) {
         boolean isRead,
         boolean retval,
         TransactionIsolation isolation,
+        long createTtl,
         long accessTtl
     ) {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
@@ -455,6 +456,7 @@ private void processLockResponse(UUID nodeId, GridNearLockResponse res) {
             isRead,
             retval,
             timeout,
+            createTtl,
             accessTtl,
             CU.empty0(),
             opCtx != null && opCtx.skipStore(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index ed37059b20cb8..67518ef1d5771 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -329,15 +329,20 @@ public void optimisticLockEntries(Collection<IgniteTxEntry> optimisticLockEntrie
         final boolean skipVals,
         final boolean needVer,
         boolean keepBinary,
+        final ExpiryPolicy expiryPlc,
         final GridInClosure3<KeyCacheObject, Object, GridCacheVersion> c
     ) {
+        IgniteCacheExpiryPolicy expiryPlc0 = optimistic() ?
+            accessPolicy(cacheCtx, keys) :
+            cacheCtx.cache().expiryPolicy(expiryPlc);
+
         if (cacheCtx.isNear()) {
             return cacheCtx.nearTx().txLoadAsync(this,
                 topVer,
                 keys,
                 readThrough,
                 /*deserializeBinary*/false,
-                accessPolicy(cacheCtx, keys),
+                expiryPlc0,
                 skipVals,
                 needVer).chain(new C1<IgniteInternalFuture<Map<Object, Object>>, Void>() {
                 @Override public Void apply(IgniteInternalFuture<Map<Object, Object>> f) {
@@ -368,7 +373,7 @@ else if (cacheCtx.isColocated()) {
                     CU.subjectId(this, cctx),
                     resolveTaskName(),
                     /*deserializeBinary*/false,
-                    accessPolicy(cacheCtx, keys),
+                    expiryPlc0,
                     skipVals,
                     /*can remap*/true,
                     needVer,
@@ -399,7 +404,7 @@ else if (cacheCtx.isColocated()) {
                     CU.subjectId(this, cctx),
                     resolveTaskName(),
                     /*deserializeBinary*/false,
-                    accessPolicy(cacheCtx, keys),
+                    expiryPlc0,
                     skipVals,
                     /*can remap*/true,
                     needVer,
@@ -433,6 +438,7 @@ else if (cacheCtx.isColocated()) {
                 skipVals,
                 keepBinary,
                 needVer,
+                expiryPlc,
                 c);
         }
     }
@@ -1163,6 +1169,7 @@ public IgniteInternalFuture<IgniteInternalTx> rollbackAsyncLocal() {
      * @param keys Keys.
      * @param retval Return value flag.
      * @param read Read flag.
+     * @param accessTtl Create ttl.
      * @param accessTtl Access ttl.
      * @param <K> Key type.
      * @param skipStore Skip store flag.
@@ -1173,6 +1180,7 @@ public <K> IgniteInternalFuture<GridCacheReturn> lockAllAsync(GridCacheContext c
         final Collection<? extends K> keys,
         boolean retval,
         boolean read,
+        long createTtl,
         long accessTtl,
         boolean skipStore,
         boolean keepBinary) {
@@ -1207,6 +1215,7 @@ public <K> IgniteInternalFuture<GridCacheReturn> lockAllAsync(GridCacheContext c
             read,
             retval,
             isolation,
+            createTtl,
             accessTtl,
             CU.empty0(),
             skipStore,
@@ -1305,6 +1314,8 @@ public <K> IgniteInternalFuture<GridCacheReturn> lockAllAsync(GridCacheContext c
 
     /** {@inheritDoc} */
     @Override protected IgniteCacheExpiryPolicy accessPolicy(GridCacheContext cacheCtx, Collection<KeyCacheObject> keys) {
+        assert optimistic();
+
         if (accessMap != null) {
             for (Map.Entry<IgniteTxKey, IgniteCacheExpiryPolicy> e : accessMap.entrySet()) {
                 if (e.getKey().cacheId() == cacheCtx.cacheId() && keys.contains(e.getKey().key()))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
index 16a35d3482747..5b44d759f9080 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
@@ -118,6 +118,7 @@ GridLocalCacheEntry entryExx(KeyCacheObject key) {
         boolean retval,
         TransactionIsolation isolation,
         boolean invalidate,
+        long createTtl,
         long accessTtl) {
         return lockAllAsync(keys, timeout, tx, CU.empty0());
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index ad818a63d8c21..ee4f7a63e1e0b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -1547,6 +1547,7 @@ private static CachePartialUpdateCheckedException partialUpdateException() {
         boolean retval,
         TransactionIsolation isolation,
         boolean invalidate,
+        long createTtl,
         long accessTtl) {
         return new GridFinishedFuture<>(new UnsupportedOperationException("Locks are not supported for " +
             "CacheAtomicityMode.ATOMIC mode (use CacheAtomicityMode.TRANSACTIONAL instead)"));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index f05d90df6cf10..a1c1123d8cfd5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -400,6 +400,7 @@ protected boolean commitAfterLock() {
         boolean skipVals,
         boolean needVer,
         boolean keepBinary,
+        final ExpiryPolicy expiryPlc,
         final GridInClosure3<KeyCacheObject, Object, GridCacheVersion> c
     ) {
         assert cacheCtx.isLocal() : cacheCtx.name();
@@ -412,7 +413,9 @@ protected boolean commitAfterLock() {
         }
 
         try {
-            IgniteCacheExpiryPolicy expiryPlc = accessPolicy(cacheCtx, keys);
+            IgniteCacheExpiryPolicy expiryPlc0 = optimistic() ?
+                accessPolicy(cacheCtx, keys) :
+                cacheCtx.cache().expiryPolicy(expiryPlc);
 
             Map<KeyCacheObject, GridCacheVersion> misses = null;
 
@@ -437,7 +440,7 @@ protected boolean commitAfterLock() {
                             CU.subjectId(this, cctx),
                             null,
                             resolveTaskName(),
-                            expiryPlc,
+                            expiryPlc0,
                             txEntry == null ? keepBinary : txEntry.keepBinary(),
                             null);
 
@@ -481,6 +484,7 @@ protected boolean commitAfterLock() {
                                     T2<CacheObject, GridCacheVersion> verVal = entry.versionedValue(cacheVal,
                                         ver,
                                         null,
+                                        null,
                                         null);
 
                                     if (log.isDebugEnabled()) {
@@ -1446,6 +1450,7 @@ protected IgniteCacheExpiryPolicy accessPolicy(GridCacheContext cacheCtx, Collec
      * @param skipVals Skip values flag.
      * @param keepCacheObjects Keep cache objects flag.
      * @param skipStore Skip store flag.
+     * @param expiryPlc Expiry policy.
      * @return Loaded key-value pairs.
      */
     private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
@@ -1457,7 +1462,8 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
         final boolean skipVals,
         final boolean keepCacheObjects,
         final boolean skipStore,
-        final boolean needVer
+        final boolean needVer,
+        final ExpiryPolicy expiryPlc
 
     ) {
         if (log.isDebugEnabled())
@@ -1486,6 +1492,7 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                 skipVals,
                 needReadVer,
                 !deserializeBinary,
+                expiryPlc,
                 new GridInClosure3<KeyCacheObject, Object, GridCacheVersion>() {
                     @Override public void apply(KeyCacheObject key, Object val, GridCacheVersion loadVer) {
                         if (isRollbackOnly()) {
@@ -1610,6 +1617,7 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                     expiryPlc = cacheCtx.expiry();
 
                 long accessTtl = expiryPlc != null ? CU.toTtl(expiryPlc.getExpiryForAccess()) : CU.TTL_NOT_CHANGED;
+                long createTtl = expiryPlc != null ? CU.toTtl(expiryPlc.getExpiryForCreation()) : CU.TTL_NOT_CHANGED;
 
                 long timeout = remainingTime();
 
@@ -1623,8 +1631,11 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                     true,
                     isolation,
                     isInvalidate(),
+                    createTtl,
                     accessTtl);
 
+                final ExpiryPolicy expiryPlc0 = expiryPlc;
+
                 PLC2<Map<K, V>> plc2 = new PLC2<Map<K, V>>() {
                     @Override public IgniteInternalFuture<Map<K, V>> postLock() throws IgniteCheckedException {
                         if (log.isDebugEnabled())
@@ -1747,7 +1758,8 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                                 skipVals,
                                 keepCacheObjects,
                                 skipStore,
-                                needVer);
+                                needVer,
+                                expiryPlc0);
                         }
 
                         return new GridFinishedFuture<>(Collections.<K, V>emptyMap());
@@ -1820,7 +1832,8 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                         skipVals,
                         keepCacheObjects,
                         skipStore,
-                        needVer);
+                        needVer,
+                        expiryPlc);
                 }
 
                 return new GridFinishedFuture<>(retMap);
@@ -2027,7 +2040,8 @@ private <K, V> IgniteInternalFuture<Void> enlistWrite(
                     hasFilters,
                     /*read through*/(entryProcessor != null || cacheCtx.config().isLoadPreviousValue()) && !skipStore,
                     retval,
-                    keepBinary);
+                    keepBinary,
+                    expiryPlc);
             }
 
             return new GridFinishedFuture<>();
@@ -2196,7 +2210,8 @@ else if (dataCenterId != null) {
                     hasFilters,
                     /*read through*/(invokeMap != null || cacheCtx.config().isLoadPreviousValue()) && !skipStore,
                     retval,
-                    keepBinary);
+                    keepBinary,
+                    expiryPlc);
             }
 
             return new GridFinishedFuture<>();
@@ -2216,6 +2231,7 @@ else if (dataCenterId != null) {
      * @param hasFilters {@code True} if filters not empty.
      * @param readThrough Read through flag.
      * @param retval Return value flag.
+     * @param expiryPlc Expiry policy.
      * @return Load future.
      */
     private IgniteInternalFuture<Void> loadMissing(
@@ -2229,7 +2245,8 @@ private IgniteInternalFuture<Void> loadMissing(
         final boolean hasFilters,
         final boolean readThrough,
         final boolean retval,
-        final boolean keepBinary) {
+        final boolean keepBinary,
+        final ExpiryPolicy expiryPlc) {
         GridInClosure3<KeyCacheObject, Object, GridCacheVersion> c =
             new GridInClosure3<KeyCacheObject, Object, GridCacheVersion>() {
                 @Override public void apply(KeyCacheObject key,
@@ -2303,6 +2320,7 @@ private IgniteInternalFuture<Void> loadMissing(
             /*skipVals*/singleRmv,
             needReadVer,
             keepBinary,
+            expiryPlc,
             c);
     }
 
@@ -2966,6 +2984,7 @@ private <K, V> IgniteInternalFuture putAsync0(
                     retval,
                     isolation,
                     isInvalidate(),
+                    -1L,
                     -1L);
 
                 PLC1<GridCacheReturn> plc1 = new PLC1<GridCacheReturn>(ret) {
@@ -3144,6 +3163,7 @@ private <K, V> IgniteInternalFuture putAllAsync0(
                     retval,
                     isolation,
                     isInvalidate(),
+                    -1L,
                     -1L);
 
                 PLC1<GridCacheReturn> plc1 = new PLC1<GridCacheReturn>(ret) {
@@ -3438,6 +3458,7 @@ private <K, V> IgniteInternalFuture<GridCacheReturn> removeAllAsync0(
                 retval,
                 isolation,
                 isInvalidate(),
+                -1L,
                 -1L);
 
             PLC1<GridCacheReturn> plc1 = new PLC1<GridCacheReturn>(ret) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
index 9fb3558a04b1f..f5687a0cafcba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
@@ -20,6 +20,7 @@
 import java.util.Collection;
 import java.util.Map;
 import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -181,6 +182,7 @@ public IgniteInternalFuture<?> removeAllDrAsync(
      * @param skipVals Skip values flag.
      * @param needVer If {@code true} version is required for loaded values.
      * @param c Closure to be applied for loaded values.
+     * @param expiryPlc Expiry policy.
      * @return Future with {@code True} value if loading took place.
      */
     public IgniteInternalFuture<Void> loadMissing(
@@ -192,5 +194,6 @@ public IgniteInternalFuture<Void> loadMissing(
         boolean skipVals,
         boolean needVer,
         boolean keepBinary,
+        final ExpiryPolicy expiryPlc,
         GridInClosure3<KeyCacheObject, Object, GridCacheVersion> c);
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 8db68b4ef50fb..2954bdb413d16 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -689,7 +689,8 @@ void recheckLock() {
     @Override public T2<CacheObject, GridCacheVersion> versionedValue(CacheObject val,
         GridCacheVersion curVer,
         GridCacheVersion newVer,
-        @Nullable ReaderArguments readerArgs) {
+        @Nullable ReaderArguments readerArgs,
+        IgniteCacheExpiryPolicy loadExpiryPlc) {
         assert false;
 
         return null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyWithStoreAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyWithStoreAbstractTest.java
index 78c59ac047b23..1f6ec2d297dd1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyWithStoreAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyWithStoreAbstractTest.java
@@ -17,6 +17,11 @@
 
 package org.apache.ignite.internal.processors.cache.expiry;
 
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import javax.cache.configuration.Factory;
 import javax.cache.expiry.Duration;
@@ -25,6 +30,7 @@
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.MutableEntry;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteTransactions;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.store.CacheStore;
@@ -36,6 +42,9 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
 
 /**
  *
@@ -173,6 +182,99 @@ public void testReadThrough() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testGetReadThrough() throws Exception {
+        getReadThrough(false, null, null);
+        getReadThrough(true, null, null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    protected void getReadThrough(boolean withExcPlc,
+        TransactionConcurrency txConcurrency,
+        TransactionIsolation txIsolation) throws Exception {
+        IgniteCache<Integer, Integer> cache = jcache(0);
+
+        if (withExcPlc)
+            cache = cache.withExpiryPolicy(new ExpiryPolicy() {
+                @Override public Duration getExpiryForCreation() {
+                    return new Duration(TimeUnit.MILLISECONDS, 501);
+                }
+
+                @Override public Duration getExpiryForAccess() {
+                    return new Duration(TimeUnit.MILLISECONDS, 601);
+                }
+
+                @Override public Duration getExpiryForUpdate() {
+                    return new Duration(TimeUnit.MILLISECONDS, 701);
+                }
+            });
+
+        Integer prim = primaryKeys(cache, 1, 1000).get(0);
+        Integer back = backupKeys(cache, 1, 1000).get(0);
+        Integer near = nearKeys(cache, 1, 1000).get(0);
+
+        Set<Integer> prims = new HashSet<>(primaryKeys(cache, 10, prim + 1));
+        Set<Integer> backs = new HashSet<>(backupKeys(cache, 10, back + 1));
+        Set<Integer> nears = new HashSet<>(nearKeys(cache, 10, near + 1));
+
+        Set<Integer> keys = new HashSet<>();
+
+        keys.add(prim);
+        keys.add(back);
+        keys.add(near);
+
+        keys.addAll(prims);
+        keys.addAll(backs);
+        keys.addAll(nears);
+
+        for (Integer key : keys)
+            storeMap.put(key, key);
+
+        IgniteTransactions transactions = grid(0).transactions();
+
+        Transaction tx = txConcurrency != null ? transactions.txStart(txConcurrency, txIsolation) : null;
+
+        try {
+            Collection<Integer> singleKeys = new HashSet<>();
+
+            singleKeys.add(prim);
+            singleKeys.add(back);
+            singleKeys.add(near);
+
+            assertEquals(3, singleKeys.size());
+
+            for (Integer key : singleKeys)
+                assertEquals(key, cache.get(key));
+
+            Map<Integer, Integer> res = new HashMap<>();
+
+            res.putAll(cache.getAll(prims));
+            res.putAll(cache.getAll(backs));
+            res.putAll(cache.getAll(nears));
+
+            assertEquals(30, res.size());
+
+            for (Map.Entry<Integer, Integer> e : res.entrySet())
+                assertEquals(e.getKey(), e.getValue());
+        }
+        finally {
+            if (tx != null)
+                tx.rollback();
+        }
+
+        for (Integer key : keys)
+            checkTtl(key, withExcPlc ? 501 : 500, true);
+
+        U.sleep(600);
+
+        for (Integer key : keys)
+            checkExpired(key);
+    }
+
     /**
      * @param key Key.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheTxExpiryPolicyWithStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheTxExpiryPolicyWithStoreTest.java
index 4b9b61a86496f..f5888f88ac696 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheTxExpiryPolicyWithStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheTxExpiryPolicyWithStoreTest.java
@@ -19,6 +19,8 @@
 
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
@@ -41,4 +43,23 @@ public class IgniteCacheTxExpiryPolicyWithStoreTest extends IgniteCacheExpiryPol
     @Override protected CacheAtomicityMode atomicityMode() {
         return TRANSACTIONAL;
     }
+
+    /** {@inheritDoc} */
+    @Override public void testGetReadThrough() throws Exception {
+        super.testGetReadThrough();
+
+        getReadThrough(false, TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
+        getReadThrough(true, TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
+        getReadThrough(false, TransactionConcurrency.OPTIMISTIC, TransactionIsolation.REPEATABLE_READ);
+        getReadThrough(true, TransactionConcurrency.OPTIMISTIC, TransactionIsolation.REPEATABLE_READ);
+        getReadThrough(false, TransactionConcurrency.OPTIMISTIC, TransactionIsolation.SERIALIZABLE);
+        getReadThrough(true, TransactionConcurrency.OPTIMISTIC, TransactionIsolation.SERIALIZABLE);
+
+        getReadThrough(false, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.READ_COMMITTED);
+        getReadThrough(true, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.READ_COMMITTED);
+        getReadThrough(false, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ);
+        getReadThrough(true, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ);
+        getReadThrough(false, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.SERIALIZABLE);
+        getReadThrough(true, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.SERIALIZABLE);
+    }
 }
\ No newline at end of file

From 3db0971d7d32798aeb3ce5dd8c0b3246a895fe91 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Tue, 24 Jan 2017 16:45:59 +0300
Subject: [PATCH 121/446] IGNITE-4598: Hadoop: implemented raw comparator for
 BytesWritable key type. This closes #1457.

---
 .../BytesWritablePartiallyRawComparator.java  | 51 ++++++++++++++
 .../hadoop/io/TextPartiallyRawComparator.java | 68 +------------------
 .../processors/hadoop/impl/HadoopUtils.java   | 66 ++++++++++++++++++
 .../hadoop/impl/v2/HadoopV2TaskContext.java   | 13 +++-
 4 files changed, 129 insertions(+), 69 deletions(-)
 create mode 100644 modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/BytesWritablePartiallyRawComparator.java

diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/BytesWritablePartiallyRawComparator.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/BytesWritablePartiallyRawComparator.java
new file mode 100644
index 0000000000000..da9240b67a3fe
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/BytesWritablePartiallyRawComparator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.io;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils;
+import org.apache.ignite.internal.processors.hadoop.io.OffheapRawMemory;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
+
+/**
+ * Partial raw comparator for {@link BytesWritable} data type.
+ * <p>
+ * Implementation is borrowed from {@code org.apache.hadoop.io.FastByteComparisons} and adopted to Ignite
+ * infrastructure.
+ */
+public class BytesWritablePartiallyRawComparator implements PartiallyRawComparator<BytesWritable>,
+    PartiallyOffheapRawComparatorEx<BytesWritable> {
+    /** Length bytes. */
+    private static final int LEN_BYTES = 4;
+
+    /** {@inheritDoc} */
+    @Override public int compare(BytesWritable val1, RawMemory val2Buf) {
+        if (val2Buf instanceof OffheapRawMemory) {
+            OffheapRawMemory val2Buf0 = (OffheapRawMemory)val2Buf;
+
+            return compare(val1, val2Buf0.pointer(), val2Buf0.length());
+        }
+        else
+            throw new UnsupportedOperationException("Text can be compared only with offheap memory.");
+    }
+
+    /** {@inheritDoc} */
+    @Override public int compare(BytesWritable val1, long val2Ptr, int val2Len) {
+        return HadoopUtils.compareBytes(val1.getBytes(), val1.getLength(), val2Ptr + LEN_BYTES, val2Len - LEN_BYTES);
+    }
+}
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java
index a2bc3d4c8e249..e82f5e4c19055 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java
@@ -17,10 +17,9 @@
 
 package org.apache.ignite.hadoop.io;
 
-import com.google.common.primitives.Longs;
-import com.google.common.primitives.UnsignedBytes;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils;
 import org.apache.ignite.internal.processors.hadoop.io.OffheapRawMemory;
 import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
 import org.apache.ignite.internal.util.GridUnsafe;
@@ -47,69 +46,6 @@ public class TextPartiallyRawComparator implements PartiallyRawComparator<Text>,
     @Override public int compare(Text val1, long val2Ptr, int val2Len) {
         int len2 = WritableUtils.decodeVIntSize(GridUnsafe.getByte(val2Ptr));
 
-        return compareBytes(val1.getBytes(), val1.getLength(), val2Ptr + len2, val2Len - len2);
-    }
-
-    /**
-     * Internal comparison routine.
-     *
-     * @param buf1 Bytes 1.
-     * @param len1 Length 1.
-     * @param ptr2 Pointer 2.
-     * @param len2 Length 2.
-     * @return Result.
-     */
-    @SuppressWarnings("SuspiciousNameCombination")
-    private static int compareBytes(byte[] buf1, int len1, long ptr2, int len2) {
-        int minLength = Math.min(len1, len2);
-
-        int minWords = minLength / Longs.BYTES;
-
-        for (int i = 0; i < minWords * Longs.BYTES; i += Longs.BYTES) {
-            long lw = GridUnsafe.getLong(buf1, GridUnsafe.BYTE_ARR_OFF + i);
-            long rw = GridUnsafe.getLong(ptr2 + i);
-
-            long diff = lw ^ rw;
-
-            if (diff != 0) {
-                if (GridUnsafe.BIG_ENDIAN)
-                    return (lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE) ? -1 : 1;
-
-                // Use binary search
-                int n = 0;
-                int y;
-                int x = (int) diff;
-
-                if (x == 0) {
-                    x = (int) (diff >>> 32);
-
-                    n = 32;
-                }
-
-                y = x << 16;
-
-                if (y == 0)
-                    n += 16;
-                else
-                    x = y;
-
-                y = x << 8;
-
-                if (y == 0)
-                    n += 8;
-
-                return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
-            }
-        }
-
-        // The epilogue to cover the last (minLength % 8) elements.
-        for (int i = minWords * Longs.BYTES; i < minLength; i++) {
-            int res = UnsignedBytes.compare(buf1[i], GridUnsafe.getByte(ptr2 + i));
-
-            if (res != 0)
-                return res;
-        }
-
-        return len1 - len2;
+        return HadoopUtils.compareBytes(val1.getBytes(), val1.getLength(), val2Ptr + len2, val2Len - len2);
     }
 }
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopUtils.java
index a34388dba79df..767e10acbaaa3 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopUtils.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.hadoop.impl;
 
+import com.google.common.primitives.Longs;
+import com.google.common.primitives.UnsignedBytes;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Writable;
@@ -32,6 +34,7 @@
 import org.apache.ignite.internal.processors.hadoop.HadoopJobStatus;
 import org.apache.ignite.internal.processors.hadoop.HadoopSplitWrapper;
 import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
+import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 import java.io.ByteArrayInputStream;
@@ -328,4 +331,67 @@ public static Configuration safeCreateConfiguration() {
             HadoopCommonUtils.restoreContextClassLoader(oldLdr);
         }
     }
+
+    /**
+     * Internal comparison routine.
+     *
+     * @param buf1 Bytes 1.
+     * @param len1 Length 1.
+     * @param ptr2 Pointer 2.
+     * @param len2 Length 2.
+     * @return Result.
+     */
+    @SuppressWarnings("SuspiciousNameCombination")
+    public static int compareBytes(byte[] buf1, int len1, long ptr2, int len2) {
+        int minLength = Math.min(len1, len2);
+
+        int minWords = minLength / Longs.BYTES;
+
+        for (int i = 0; i < minWords * Longs.BYTES; i += Longs.BYTES) {
+            long lw = GridUnsafe.getLong(buf1, GridUnsafe.BYTE_ARR_OFF + i);
+            long rw = GridUnsafe.getLong(ptr2 + i);
+
+            long diff = lw ^ rw;
+
+            if (diff != 0) {
+                if (GridUnsafe.BIG_ENDIAN)
+                    return (lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE) ? -1 : 1;
+
+                // Use binary search
+                int n = 0;
+                int y;
+                int x = (int) diff;
+
+                if (x == 0) {
+                    x = (int) (diff >>> 32);
+
+                    n = 32;
+                }
+
+                y = x << 16;
+
+                if (y == 0)
+                    n += 16;
+                else
+                    x = y;
+
+                y = x << 8;
+
+                if (y == 0)
+                    n += 8;
+
+                return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
+            }
+        }
+
+        // The epilogue to cover the last (minLength % 8) elements.
+        for (int i = minWords * Longs.BYTES; i < minLength; i++) {
+            int res = UnsignedBytes.compare(buf1[i], GridUnsafe.getByte(ptr2 + i));
+
+            if (res != 0)
+                return res;
+        }
+
+        return len1 - len2;
+    }
 }
\ No newline at end of file
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
index 475e43db50eb0..b14dc47c4a8eb 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.ByteWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.serializer.Deserializer;
@@ -40,6 +41,7 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.hadoop.io.BytesWritablePartiallyRawComparator;
 import org.apache.ignite.hadoop.io.PartiallyRawComparator;
 import org.apache.ignite.hadoop.io.TextPartiallyRawComparator;
 import org.apache.ignite.internal.processors.hadoop.HadoopClassLoader;
@@ -49,7 +51,6 @@
 import org.apache.ignite.internal.processors.hadoop.HadoopJob;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty;
-import org.apache.ignite.internal.processors.hadoop.HadoopMapperAwareTaskOutput;
 import org.apache.ignite.internal.processors.hadoop.HadoopPartitioner;
 import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
 import org.apache.ignite.internal.processors.hadoop.HadoopSplitWrapper;
@@ -156,6 +157,7 @@ public static void close() throws IgniteCheckedException {
 
         COMBINE_KEY_GROUPING_SUPPORTED = ok;
 
+        PARTIAL_COMPARATORS.put(ByteWritable.class.getName(), BytesWritablePartiallyRawComparator.class.getName());
         PARTIAL_COMPARATORS.put(Text.class.getName(), TextPartiallyRawComparator.class.getName());
     }
 
@@ -602,11 +604,16 @@ private void initializePartiallyRawComparator(JobConf conf) {
         if (clsName == null) {
             Class keyCls = conf.getMapOutputKeyClass();
 
-            if (keyCls != null) {
+            while (keyCls != null) {
                 clsName = PARTIAL_COMPARATORS.get(keyCls.getName());
 
-                if (clsName != null)
+                if (clsName != null) {
                     conf.set(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(), clsName);
+
+                    break;
+                }
+
+                keyCls = keyCls.getSuperclass();
             }
         }
     }

From 295f80d10da02692540435d46961a9c3cca99b3c Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 25 Jan 2017 13:58:57 +0700
Subject: [PATCH 122/446] IGNITE-4520 Added credential request for
 authentication on proxy.

(cherry picked from commit ef04f35)
---
 .../web-agent/bin/ignite-web-agent.bat        |  4 +-
 .../web-agent/bin/ignite-web-agent.sh         |  2 +
 .../ignite/console/agent/AgentLauncher.java   | 90 +++++++++++++++++--
 3 files changed, 88 insertions(+), 8 deletions(-)

diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.bat b/modules/web-console/web-agent/bin/ignite-web-agent.bat
index 8291b5531da50..1f1b52dc83d23 100644
--- a/modules/web-console/web-agent/bin/ignite-web-agent.bat
+++ b/modules/web-console/web-agent/bin/ignite-web-agent.bat
@@ -60,7 +60,9 @@ if %ERRORLEVEL% equ 0 (
     if "%JVM_OPTS%" == "" set JVM_OPTS=-Xms1g -Xmx1g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=256m
 )
 
-"%JAVA_HOME%\bin\java.exe" %JVM_OPTS% -cp "*" org.apache.ignite.console.agent.AgentLauncher  %*
+set JVM_OPTS=%JVM_OPTS% -Djava.net.useSystemProxies=true
+
+"%JAVA_HOME%\bin\java.exe" %JVM_OPTS% -cp "*" org.apache.ignite.console.agent.AgentLauncher %*
 
 set JAVA_ERRORLEVEL=%ERRORLEVEL%
 
diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.sh b/modules/web-console/web-agent/bin/ignite-web-agent.sh
index 2e9f041f8ca7c..c2958fc0c7c9a 100644
--- a/modules/web-console/web-agent/bin/ignite-web-agent.sh
+++ b/modules/web-console/web-agent/bin/ignite-web-agent.sh
@@ -88,4 +88,6 @@ if [ -z "$JVM_OPTS" ] ; then
     fi
 fi
 
+JVM_OPTS="${JVM_OPTS} -Djava.net.useSystemProxies=true"
+
 "$JAVA" ${JVM_OPTS} -cp "*" org.apache.ignite.console.agent.AgentLauncher "$@"
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 0c03d7787b15d..049791f9107dc 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -25,11 +25,15 @@
 import io.socket.emitter.Emitter;
 import java.io.File;
 import java.io.IOException;
+import java.net.Authenticator;
 import java.net.ConnectException;
+import java.net.PasswordAuthentication;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.UnknownHostException;
 import java.util.Arrays;
+import java.util.Scanner;
 import java.util.concurrent.CountDownLatch;
 import java.util.jar.Attributes;
 import java.util.jar.Manifest;
@@ -76,9 +80,6 @@ public class AgentLauncher {
     /** */
     private static final String EVENT_AGENT_CLOSE = "agent:close";
 
-    /** */
-    private static final int RECONNECT_INTERVAL = 3000;
-
     /**
      * Create a trust manager that trusts all certificates It is not using a particular keyStore
      */
@@ -121,6 +122,15 @@ public void checkServerTrusted(
                     System.exit(1);
                 }
 
+                ignore = X.cause(e, UnknownHostException.class);
+
+                if (ignore != null) {
+                    log.error("Failed to establish connection to server, due to errors with DNS or missing proxy settings.");
+                    log.error("Documentation for proxy configuration can be found here: http://apacheignite.readme.io/docs/web-agent#section-proxy-configuration");
+
+                    System.exit(1);
+                }
+
                 ignore = X.cause(e, IOException.class);
 
                 if (ignore != null && "404".equals(ignore.getMessage())) {
@@ -129,6 +139,29 @@ public void checkServerTrusted(
                     return;
                 }
 
+                if (ignore != null && "407".equals(ignore.getMessage())) {
+                    log.error("Failed to establish connection to server, due to proxy requires authentication.");
+
+                    String userName = System.getProperty("https.proxyUsername", System.getProperty("http.proxyUsername"));
+
+                    if (userName == null || userName.trim().isEmpty())
+                        userName = readLine("Enter proxy user name: ");
+                    else
+                        System.out.println("Read username from system properties: " + userName);
+
+                    char[] pwd = readPassword("Enter proxy password: ");
+
+                    final PasswordAuthentication pwdAuth = new PasswordAuthentication(userName, pwd);
+
+                    Authenticator.setDefault(new Authenticator() {
+                        @Override protected PasswordAuthentication getPasswordAuthentication() {
+                            return pwdAuth;
+                        }
+                    });
+
+                    return;
+                }
+
                 log.error("Connection error.", e);
             }
         }
@@ -143,6 +176,32 @@ public void checkServerTrusted(
         }
     };
 
+    /**
+     * @param fmt Format string.
+     * @param args Arguments.
+     */
+    private static String readLine(String fmt, Object ... args) {
+        if (System.console() != null)
+            return System.console().readLine(fmt, args);
+
+        System.out.print(String.format(fmt, args));
+
+        return new Scanner(System.in).nextLine();
+    }
+
+    /**
+     * @param fmt Format string.
+     * @param args Arguments.
+     */
+    private static char[] readPassword(String fmt, Object ... args) {
+        if (System.console() != null)
+            return System.console().readPassword(fmt, args);
+
+        System.out.print(String.format(fmt, args));
+
+        return new Scanner(System.in).nextLine().toCharArray();
+    }
+
     /**
      * @param args Args.
      */
@@ -214,9 +273,9 @@ public static void main(String[] args) throws Exception {
             System.out.println("Security token is required to establish connection to the web console.");
             System.out.println(String.format("It is available on the Profile page: https://%s/profile", webHost));
 
-            System.out.print("Enter security tokens separated by comma: ");
+            String tokens = String.valueOf(readPassword("Enter security tokens separated by comma: "));
 
-            cfg.tokens(Arrays.asList(System.console().readLine().trim().split(",")));
+            cfg.tokens(Arrays.asList(tokens.trim().split(",")));
         }
 
         final RestHandler restHnd = new RestHandler(cfg);
@@ -226,12 +285,29 @@ public static void main(String[] args) throws Exception {
 
             URI uri = URI.create(cfg.serverUri());
 
+            // Create proxy authenticator using passed properties.
+            switch (uri.getScheme()) {
+                case "http":
+                case "https":
+                    final String username = System.getProperty(uri.getScheme() + ".proxyUsername");
+                    final char[] pwd = System.getProperty(uri.getScheme() +  ".proxyPassword", "").toCharArray();
+
+                    Authenticator.setDefault(new Authenticator() {
+                        @Override protected PasswordAuthentication getPasswordAuthentication() {
+                            return new PasswordAuthentication(username, pwd);
+                        }
+                    });
+
+                    break;
+
+                default:
+                    // No-op.
+            }
+
             IO.Options opts = new IO.Options();
 
             opts.path = "/agents";
 
-            opts.reconnectionDelay = RECONNECT_INTERVAL;
-
             // Workaround for use self-signed certificate
             if (Boolean.getBoolean("trust.all")) {
                 SSLContext ctx = SSLContext.getInstance("TLS");

From f25f85afb9d9d5e91116086a6b3998a5bbb082b9 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 25 Jan 2017 13:58:57 +0700
Subject: [PATCH 123/446] IGNITE-4520 Added credential request for
 authentication on proxy.

(cherry picked from commit ef04f35)
---
 .../web-agent/bin/ignite-web-agent.bat        |  4 +-
 .../web-agent/bin/ignite-web-agent.sh         |  2 +
 .../ignite/console/agent/AgentLauncher.java   | 90 +++++++++++++++++--
 3 files changed, 88 insertions(+), 8 deletions(-)

diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.bat b/modules/web-console/web-agent/bin/ignite-web-agent.bat
index 8291b5531da50..1f1b52dc83d23 100644
--- a/modules/web-console/web-agent/bin/ignite-web-agent.bat
+++ b/modules/web-console/web-agent/bin/ignite-web-agent.bat
@@ -60,7 +60,9 @@ if %ERRORLEVEL% equ 0 (
     if "%JVM_OPTS%" == "" set JVM_OPTS=-Xms1g -Xmx1g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=256m
 )
 
-"%JAVA_HOME%\bin\java.exe" %JVM_OPTS% -cp "*" org.apache.ignite.console.agent.AgentLauncher  %*
+set JVM_OPTS=%JVM_OPTS% -Djava.net.useSystemProxies=true
+
+"%JAVA_HOME%\bin\java.exe" %JVM_OPTS% -cp "*" org.apache.ignite.console.agent.AgentLauncher %*
 
 set JAVA_ERRORLEVEL=%ERRORLEVEL%
 
diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.sh b/modules/web-console/web-agent/bin/ignite-web-agent.sh
index 2e9f041f8ca7c..c2958fc0c7c9a 100644
--- a/modules/web-console/web-agent/bin/ignite-web-agent.sh
+++ b/modules/web-console/web-agent/bin/ignite-web-agent.sh
@@ -88,4 +88,6 @@ if [ -z "$JVM_OPTS" ] ; then
     fi
 fi
 
+JVM_OPTS="${JVM_OPTS} -Djava.net.useSystemProxies=true"
+
 "$JAVA" ${JVM_OPTS} -cp "*" org.apache.ignite.console.agent.AgentLauncher "$@"
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 0c03d7787b15d..049791f9107dc 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -25,11 +25,15 @@
 import io.socket.emitter.Emitter;
 import java.io.File;
 import java.io.IOException;
+import java.net.Authenticator;
 import java.net.ConnectException;
+import java.net.PasswordAuthentication;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.UnknownHostException;
 import java.util.Arrays;
+import java.util.Scanner;
 import java.util.concurrent.CountDownLatch;
 import java.util.jar.Attributes;
 import java.util.jar.Manifest;
@@ -76,9 +80,6 @@ public class AgentLauncher {
     /** */
     private static final String EVENT_AGENT_CLOSE = "agent:close";
 
-    /** */
-    private static final int RECONNECT_INTERVAL = 3000;
-
     /**
      * Create a trust manager that trusts all certificates It is not using a particular keyStore
      */
@@ -121,6 +122,15 @@ public void checkServerTrusted(
                     System.exit(1);
                 }
 
+                ignore = X.cause(e, UnknownHostException.class);
+
+                if (ignore != null) {
+                    log.error("Failed to establish connection to server, due to errors with DNS or missing proxy settings.");
+                    log.error("Documentation for proxy configuration can be found here: http://apacheignite.readme.io/docs/web-agent#section-proxy-configuration");
+
+                    System.exit(1);
+                }
+
                 ignore = X.cause(e, IOException.class);
 
                 if (ignore != null && "404".equals(ignore.getMessage())) {
@@ -129,6 +139,29 @@ public void checkServerTrusted(
                     return;
                 }
 
+                if (ignore != null && "407".equals(ignore.getMessage())) {
+                    log.error("Failed to establish connection to server, due to proxy requires authentication.");
+
+                    String userName = System.getProperty("https.proxyUsername", System.getProperty("http.proxyUsername"));
+
+                    if (userName == null || userName.trim().isEmpty())
+                        userName = readLine("Enter proxy user name: ");
+                    else
+                        System.out.println("Read username from system properties: " + userName);
+
+                    char[] pwd = readPassword("Enter proxy password: ");
+
+                    final PasswordAuthentication pwdAuth = new PasswordAuthentication(userName, pwd);
+
+                    Authenticator.setDefault(new Authenticator() {
+                        @Override protected PasswordAuthentication getPasswordAuthentication() {
+                            return pwdAuth;
+                        }
+                    });
+
+                    return;
+                }
+
                 log.error("Connection error.", e);
             }
         }
@@ -143,6 +176,32 @@ public void checkServerTrusted(
         }
     };
 
+    /**
+     * @param fmt Format string.
+     * @param args Arguments.
+     */
+    private static String readLine(String fmt, Object ... args) {
+        if (System.console() != null)
+            return System.console().readLine(fmt, args);
+
+        System.out.print(String.format(fmt, args));
+
+        return new Scanner(System.in).nextLine();
+    }
+
+    /**
+     * @param fmt Format string.
+     * @param args Arguments.
+     */
+    private static char[] readPassword(String fmt, Object ... args) {
+        if (System.console() != null)
+            return System.console().readPassword(fmt, args);
+
+        System.out.print(String.format(fmt, args));
+
+        return new Scanner(System.in).nextLine().toCharArray();
+    }
+
     /**
      * @param args Args.
      */
@@ -214,9 +273,9 @@ public static void main(String[] args) throws Exception {
             System.out.println("Security token is required to establish connection to the web console.");
             System.out.println(String.format("It is available on the Profile page: https://%s/profile", webHost));
 
-            System.out.print("Enter security tokens separated by comma: ");
+            String tokens = String.valueOf(readPassword("Enter security tokens separated by comma: "));
 
-            cfg.tokens(Arrays.asList(System.console().readLine().trim().split(",")));
+            cfg.tokens(Arrays.asList(tokens.trim().split(",")));
         }
 
         final RestHandler restHnd = new RestHandler(cfg);
@@ -226,12 +285,29 @@ public static void main(String[] args) throws Exception {
 
             URI uri = URI.create(cfg.serverUri());
 
+            // Create proxy authenticator using passed properties.
+            switch (uri.getScheme()) {
+                case "http":
+                case "https":
+                    final String username = System.getProperty(uri.getScheme() + ".proxyUsername");
+                    final char[] pwd = System.getProperty(uri.getScheme() +  ".proxyPassword", "").toCharArray();
+
+                    Authenticator.setDefault(new Authenticator() {
+                        @Override protected PasswordAuthentication getPasswordAuthentication() {
+                            return new PasswordAuthentication(username, pwd);
+                        }
+                    });
+
+                    break;
+
+                default:
+                    // No-op.
+            }
+
             IO.Options opts = new IO.Options();
 
             opts.path = "/agents";
 
-            opts.reconnectionDelay = RECONNECT_INTERVAL;
-
             // Workaround for use self-signed certificate
             if (Boolean.getBoolean("trust.all")) {
                 SSLContext ctx = SSLContext.getInstance("TLS");

From ccc0f41794bfd2deecc3e0bd042f82d81c9809d0 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Fri, 20 Jan 2017 16:22:24 +0700
Subject: [PATCH 124/446] IGNITE-4548 CacheJdbcStore: support mapping of enum
 types.

(cherry picked from commit f1fca3a)
---
 .../store/jdbc/CacheAbstractJdbcStore.java    | 12 +++++-
 .../jdbc/JdbcTypesDefaultTransformer.java     | 19 +++++++++
 .../store/jdbc/JdbcTypesTransformer.java      | 17 ++++++++
 .../CacheJdbcPojoStoreAbstractSelfTest.java   | 23 +++++++----
 .../store/jdbc/CacheJdbcPojoStoreTest.java    |  3 ++
 ...dbcStoreAbstractMultithreadedSelfTest.java | 17 ++++----
 .../ignite/cache/store/jdbc/model/Gender.java | 41 +++++++++++++++++++
 .../ignite/cache/store/jdbc/model/Person.java | 31 +++++++++++++-
 .../test/config/jdbc-pojo-store-builtin.xml   |  8 ++++
 .../src/test/config/jdbc-pojo-store-obj.xml   |  8 ++++
 10 files changed, 162 insertions(+), 17 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Gender.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
index 4bfd92b836268..e7ce52651a868 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
@@ -80,6 +80,8 @@
 import static org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory.DFLT_BATCH_SIZE;
 import static org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory.DFLT_PARALLEL_LOAD_CACHE_MINIMUM_THRESHOLD;
 import static org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory.DFLT_WRITE_ATTEMPTS;
+import static org.apache.ignite.cache.store.jdbc.JdbcTypesTransformer.NUMERIC_TYPES;
+import static org.apache.ignite.cache.store.jdbc.JdbcTypesTransformer.NUMERIC_TYPES;
 
 /**
  * Implementation of {@link CacheStore} backed by JDBC.
@@ -1393,8 +1395,15 @@ protected void fillParameter(PreparedStatement stmt, int idx, JdbcTypeField fiel
                             fieldVal = fieldVal.toString();
 
                             break;
+                        default:
+                            // No-op.
                     }
                 }
+                else if (field.getJavaFieldType().isEnum() && fieldVal instanceof Enum) {
+                    Enum val = (Enum)fieldVal;
+                    
+                    fieldVal = NUMERIC_TYPES.contains(field.getDatabaseFieldType()) ? val.ordinal() : val.name();
+                }
 
                 stmt.setObject(idx, fieldVal);
             }
@@ -2068,12 +2077,13 @@ private LoadWorker(Connection conn, EntryMapping em) {
 
                 int idx = 1;
 
-                for (Object key : keys)
+                for (Object key : keys) {
                     for (JdbcTypeField field : em.keyColumns()) {
                         Object fieldVal = extractParameter(em.cacheName, em.keyType(), em.keyKind(), field.getJavaFieldName(), key);
 
                         fillParameter(stmt, idx++, field, fieldVal);
                     }
+                }
 
                 ResultSet rs = stmt.executeQuery();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
index c32eaa227dfd0..c387b779e60c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
@@ -114,6 +114,25 @@ public class JdbcTypesDefaultTransformer implements JdbcTypesTransformer {
                 return UUID.fromString((String)res);
         }
 
+        if (type.isEnum()) {
+            if (NUMERIC_TYPES.contains(rs.getMetaData().getColumnType(colIdx))) {
+                int ordinal = rs.getInt(colIdx);
+
+                Object[] values = type.getEnumConstants();
+
+                return rs.wasNull() || ordinal >= values.length ? null : values[ordinal];
+            }
+
+            String str = rs.getString(colIdx);
+
+            try {
+                return rs.wasNull() ? null : Enum.valueOf((Class<? extends Enum>) type, str.trim());
+            }
+            catch (IllegalArgumentException ignore) {
+                return null;
+            }
+        }
+
         return rs.getObject(colIdx);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesTransformer.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesTransformer.java
index 76fb00b90b2ed..fc0bc883d0778 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesTransformer.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesTransformer.java
@@ -20,11 +20,28 @@
 import java.io.Serializable;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static java.sql.Types.BIGINT;
+import static java.sql.Types.DECIMAL;
+import static java.sql.Types.DOUBLE;
+import static java.sql.Types.FLOAT;
+import static java.sql.Types.INTEGER;
+import static java.sql.Types.NUMERIC;
+import static java.sql.Types.REAL;
+import static java.sql.Types.SMALLINT;
+import static java.sql.Types.TINYINT;
 
 /**
  * API for implementing custom mapping logic for loaded from store data.
  */
 public interface JdbcTypesTransformer extends Serializable {
+    /** Numeric types. */
+    public final List<Integer> NUMERIC_TYPES =
+        U.sealList(TINYINT, SMALLINT, INTEGER, BIGINT, REAL, FLOAT, DOUBLE, NUMERIC, DECIMAL);
+
+
     /**
      * Retrieves the value of the designated column in the current row of this <code>ResultSet</code> object and
      * will convert to the requested Java data type.
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
index 368a28e010c3b..1de44f7243fbb 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect;
 import org.apache.ignite.cache.store.jdbc.model.Person;
+import org.apache.ignite.cache.store.jdbc.model.Gender;
 import org.apache.ignite.cache.store.jdbc.model.PersonKey;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ConnectorConfiguration;
@@ -112,7 +113,8 @@ protected Connection getConnection() throws SQLException {
             " id INTEGER PRIMARY KEY," +
             " org_id INTEGER," +
             " birthday DATE," +
-            " name VARCHAR(50))");
+            " name VARCHAR(50)," +
+            " gender VARCHAR(50))");
 
         conn.commit();
 
@@ -201,7 +203,8 @@ protected JdbcType[] storeTypes() {
             new JdbcTypeField(Types.INTEGER, "ID", Integer.class, "id"),
             new JdbcTypeField(Types.INTEGER, "ORG_ID", Integer.class, "orgId"),
             new JdbcTypeField(Types.DATE, "BIRTHDAY", Date.class, "birthday"),
-            new JdbcTypeField(Types.VARCHAR, "NAME", String.class, "name"));
+            new JdbcTypeField(Types.VARCHAR, "NAME", String.class, "name"),
+            new JdbcTypeField(Types.VARCHAR, "GENDER", Gender.class, "gender"));
 
         return storeTypes;
     }
@@ -260,7 +263,7 @@ protected void fillSampleDatabase(Connection conn) throws SQLException {
         conn.commit();
 
         PreparedStatement prnStmt = conn.prepareStatement(
-            "INSERT INTO Person(id, org_id, birthday, name) VALUES (?, ?, ?, ?)");
+            "INSERT INTO Person(id, org_id, birthday, name, gender) VALUES (?, ?, ?, ?, ?)");
 
         Random rnd = new Random();
 
@@ -269,6 +272,7 @@ protected void fillSampleDatabase(Connection conn) throws SQLException {
             prnStmt.setInt(2, i % 100);
             prnStmt.setDate(3, Date.valueOf(String.format("%d-%d-%d", 1970 + rnd.nextInt(50), 1 + rnd.nextInt(11), 1 + rnd.nextInt(27))));
             prnStmt.setString(4, "name" + i);
+            prnStmt.setString(5, Gender.random().toString());
 
             prnStmt.addBatch();
         }
@@ -319,7 +323,7 @@ protected void checkCacheLoad() {
     protected void checkCacheLoadWithSql() {
         IgniteCache<Object, Object> c1 = grid().cache(CACHE_NAME);
 
-        c1.loadCache(null, "org.apache.ignite.cache.store.jdbc.model.PersonKey", "select id, org_id, name, birthday from Person");
+        c1.loadCache(null, "org.apache.ignite.cache.store.jdbc.model.PersonKey", "select id, org_id, name, birthday, gender from Person");
 
         assertEquals(PERSON_CNT, c1.size());
     }
@@ -397,7 +401,9 @@ private void checkPutRemove() throws Exception {
 
         Connection conn = getConnection();
         try {
-            PreparedStatement stmt = conn.prepareStatement("SELECT ID, ORG_ID, BIRTHDAY, NAME FROM PERSON WHERE ID = ?");
+            Random rnd = new Random();
+
+            PreparedStatement stmt = conn.prepareStatement("SELECT ID, ORG_ID, BIRTHDAY, NAME, GENDER FROM PERSON WHERE ID = ?");
 
             stmt.setInt(1, -1);
 
@@ -408,8 +414,9 @@ private void checkPutRemove() throws Exception {
             U.closeQuiet(rs);
 
             Date testDate = Date.valueOf("2001-05-05");
+            Gender testGender = Gender.random();
 
-            Person val = new Person(-1, -2, testDate, "Person-to-test-put-insert", 999);
+            Person val = new Person(-1, -2, testDate, "Person-to-test-put-insert", 999, testGender);
 
             Object key = builtinKeys ? Integer.valueOf(-1) : new PersonKey(-1);
 
@@ -424,6 +431,7 @@ private void checkPutRemove() throws Exception {
             assertEquals(-2, rs.getInt(2));
             assertEquals(testDate, rs.getDate(3));
             assertEquals("Person-to-test-put-insert", rs.getString(4));
+            assertEquals(testGender.toString(), rs.getString(5));
 
             assertFalse("Unexpected more data in result set", rs.next());
 
@@ -432,7 +440,7 @@ private void checkPutRemove() throws Exception {
             // Test put-update.
             testDate = Date.valueOf("2016-04-04");
 
-            c1.put(key, new Person(-1, -3, testDate, "Person-to-test-put-update", 999));
+            c1.put(key, new Person(-1, -3, testDate, "Person-to-test-put-update", 999, testGender));
 
             rs = stmt.executeQuery();
 
@@ -442,6 +450,7 @@ private void checkPutRemove() throws Exception {
             assertEquals(-3, rs.getInt(2));
             assertEquals(testDate, rs.getDate(3));
             assertEquals("Person-to-test-put-update", rs.getString(4));
+            assertEquals(testGender.toString(), rs.getString(5));
 
             assertFalse("Unexpected more data in result set", rs.next());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
index 4a0b1daf646d1..849cab79a03be 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
@@ -25,6 +25,7 @@
 import java.sql.Types;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import javax.cache.integration.CacheWriterException;
@@ -233,6 +234,8 @@ public CacheJdbcPojoStoreTest() throws Exception {
     public void testLoadCache() throws Exception {
         Connection conn = store.openConnection(false);
 
+        Random rnd = new Random();
+
         PreparedStatement orgStmt = conn.prepareStatement("INSERT INTO Organization(id, name, city) VALUES (?, ?, ?)");
 
         for (int i = 0; i < ORGANIZATION_CNT; i++) {
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreAbstractMultithreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreAbstractMultithreadedSelfTest.java
index e831445c896da..f1a321ba01216 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreAbstractMultithreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreAbstractMultithreadedSelfTest.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheTypeMetadata;
+import org.apache.ignite.cache.store.jdbc.model.Gender;
 import org.apache.ignite.cache.store.jdbc.model.Organization;
 import org.apache.ignite.cache.store.jdbc.model.OrganizationKey;
 import org.apache.ignite.cache.store.jdbc.model.Person;
@@ -208,7 +209,7 @@ public void testMultithreadedPut() throws Exception {
                         cache.put(new OrganizationKey(id), new Organization(id, "Name" + id, "City" + id));
                     else
                         cache.put(new PersonKey(id), new Person(id, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + id, 1));
+                            new Date(System.currentTimeMillis()), "Name" + id, 1, Gender.random()));
                 }
 
                 return null;
@@ -228,7 +229,7 @@ public void testMultithreadedPut() throws Exception {
                         cache.putIfAbsent(new OrganizationKey(id), new Organization(id, "Name" + id, "City" + id));
                     else
                         cache.putIfAbsent(new PersonKey(id), new Person(id, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + id, i));
+                            new Date(System.currentTimeMillis()), "Name" + id, i, Gender.random()));
                 }
 
                 return null;
@@ -268,7 +269,7 @@ public void testMultithreadedPutAll() throws Exception {
                             map.put(new OrganizationKey(id), new Organization(id, "Name" + id, "City" + id));
                         else
                             map.put(new PersonKey(id), new Person(id, rnd.nextInt(),
-                                new Date(System.currentTimeMillis()), "Name" + id, 1));
+                                new Date(System.currentTimeMillis()), "Name" + id, 1, Gender.random()));
                     }
 
                     IgniteCache<Object, Object> cache = jcache();
@@ -294,11 +295,11 @@ public void testMultithreadedExplicitTx() throws Exception {
 
                     try (Transaction tx = grid().transactions().txStart()) {
                         cache.put(new PersonKey(1), new Person(1, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + 1, 1));
+                            new Date(System.currentTimeMillis()), "Name" + 1, 1, Gender.random()));
                         cache.put(new PersonKey(2), new Person(2, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + 2, 2));
+                            new Date(System.currentTimeMillis()), "Name" + 2, 2, Gender.random()));
                         cache.put(new PersonKey(3), new Person(3, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + 3, 3));
+                            new Date(System.currentTimeMillis()), "Name" + 3, 3, Gender.random()));
 
                         cache.get(new PersonKey(1));
                         cache.get(new PersonKey(4));
@@ -306,9 +307,9 @@ public void testMultithreadedExplicitTx() throws Exception {
                         Map<PersonKey, Person> map =  U.newHashMap(2);
 
                         map.put(new PersonKey(5), new Person(5, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + 5, 5));
+                            new Date(System.currentTimeMillis()), "Name" + 5, 5, Gender.random()));
                         map.put(new PersonKey(6), new Person(6, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + 6, 6));
+                            new Date(System.currentTimeMillis()), "Name" + 6, 6, Gender.random()));
 
                         cache.putAll(map);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Gender.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Gender.java
new file mode 100644
index 0000000000000..8ddb0e2a3670e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Gender.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.jdbc.model;
+
+import java.io.Serializable;
+import java.util.Random;
+
+/**
+ * Person gender enum.
+ */
+public enum Gender implements Serializable {
+    /** */
+    MALE,
+    /** */
+    FEMALE;
+
+    /** */
+    private static final Random RAND = new Random();
+
+    /**
+     * Used for testing purposes.
+     */
+    public static Gender random() {
+        return values()[RAND.nextInt(values().length)];
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java
index 52ddfc8ecdcbb..89258b403de23 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java
@@ -44,6 +44,9 @@ public class Person implements Serializable {
     /** Value for salary. */
     private Integer salary;
 
+    /** Value of person gender. */
+    private Gender gender;
+
     /**
      * Empty constructor.
      */
@@ -59,13 +62,15 @@ public Person(
         Integer orgId,
         Date birthday,
         String name,
-        Integer salary
+        Integer salary,
+        Gender gender
     ) {
         this.id = id;
         this.orgId = orgId;
         this.birthday = birthday;
         this.name = name;
         this.salary = salary;
+        this.gender = gender;
     }
 
     /**
@@ -159,6 +164,24 @@ public void setSalary(Integer salary) {
         this.salary = salary;
     }
 
+    /**
+     * Gets gender.
+     *
+     * @return Gender.
+     */
+    public Gender getGender() {
+        return gender;
+    }
+
+    /**
+     * Sets gender.
+     *
+     * @param gender New value for gender.
+     */
+    public void setGender(Gender gender) {
+        this.gender = gender;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean equals(Object o) {
         if (this == o)
@@ -178,6 +201,9 @@ public void setSalary(Integer salary) {
         if (name != null ? !name.equals(that.name) : that.name != null)
             return false;
 
+        if (gender != null ? !gender.equals(that.gender) : that.gender != null)
+            return false;
+
         return true;
     }
 
@@ -189,6 +215,8 @@ public void setSalary(Integer salary) {
 
         res = 31 * res + (name != null ? name.hashCode() : 0);
 
+        res = 31 * res + (gender != null ? gender.hashCode() : 0);
+
         return res;
     }
 
@@ -198,6 +226,7 @@ public void setSalary(Integer salary) {
             ", orgId=" + orgId +
             ", birthday=" + (birthday == null ? null : birthday.getTime()) +
             ", name=" + name +
+            ", gender=" + gender +
             "]";
     }
 }
diff --git a/modules/spring/src/test/config/jdbc-pojo-store-builtin.xml b/modules/spring/src/test/config/jdbc-pojo-store-builtin.xml
index dfaf828fa8d22..bfb109c035d94 100644
--- a/modules/spring/src/test/config/jdbc-pojo-store-builtin.xml
+++ b/modules/spring/src/test/config/jdbc-pojo-store-builtin.xml
@@ -151,6 +151,14 @@
                                                     <property name="javaFieldName" value="orgId"/>
                                                     <property name="javaFieldType" value="java.lang.Integer"/>
                                                 </bean>
+                                                <bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
+                                                    <property name="databaseFieldName" value="GENDER"/>
+                                                    <property name="databaseFieldType">
+                                                        <util:constant static-field="java.sql.Types.VARCHAR"/>
+                                                    </property>
+                                                    <property name="javaFieldName" value="gender"/>
+                                                    <property name="javaFieldType" value="org.apache.ignite.cache.store.jdbc.model.Gender"/>
+                                                </bean>
                                             </list>
                                         </property>
                                     </bean>
diff --git a/modules/spring/src/test/config/jdbc-pojo-store-obj.xml b/modules/spring/src/test/config/jdbc-pojo-store-obj.xml
index 9bc99776a5bec..40a14dc4f4fda 100644
--- a/modules/spring/src/test/config/jdbc-pojo-store-obj.xml
+++ b/modules/spring/src/test/config/jdbc-pojo-store-obj.xml
@@ -151,6 +151,14 @@
                                                     <property name="javaFieldName" value="orgId"/>
                                                     <property name="javaFieldType" value="java.lang.Integer"/>
                                                 </bean>
+                                                <bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
+                                                    <property name="databaseFieldName" value="GENDER"/>
+                                                    <property name="databaseFieldType">
+                                                        <util:constant static-field="java.sql.Types.VARCHAR"/>
+                                                    </property>
+                                                    <property name="javaFieldName" value="gender"/>
+                                                    <property name="javaFieldType" value="org.apache.ignite.cache.store.jdbc.model.Gender"/>
+                                                </bean>
                                             </list>
                                         </property>
                                     </bean>

From 6f6ff397c8a2fb3f1041958a3338294cb44af998 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 25 Jan 2017 16:48:05 +0700
Subject: [PATCH 125/446] IGNITE-1596 Fixed version sort.

(cherry picked from commit 128ba07)
---
 modules/web-console/backend/app/agent.js | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index 791ea50ed58b9..961253f148a70 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -650,14 +650,14 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo)
                         const bParts = b.split('.');
 
                         for (let i = 0; i < aParts.length; ++i) {
-                            if (bParts.length === i)
-                                return 1;
+                            if (aParts[i] !== bParts[i])
+                                return aParts[i] < bParts[i] ? 1 : -1;
+                        }
 
-                            if (aParts[i] === aParts[i])
-                                continue;
+                        if (aParts.length === bParts.length)
+                            return 0;
 
-                            return aParts[i] > bParts[i] ? 1 : -1;
-                        }
+                        return aParts.length < bParts.length ? 1 : -1;
                     }));
 
                     // Latest version of agent distribution.

From faa20530a707014ac34477cba8adaaa4b67de1bb Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 25 Jan 2017 16:48:05 +0700
Subject: [PATCH 126/446] IGNITE-1596 Fixed version sort.

(cherry picked from commit 128ba07)
---
 modules/web-console/backend/app/agent.js | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index 791ea50ed58b9..961253f148a70 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -650,14 +650,14 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo)
                         const bParts = b.split('.');
 
                         for (let i = 0; i < aParts.length; ++i) {
-                            if (bParts.length === i)
-                                return 1;
+                            if (aParts[i] !== bParts[i])
+                                return aParts[i] < bParts[i] ? 1 : -1;
+                        }
 
-                            if (aParts[i] === aParts[i])
-                                continue;
+                        if (aParts.length === bParts.length)
+                            return 0;
 
-                            return aParts[i] > bParts[i] ? 1 : -1;
-                        }
+                        return aParts.length < bParts.length ? 1 : -1;
                     }));
 
                     // Latest version of agent distribution.

From 9d9e61fc9db0f79daaa87b7ec2c89c652bd3beaa Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 25 Jan 2017 14:37:57 +0300
Subject: [PATCH 127/446] Added benchmark
 IgnitePutOffHeapIndexedValue8Benchmark.

---
 .../yardstick/config/ignite-base-config.xml   | 25 ++++++++++++++++
 ...gnitePutOffHeapIndexedValue8Benchmark.java | 30 +++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutOffHeapIndexedValue8Benchmark.java

diff --git a/modules/yardstick/config/ignite-base-config.xml b/modules/yardstick/config/ignite-base-config.xml
index 615cb422f3eae..e363674eeb16c 100644
--- a/modules/yardstick/config/ignite-base-config.xml
+++ b/modules/yardstick/config/ignite-base-config.xml
@@ -126,6 +126,31 @@
                     </property>
                 </bean>
 
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="name" value="atomic-index-offheap"/>
+
+                    <property name="cacheMode" value="PARTITIONED"/>
+
+                    <property name="atomicityMode" value="ATOMIC"/>
+
+                    <property name="swapEnabled" value="false"/>
+
+                    <property name="memoryMode" value="OFFHEAP_TIERED"/>
+
+                    <property name="indexedTypes">
+                        <list>
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Person1</value>
+
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Person2</value>
+
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Person8</value>
+                        </list>
+                    </property>
+                </bean>
+
                 <bean class="org.apache.ignite.configuration.CacheConfiguration">
                     <property name="name" value="atomic-index-with-eviction"/>
 
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutOffHeapIndexedValue8Benchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutOffHeapIndexedValue8Benchmark.java
new file mode 100644
index 0000000000000..85202cddb279e
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutOffHeapIndexedValue8Benchmark.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import org.apache.ignite.IgniteCache;
+
+/**
+ * Ignite benchmark that performs put operations for entity with indexed fields.
+ */
+public class IgnitePutOffHeapIndexedValue8Benchmark extends IgnitePutIndexedValue8Benchmark {
+    /** {@inheritDoc} */
+    @Override protected IgniteCache<Integer, Object> cache() {
+        return ignite().cache("atomic-index-offheap");
+    }
+}

From f5e601e2973bfa81593241e55e4b6f97c0e55c3c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 26 Jan 2017 10:18:34 +0300
Subject: [PATCH 128/446] IGNITE-4036 - Fix. Near cache is not expired together
 with corresponding server cache

---
 .../processors/cache/EntryGetResult.java      |  40 +++++-
 .../cache/EntryGetWithTtlResult.java          |  58 +++++++++
 .../processors/cache/GridCacheAdapter.java    |  69 +++++-----
 .../processors/cache/GridCacheContext.java    | 122 ++++++++++++++----
 .../processors/cache/GridCacheEntryEx.java    |   2 +-
 .../processors/cache/GridCacheMapEntry.java   |  29 +++--
 .../distributed/dht/GridDhtCacheAdapter.java  |   4 +-
 .../distributed/dht/GridDhtGetFuture.java     |  24 ++--
 .../dht/GridDhtGetSingleFuture.java           |  24 ++--
 .../dht/GridPartitionedGetFuture.java         |  19 ++-
 .../dht/GridPartitionedSingleGetFuture.java   |   7 +-
 .../dht/atomic/GridDhtAtomicCache.java        |  12 +-
 .../dht/colocated/GridDhtColocatedCache.java  |  15 ++-
 .../distributed/near/GridNearGetFuture.java   |   9 +-
 .../distributed/near/GridNearTxLocal.java     |   8 +-
 .../local/atomic/GridLocalAtomicCache.java    |  13 +-
 .../transactions/IgniteTxLocalAdapter.java    |  62 ++++++---
 .../cache/GridCacheTestEntryEx.java           |   2 +-
 .../IgniteCacheExpiryPolicyAbstractTest.java  |  44 ++++++-
 19 files changed, 411 insertions(+), 152 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetWithTtlResult.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetResult.java
index a34ddae6d10af..9d064484f1962 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetResult.java
@@ -24,7 +24,7 @@
  */
 public class EntryGetResult {
     /** */
-    private final CacheObject val;
+    private Object val;
 
     /** */
     private final GridCacheVersion ver;
@@ -35,18 +35,34 @@ public class EntryGetResult {
     /**
      * @param val Value.
      * @param ver Version.
+     * @param reserved Reserved flag.
      */
-    EntryGetResult(CacheObject val, GridCacheVersion ver, boolean reserved) {
+    public EntryGetResult(Object val, GridCacheVersion ver, boolean reserved) {
         this.val = val;
         this.ver = ver;
         this.reserved = reserved;
     }
 
+    /**
+     * @param val Value.
+     * @param ver Version.
+     */
+    public EntryGetResult(Object val, GridCacheVersion ver) {
+        this(val, ver, false);
+    }
+
     /**
      * @return Value.
      */
-    public CacheObject value() {
-        return val;
+    public <T> T value() {
+        return (T)val;
+    }
+
+    /**
+     * @param val Value.
+     */
+    public void value(Object val) {
+        this.val = val;
     }
 
     /**
@@ -57,9 +73,23 @@ public GridCacheVersion version() {
     }
 
     /**
-     * @return Reserved flag,
+     * @return Reserved flag.
      */
     public boolean reserved() {
         return reserved;
     }
+
+    /**
+     * @return Entry expire time.
+     */
+    public long expireTime() {
+        return 0L;
+    }
+
+    /**
+     * @return Entry time to live.
+     */
+    public long ttl() {
+        return 0L;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetWithTtlResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetWithTtlResult.java
new file mode 100644
index 0000000000000..fddf16e1de43e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/EntryGetWithTtlResult.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+
+/**
+ *
+ */
+public class EntryGetWithTtlResult extends EntryGetResult {
+    /** */
+    private final long expireTime;
+
+    /** */
+    private final long ttl;
+
+    /**
+     * @param val Value.
+     * @param ver Version.
+     * @param reserved Reserved flag.
+     * @param expireTime Entry expire time.
+     * @param ttl Entry time to live.
+     */
+    public EntryGetWithTtlResult(Object val, GridCacheVersion ver, boolean reserved, long expireTime, long ttl) {
+        super(val, ver, reserved);
+        this.expireTime = expireTime;
+        this.ttl = ttl;
+    }
+
+    /**
+     * @return Entry expire time.
+     */
+    @Override public long expireTime() {
+        return expireTime;
+    }
+
+    /**
+     * @return Entry time to live.
+     */
+    @Override public long ttl() {
+        return ttl;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index dc8f0308a0099..11bf34b221bb1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1421,12 +1421,13 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
         if (keepBinary)
             key = (K)ctx.toCacheKeyObject(key);
 
-        T2<V, GridCacheVersion> t = (T2<V, GridCacheVersion>)get(key, !keepBinary, true);
+        EntryGetResult t
+            = (EntryGetResult)get(key, !keepBinary, true);
 
         CacheEntry<K, V> val = t != null ? new CacheEntryImplEx<>(
             keepBinary ? (K)ctx.unwrapBinaryIfNeeded(key, true, false) : key,
-            t.get1(),
-            t.get2())
+            (V)t.value(),
+            t.version())
             : null;
 
         if (ctx.config().getInterceptor() != null) {
@@ -1434,7 +1435,7 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
 
             V val0 = (V)ctx.config().getInterceptor().onGet(key, t != null ? val.getValue() : null);
 
-            val = (val0 != null) ? new CacheEntryImplEx<>(key, val0, t != null ? t.get2() : null) : null;
+            val = (val0 != null) ? new CacheEntryImplEx<>(key, val0, t != null ? t.version() : null) : null;
         }
 
         if (statsEnabled)
@@ -1484,29 +1485,29 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
 
         final K key0 = keepBinary ? (K)ctx.toCacheKeyObject(key) : key;
 
-        IgniteInternalFuture<T2<V, GridCacheVersion>> fut =
-            (IgniteInternalFuture<T2<V, GridCacheVersion>>)getAsync(key0, !keepBinary, true);
+        IgniteInternalFuture<EntryGetResult> fut =
+            (IgniteInternalFuture<EntryGetResult>)getAsync(key0, !keepBinary, true);
 
         final boolean intercept = ctx.config().getInterceptor() != null;
 
         IgniteInternalFuture<CacheEntry<K, V>> fr = fut.chain(
-            new CX1<IgniteInternalFuture<T2<V, GridCacheVersion>>, CacheEntry<K, V>>() {
-                @Override public CacheEntry<K, V> applyx(IgniteInternalFuture<T2<V, GridCacheVersion>> f)
+            new CX1<IgniteInternalFuture<EntryGetResult>, CacheEntry<K, V>>() {
+                @Override public CacheEntry<K, V> applyx(IgniteInternalFuture<EntryGetResult> f)
                     throws IgniteCheckedException {
-                    T2<V, GridCacheVersion> t = f.get();
+                    EntryGetResult t = f.get();
 
                 K key = keepBinary ? (K)ctx.unwrapBinaryIfNeeded(key0, true, false) : key0;
 
                 CacheEntry val = t != null ? new CacheEntryImplEx<>(
                     key,
-                    t.get1(),
-                    t.get2())
+                    t.value(),
+                    t.version())
                     : null;
 
                 if (intercept) {
                     V val0 = (V)ctx.config().getInterceptor().onGet(key, t != null ? val.getValue() : null);
 
-                    return val0 != null ? new CacheEntryImplEx(key, val0, t != null ? t.get2() : null) : null;
+                    return val0 != null ? new CacheEntryImplEx(key, val0, t != null ? t.version() : null) : null;
                 }
                 else
                     return val;
@@ -1514,7 +1515,7 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
         });
 
         if (statsEnabled)
-            fut.listen(new UpdateGetTimeStatClosure<T2<V, GridCacheVersion>>(metrics0(), start));
+            fut.listen(new UpdateGetTimeStatClosure<EntryGetResult>(metrics0(), start));
 
         return fr;
     }
@@ -1547,15 +1548,15 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
 
         long start = statsEnabled ? System.nanoTime() : 0L;
 
-        Map<K, T2<V, GridCacheVersion>> map = (Map<K, T2<V, GridCacheVersion>>)getAll(keys, !ctx.keepBinary(), true);
+        Map<K, EntryGetResult> map = (Map<K, EntryGetResult>)getAll(keys, !ctx.keepBinary(), true);
 
         Collection<CacheEntry<K, V>> res = new HashSet<>();
 
         if (ctx.config().getInterceptor() != null)
             res = interceptGetEntries(keys, map);
         else
-            for (Map.Entry<K, T2<V, GridCacheVersion>> e : map.entrySet())
-                res.add(new CacheEntryImplEx<>(e.getKey(), e.getValue().get1(), e.getValue().get2()));
+            for (Map.Entry<K, EntryGetResult> e : map.entrySet())
+                res.add(new CacheEntryImplEx<>(e.getKey(), (V)e.getValue().value(), e.getValue().version()));
 
         if (statsEnabled)
             metrics0().addGetTimeNanos(System.nanoTime() - start);
@@ -1595,24 +1596,24 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
 
-        IgniteInternalFuture<Map<K, T2<V, GridCacheVersion>>> fut =
-            (IgniteInternalFuture<Map<K, T2<V, GridCacheVersion>>>)
+        IgniteInternalFuture<Map<K, EntryGetResult>> fut =
+            (IgniteInternalFuture<Map<K, EntryGetResult>>)
                 ((IgniteInternalFuture)getAllAsync(keys, !ctx.keepBinary(), true));
 
         final boolean intercept = ctx.config().getInterceptor() != null;
 
         IgniteInternalFuture<Collection<CacheEntry<K, V>>> rf =
-            fut.chain(new CX1<IgniteInternalFuture<Map<K, T2<V, GridCacheVersion>>>, Collection<CacheEntry<K, V>>>() {
+            fut.chain(new CX1<IgniteInternalFuture<Map<K, EntryGetResult>>, Collection<CacheEntry<K, V>>>() {
                 @Override public Collection<CacheEntry<K, V>> applyx(
-                    IgniteInternalFuture<Map<K, T2<V, GridCacheVersion>>> f) throws IgniteCheckedException {
+                    IgniteInternalFuture<Map<K, EntryGetResult>> f) throws IgniteCheckedException {
                     if (intercept)
                         return interceptGetEntries(keys, f.get());
                     else {
                         Map<K, CacheEntry<K, V>> res = U.newHashMap(f.get().size());
 
-                        for (Map.Entry<K, T2<V, GridCacheVersion>> e : f.get().entrySet())
+                        for (Map.Entry<K, EntryGetResult> e : f.get().entrySet())
                             res.put(e.getKey(),
-                                new CacheEntryImplEx<>(e.getKey(), e.getValue().get1(), e.getValue().get2()));
+                                new CacheEntryImplEx<>(e.getKey(), (V)e.getValue().value(), e.getValue().version()));
 
                         return res.values();
                     }
@@ -1620,7 +1621,7 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
             });
 
         if (statsEnabled)
-            fut.listen(new UpdateGetTimeStatClosure<Map<K, T2<V, GridCacheVersion>>>(metrics0(), start));
+            fut.listen(new UpdateGetTimeStatClosure<Map<K, EntryGetResult>>(metrics0(), start));
 
         return rf;
     }
@@ -1675,7 +1676,7 @@ private Map<K, V> interceptGet(@Nullable Collection<? extends K> keys, Map<K, V>
      */
     @SuppressWarnings("IfMayBeConditional")
     private Collection<CacheEntry<K, V>> interceptGetEntries(
-        @Nullable Collection<? extends K> keys, Map<K, T2<V, GridCacheVersion>> map) {
+        @Nullable Collection<? extends K> keys, Map<K, EntryGetResult> map) {
         Map<K, CacheEntry<K, V>> res;
 
         if (F.isEmpty(keys)) {
@@ -1690,11 +1691,11 @@ private Collection<CacheEntry<K, V>> interceptGetEntries(
 
         assert interceptor != null;
 
-        for (Map.Entry<K, T2<V, GridCacheVersion>> e : map.entrySet()) {
-            V val = interceptor.onGet(e.getKey(), e.getValue().get1());
+        for (Map.Entry<K, EntryGetResult> e : map.entrySet()) {
+            V val = interceptor.onGet(e.getKey(), (V)e.getValue().value());
 
             if (val != null)
-                res.put(e.getKey(), new CacheEntryImplEx<>(e.getKey(), val, e.getValue().get2()));
+                res.put(e.getKey(), new CacheEntryImplEx<>(e.getKey(), val, e.getValue().version()));
         }
 
         if (map.size() != keys.size()) { // Not all requested keys were in cache.
@@ -1976,12 +1977,12 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                             if (res != null) {
                                 ctx.addResult(map,
                                     key,
-                                    res.value(),
+                                    res,
                                     skipVals,
                                     keepCacheObjects,
                                     deserializeBinary,
                                     true,
-                                    needVer ? res.version() : null);
+                                    needVer);
 
                                 if (tx == null || (!tx.implicit() && tx.isolation() == READ_COMMITTED))
                                     ctx.evicts().touch(entry, topVer);
@@ -2025,7 +2026,7 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                             GridCacheEntryEx entry = entryEx(key);
 
                                             try {
-                                                T2<CacheObject, GridCacheVersion> verVal = entry.versionedValue(
+                                                EntryGetResult verVal = entry.versionedValue(
                                                     cacheVal,
                                                     res.version(),
                                                     null,
@@ -2035,19 +2036,19 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                                 if (log.isDebugEnabled())
                                                     log.debug("Set value loaded from store into entry [" +
                                                         "oldVer=" + res.version() +
-                                                        ", newVer=" + verVal.get2() + ", " +
+                                                        ", newVer=" + verVal.version() + ", " +
                                                         "entry=" + entry + ']');
 
                                                 // Don't put key-value pair into result map if value is null.
-                                                if (verVal.get1() != null) {
+                                                if (verVal.value() != null) {
                                                     ctx.addResult(map,
                                                         key,
-                                                        verVal.get1(),
+                                                        verVal,
                                                         skipVals,
                                                         keepCacheObjects,
                                                         deserializeBinary,
                                                         true,
-                                                        needVer ? verVal.get2() : null);
+                                                        needVer);
                                                 }
 
                                                 if (tx0 == null || (!tx0.implicit() &&
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 424e325dd07f2..6322f9f03550a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -90,7 +90,6 @@
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.GPC;
@@ -1895,7 +1894,65 @@ public <K1, V1> void addResult(Map<K1, V1> map,
         boolean keepCacheObjects,
         boolean deserializeBinary,
         boolean cpy,
-        final GridCacheVersion ver) {
+        final GridCacheVersion ver,
+        final long expireTime,
+        final long ttl) {
+        // Creates EntryGetResult
+        addResult(map, key, val, skipVals, keepCacheObjects, deserializeBinary, cpy, null,
+            ver, expireTime, ttl, ver != null);
+    }
+
+    /**
+     * @param map Map.
+     * @param key Key.
+     * @param getRes EntryGetResult.
+     * @param skipVals Skip values.
+     * @param keepCacheObjects Keep CacheObject.
+     * @param deserializeBinary Deserialize binary flag.
+     * @param cpy Copy flag.
+     * @param needVer Need version flag.
+     */
+    @SuppressWarnings("unchecked")
+    public <K1, V1> void addResult(Map<K1, V1> map,
+        KeyCacheObject key,
+        EntryGetResult getRes,
+        boolean skipVals,
+        boolean keepCacheObjects,
+        boolean deserializeBinary,
+        boolean cpy,
+        boolean needVer) {
+        // Uses getRes as result.
+        addResult(map, key, getRes.<CacheObject>value(), skipVals, keepCacheObjects, deserializeBinary, cpy, getRes,
+            null, 0, 0, needVer);
+    }
+
+    /**
+     * @param map Map.
+     * @param key Key.
+     * @param val Value.
+     * @param skipVals Skip values.
+     * @param keepCacheObjects Keep CacheObject.
+     * @param deserializeBinary Deserialize binary.
+     * @param cpy Copy flag.
+     * @param getRes EntryGetResult.
+     * @param ver Version.
+     * @param expireTime Entry expire time.
+     * @param ttl Entry TTL.
+     * @param needVer Need version flag.
+     */
+    @SuppressWarnings("unchecked")
+    public <K1, V1> void addResult(Map<K1, V1> map,
+        KeyCacheObject key,
+        CacheObject val,
+        boolean skipVals,
+        boolean keepCacheObjects,
+        boolean deserializeBinary,
+        boolean cpy,
+        @Nullable EntryGetResult getRes,
+        final GridCacheVersion ver,
+        final long expireTime,
+        final long ttl,
+        boolean needVer) {
         assert key != null;
         assert val != null || skipVals;
 
@@ -1907,32 +1964,53 @@ public <K1, V1> void addResult(Map<K1, V1> map,
             assert key0 != null : key;
             assert val0 != null : val;
 
-            map.put((K1)key0, ver != null ? (V1)new T2<>(val0, ver) : (V1)val0);
+            V1 v = createValue(ver, expireTime, ttl, val0, getRes, needVer);
+
+            map.put((K1)key0, v);
+        }
+        else {
+            Object val0 = skipVals ? true : val;
+
+            V1 v = createValue(ver, expireTime, ttl, val0, getRes, needVer);
+
+            map.put((K1)key, v);
         }
-        else
-            map.put((K1)key,
-                (V1)(ver != null ?
-                    (V1)new T2<>(skipVals ? true : val, ver) :
-                    skipVals ? true : val));
     }
 
     /**
-     * @param map Map.
-     * @param key Key.
+     * Creates new EntryGetResult or uses existing one.
+     *
+     * @param ver Version.
+     * @param expireTime Entry expire time.
+     * @param ttl Entry TTL.
      * @param val Value.
-     * @param skipVals Skip values flag.
-     * @param keepCacheObjects Keep cache objects flag.
-     * @param deserializeBinary Deserialize binary flag.
-     * @param cpy Copy flag.
+     * @param getRes EntryGetResult
+     * @param needVer Need version flag.
+     * @return EntryGetResult or value.
      */
-    public <K1, V1> void addResult(Map<K1, V1> map,
-        KeyCacheObject key,
-        CacheObject val,
-        boolean skipVals,
-        boolean keepCacheObjects,
-        boolean deserializeBinary,
-        boolean cpy) {
-        addResult(map, key, val, skipVals, keepCacheObjects, deserializeBinary, cpy, null);
+    @SuppressWarnings("unchecked")
+    private <V1> V1 createValue(final GridCacheVersion ver,
+        final long expireTime,
+        final long ttl,
+        final Object val,
+        @Nullable final EntryGetResult getRes,
+        final boolean needVer) {
+        final V1 v;
+
+        if (!needVer)
+            v = (V1) val;
+        else if (getRes == null) {
+            v = expireTime != 0 || ttl != 0
+                ? (V1)new EntryGetWithTtlResult(val, ver, false, expireTime, ttl)
+                : (V1)new EntryGetResult(val, ver, false);
+        }
+        else {
+            getRes.value(val);
+
+            v = (V1)getRes;
+        }
+
+        return v;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index 3c42d5360a4b1..ccd22850d9eb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -761,7 +761,7 @@ public <K, V> GridCacheVersionedEntryEx<K, V> versionedEntry(final boolean keepB
      * @throws IgniteCheckedException If index could not be updated.
      * @throws GridCacheEntryRemovedException If entry was removed.
      */
-    public T2<CacheObject, GridCacheVersion> versionedValue(CacheObject val,
+    public EntryGetResult versionedValue(CacheObject val,
         @Nullable GridCacheVersion curVer,
         @Nullable GridCacheVersion newVer,
         @Nullable ReaderArguments readerArgs,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 7e26719d99076..58b4ae3130c9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -69,7 +69,6 @@
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.T3;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -975,7 +974,7 @@ private Object innerGet0(
                 assert !deferred;
 
                 // If return value is consistent, then done.
-                res = retVer ? new EntryGetResult(ret, resVer, false) : ret;
+                res = retVer ? entryGetResult(ret, resVer, false) : ret;
             }
             else if (reserveForLoad && !obsolete) {
                 assert !readThrough;
@@ -986,7 +985,7 @@ else if (reserveForLoad && !obsolete) {
                 if (reserve)
                     flags |= IS_EVICT_DISABLED;
 
-                res = new EntryGetResult(null, resVer, reserve);
+                res = entryGetResult(null, resVer, reserve);
             }
         }
 
@@ -1092,6 +1091,20 @@ else if (tx.dht()) {
         return ret;
     }
 
+    /**
+     * Creates EntryGetResult or EntryGetWithTtlResult if expire time information exists.
+     *
+     * @param val Value.
+     * @param ver Version.
+     * @param reserve Reserve flag.
+     * @return EntryGetResult.
+     */
+    private EntryGetResult entryGetResult(CacheObject val, GridCacheVersion ver, boolean reserve) {
+        return extras == null || extras.expireTime() == 0
+            ? new EntryGetResult(val, ver, reserve)
+            : new EntryGetWithTtlResult(val, ver, reserve, rawExpireTime(), rawTtl());
+    }
+
     /** {@inheritDoc} */
     @SuppressWarnings({"unchecked", "TooBroadScope"})
     @Nullable @Override public final CacheObject innerReload()
@@ -3382,7 +3395,7 @@ int hash() {
     }
 
     /**
-     * TODO: GG-4009: do we need to generate event and invalidate value?
+     * TODO: IGNITE-3500: do we need to generate event and invalidate value?
      *
      * @return {@code true} if expired.
      * @throws IgniteCheckedException In case of failure.
@@ -3621,7 +3634,7 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
     }
 
     /** {@inheritDoc} */
-    @Override public synchronized T2<CacheObject, GridCacheVersion> versionedValue(CacheObject val,
+    @Override public synchronized EntryGetResult versionedValue(CacheObject val,
         GridCacheVersion curVer,
         GridCacheVersion newVer,
         @Nullable ReaderArguments readerArgs,
@@ -3637,7 +3650,7 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
                 GridCacheMvcc mvcc = mvccExtras();
 
                 if (mvcc != null && !mvcc.isEmpty())
-                    return new T2<>(this.val, ver);
+                    return entryGetResult(this.val, ver, false);
 
                 if (newVer == null)
                     newVer = cctx.versions().next();
@@ -3671,13 +3684,13 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
                 // Version does not change for load ops.
                 update(val, expTime, ttl, newVer, true);
 
-                return new T2<>(val, newVer);
+                return entryGetResult(val, newVer, false);
             }
 
             assert !evictionDisabled() : this;
         }
 
-        return new T2<>(this.val, ver);
+        return entryGetResult(this.val, ver, false);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index bc34df781c0a2..dcd379a040bd0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -41,6 +41,7 @@
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheOperationContext;
 import org.apache.ignite.internal.processors.cache.CachePeekModes;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheClearAllRunnable;
 import org.apache.ignite.internal.processors.cache.GridCacheConcurrentMap;
@@ -74,7 +75,6 @@
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.CI3;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -648,7 +648,7 @@ else if (log.isDebugEnabled())
      * @param canRemap Can remap flag.
      * @return Get future.
      */
-    IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> getDhtAllAsync(
+    IgniteInternalFuture<Map<KeyCacheObject, EntryGetResult>> getDhtAllAsync(
         Collection<KeyCacheObject> keys,
         @Nullable final ReaderArguments readerArgs,
         boolean readThrough,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
index 3bf44895a62a6..8b92e9fb1daf0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
@@ -46,7 +47,6 @@
 import org.apache.ignite.internal.util.typedef.C2;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
@@ -390,7 +390,7 @@ private IgniteInternalFuture<Collection<GridCacheEntryInfo>> getAsync(
                 txFut.markInitialized();
         }
 
-        IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> fut;
+        IgniteInternalFuture<Map<KeyCacheObject, EntryGetResult>> fut;
 
         if (txFut == null || txFut.isDone()) {
             fut = cache().getDhtAllAsync(
@@ -411,8 +411,8 @@ private IgniteInternalFuture<Collection<GridCacheEntryInfo>> getAsync(
             // transactions to complete.
             fut = new GridEmbeddedFuture<>(
                 txFut,
-                new C2<Boolean, Exception, IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>>>() {
-                    @Override public IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> apply(Boolean b, Exception e) {
+                new C2<Boolean, Exception, IgniteInternalFuture<Map<KeyCacheObject, EntryGetResult>>>() {
+                    @Override public IgniteInternalFuture<Map<KeyCacheObject, EntryGetResult>> apply(Boolean b, Exception e) {
                         if (e != null)
                             throw new GridClosureException(e);
 
@@ -438,9 +438,9 @@ private IgniteInternalFuture<Collection<GridCacheEntryInfo>> getAsync(
         }
 
         return new GridEmbeddedFuture<>(
-            new C2<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>, Exception, Collection<GridCacheEntryInfo>>() {
+            new C2<Map<KeyCacheObject, EntryGetResult>, Exception, Collection<GridCacheEntryInfo>>() {
                 @Override public Collection<GridCacheEntryInfo> apply(
-                    Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>> map, Exception e
+                    Map<KeyCacheObject, EntryGetResult> map, Exception e
                 ) {
                     if (e != null) {
                         onDone(e);
@@ -458,14 +458,14 @@ private IgniteInternalFuture<Collection<GridCacheEntryInfo>> getAsync(
      * @param map Map to convert.
      * @return List of infos.
      */
-    private Collection<GridCacheEntryInfo> toEntryInfos(Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>> map) {
+    private Collection<GridCacheEntryInfo> toEntryInfos(Map<KeyCacheObject, EntryGetResult> map) {
         if (map.isEmpty())
             return Collections.emptyList();
 
         Collection<GridCacheEntryInfo> infos = new ArrayList<>(map.size());
 
-        for (Map.Entry<KeyCacheObject, T2<CacheObject, GridCacheVersion>> entry : map.entrySet()) {
-            T2<CacheObject, GridCacheVersion> val = entry.getValue();
+        for (Map.Entry<KeyCacheObject, EntryGetResult> entry : map.entrySet()) {
+            EntryGetResult val = entry.getValue();
 
             assert val != null;
 
@@ -473,8 +473,10 @@ private Collection<GridCacheEntryInfo> toEntryInfos(Map<KeyCacheObject, T2<Cache
 
             info.cacheId(cctx.cacheId());
             info.key(entry.getKey());
-            info.value(skipVals ? null : val.get1());
-            info.version(val.get2());
+            info.value(skipVals ? null : (CacheObject)val.value());
+            info.version(val.version());
+            info.expireTime(val.expireTime());
+            info.ttl(val.ttl());
 
             infos.add(info);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
index 49bebd6a74b45..f3a27bfd2d102 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
@@ -39,7 +40,6 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
@@ -348,7 +348,7 @@ private void getAsync() {
             }
         }
 
-        IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> fut;
+        IgniteInternalFuture<Map<KeyCacheObject, EntryGetResult>> fut;
 
         if (rdrFut == null || rdrFut.isDone()) {
             fut = cache().getDhtAllAsync(
@@ -375,7 +375,7 @@ private void getAsync() {
                             return;
                         }
 
-                        IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> fut0 =
+                        IgniteInternalFuture<Map<KeyCacheObject, EntryGetResult>> fut0 =
                             cache().getDhtAllAsync(
                                 Collections.singleton(key),
                                 args,
@@ -403,11 +403,11 @@ private void getAsync() {
     /**
      * @return Listener for get future.
      */
-    @NotNull private IgniteInClosure<IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>>>
+    @NotNull private IgniteInClosure<IgniteInternalFuture<Map<KeyCacheObject, EntryGetResult>>>
     createGetFutureListener() {
-        return new IgniteInClosure<IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>>>() {
+        return new IgniteInClosure<IgniteInternalFuture<Map<KeyCacheObject, EntryGetResult>>>() {
             @Override public void apply(
-                IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> fut
+                IgniteInternalFuture<Map<KeyCacheObject, EntryGetResult>> fut
             ) {
                 onResult(fut);
             }
@@ -417,7 +417,7 @@ private void getAsync() {
     /**
      * @param fut Completed future to finish this process with.
      */
-    private void onResult(IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>>> fut) {
+    private void onResult(IgniteInternalFuture<Map<KeyCacheObject, EntryGetResult>> fut) {
         assert fut.isDone();
 
         if (fut.error() != null)
@@ -436,11 +436,11 @@ private void onResult(IgniteInternalFuture<Map<KeyCacheObject, T2<CacheObject, G
      * @param map Map to convert.
      * @return List of infos.
      */
-    private GridCacheEntryInfo toEntryInfo(Map<KeyCacheObject, T2<CacheObject, GridCacheVersion>> map) {
+    private GridCacheEntryInfo toEntryInfo(Map<KeyCacheObject, EntryGetResult> map) {
         if (map.isEmpty())
             return null;
 
-        T2<CacheObject, GridCacheVersion> val = map.get(key);
+        EntryGetResult val = map.get(key);
 
         assert val != null;
 
@@ -448,8 +448,10 @@ private GridCacheEntryInfo toEntryInfo(Map<KeyCacheObject, T2<CacheObject, GridC
 
         info.cacheId(cctx.cacheId());
         info.key(key);
-        info.value(skipVals ? null : val.get1());
-        info.version(val.get2());
+        info.value(skipVals ? null : (CacheObject)val.value());
+        info.version(val.version());
+        info.expireTime(val.expireTime());
+        info.ttl(val.ttl());
 
         return info;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 5892b374951f4..c41711c245028 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -447,11 +447,12 @@ private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
                 if (entry != null) {
                     boolean isNew = entry.isNewLocked();
 
+                    EntryGetResult getRes = null;
                     CacheObject v = null;
                     GridCacheVersion ver = null;
 
                     if (needVer) {
-                        EntryGetResult res = entry.innerGetVersioned(
+                        getRes = entry.innerGetVersioned(
                             null,
                             null,
                             /*swap*/true,
@@ -465,9 +466,9 @@ private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
                             !deserializeBinary,
                             null);
 
-                        if (res != null) {
-                            v = res.value();
-                            ver = res.version();
+                        if (getRes != null) {
+                            v = getRes.value();
+                            ver = getRes.version();
                         }
                     }
                     else {
@@ -501,7 +502,11 @@ private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
                             keepCacheObjects,
                             deserializeBinary,
                             true,
-                            ver);
+                            getRes,
+                            ver,
+                            0,
+                            0,
+                            needVer);
 
                         return true;
                     }
@@ -560,7 +565,9 @@ private Map<K, V> createResultMap(Collection<GridCacheEntryInfo> infos) {
                     keepCacheObjects,
                     deserializeBinary,
                     false,
-                    needVer ? info.version() : null);
+                    needVer ? info.version() : null,
+                    0,
+                    0);
             }
 
             return map;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index 7c14f35b34cf4..2b5624b489b6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -51,7 +51,6 @@
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteProductVersion;
@@ -615,7 +614,7 @@ private void setSkipValueResult(boolean res, @Nullable GridCacheVersion ver) {
         if (needVer) {
             assert ver != null || !res;
 
-            onDone(new T2<>(res, ver));
+            onDone(new EntryGetResult(res, ver));
         }
         else
             onDone(res);
@@ -633,10 +632,10 @@ private void setResult(@Nullable CacheObject val, @Nullable GridCacheVersion ver
                 if (!keepCacheObjects) {
                     Object res = cctx.unwrapBinaryIfNeeded(val, !deserializeBinary);
 
-                    onDone(needVer ? new T2<>(res, ver) : res);
+                    onDone(needVer ? new EntryGetResult(res, ver) : res);
                 }
                 else
-                    onDone(needVer ? new T2<>(val, ver) : val);
+                    onDone(needVer ? new EntryGetResult(val, ver) : val);
             }
             else
                 onDone(null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 2f97bccf88a71..72e1bb1cc3080 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1491,11 +1491,12 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
                         if (entry != null) {
                             boolean isNew = entry.isNewLocked();
 
+                            EntryGetResult getRes = null;
                             CacheObject v = null;
                             GridCacheVersion ver = null;
 
                             if (needVer) {
-                                EntryGetResult res = entry.innerGetVersioned(
+                                getRes = entry.innerGetVersioned(
                                     null,
                                     null,
                                     /*swap*/true,
@@ -1509,9 +1510,9 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
                                     true,
                                     null);
 
-                                if (res != null) {
-                                    v = res.value();
-                                    ver = res.version();
+                                if (getRes != null) {
+                                    v = getRes.value();
+                                    ver = getRes.version();
                                 }
                             }
                             else {
@@ -1539,7 +1540,8 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
                                 success = false;
                             }
                             else
-                                ctx.addResult(locVals, key, v, skipVals, false, deserializeBinary, true, ver);
+                                ctx.addResult(locVals, key, v, skipVals, false, deserializeBinary, true,
+                                    getRes, ver, 0, 0, needVer);
                         }
                         else
                             success = false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 5ed30dbcadcba..ccdc51d2f6139 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -481,11 +481,12 @@ public final IgniteInternalFuture<Map<K, V>> loadAsync(
                         if (entry != null) {
                             boolean isNew = entry.isNewLocked();
 
+                            EntryGetResult getRes = null;
                             CacheObject v = null;
                             GridCacheVersion ver = null;
 
                             if (needVer) {
-                                EntryGetResult res = entry.innerGetVersioned(
+                                getRes = entry.innerGetVersioned(
                                     null,
                                     null,
                                     /*swap*/true,
@@ -499,9 +500,9 @@ public final IgniteInternalFuture<Map<K, V>> loadAsync(
                                     !deserializeBinary,
                                     null);
 
-                                if (res != null) {
-                                    v = res.value();
-                                    ver = res.version();
+                                if (getRes != null) {
+                                    v = getRes.value();
+                                    ver = getRes.version();
                                 }
                             }
                             else {
@@ -540,7 +541,11 @@ public final IgniteInternalFuture<Map<K, V>> loadAsync(
                                     keepCacheObj,
                                     deserializeBinary,
                                     true,
-                                    ver);
+                                    getRes,
+                                    ver,
+                                    0,
+                                    0,
+                                    needVer);
                             }
                         }
                         else
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index 8c64e3e9603bd..cb47498be0167 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -56,7 +56,6 @@
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -665,7 +664,7 @@ private void addResult(KeyCacheObject key, CacheObject v, GridCacheVersion ver)
         if (keepCacheObjects) {
             K key0 = (K)key;
             V val0 = needVer ?
-                (V)new T2<>(skipVals ? true : v, ver) :
+                (V)new EntryGetResult(skipVals ? true : v, ver) :
                 (V)(skipVals ? true : v);
 
             add(new GridFinishedFuture<>(Collections.singletonMap(key0, val0)));
@@ -673,7 +672,7 @@ private void addResult(KeyCacheObject key, CacheObject v, GridCacheVersion ver)
         else {
             K key0 = (K)cctx.unwrapBinaryIfNeeded(key, !deserializeBinary, false);
             V val0 = needVer ?
-                (V)new T2<>(!skipVals ?
+                (V)new EntryGetResult(!skipVals ?
                     (V)cctx.unwrapBinaryIfNeeded(v, !deserializeBinary, false) :
                     (V)Boolean.TRUE, ver) :
                 !skipVals ?
@@ -759,7 +758,9 @@ private Map<K, V> loadEntries(
                         keepCacheObjects,
                         deserializeBinary,
                         false,
-                        needVer ? info.version() : null);
+                        needVer ? info.version() : null,
+                        0,
+                        0);
                 }
                 catch (GridCacheEntryRemovedException ignore) {
                     if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 67518ef1d5771..ae9edcdddc0b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
+import org.apache.ignite.internal.processors.cache.EntryGetResult;
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheMvccFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheReturn;
@@ -60,7 +61,6 @@
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -476,10 +476,10 @@ private void processLoaded(
             GridCacheVersion ver;
 
             if (needVer) {
-                T2<Object, GridCacheVersion> t = (T2)val;
+                EntryGetResult getRes = (EntryGetResult)val;
 
-                v = t.get1();
-                ver = t.get2();
+                v = getRes.value();
+                ver = getRes.version();
             }
             else {
                 v = val;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index ee4f7a63e1e0b..7da11b6b870be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -513,7 +513,6 @@ private Map<K, V> getAllInternal(@Nullable Collection<? extends K> keys,
 
                     if (entry != null) {
                         CacheObject v;
-                        GridCacheVersion ver;
 
                         if (needVer) {
                             EntryGetResult res = entry.innerGetVersioned(
@@ -531,18 +530,15 @@ private Map<K, V> getAllInternal(@Nullable Collection<? extends K> keys,
                                 null);
 
                             if (res != null) {
-                                v = res.value();
-                                ver = res.version();
-
                                 ctx.addResult(
                                     vals,
                                     cacheKey,
-                                    v,
+                                    res,
                                     skipVals,
                                     false,
                                     deserializeBinary,
                                     true,
-                                    ver);
+                                    needVer);
                             }
                             else
                                 success = false;
@@ -569,7 +565,10 @@ private Map<K, V> getAllInternal(@Nullable Collection<? extends K> keys,
                                     skipVals,
                                     false,
                                     deserializeBinary,
-                                    true);
+                                    true,
+                                    null,
+                                    0,
+                                    0);
                             }
                             else
                                 success = false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index a1c1123d8cfd5..777489e026edb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -481,7 +481,7 @@ protected boolean commitAfterLock() {
                                 GridCacheEntryEx entry = cacheCtx.cache().entryEx(key);
 
                                 try {
-                                    T2<CacheObject, GridCacheVersion> verVal = entry.versionedValue(cacheVal,
+                                    EntryGetResult verVal = entry.versionedValue(cacheVal,
                                         ver,
                                         null,
                                         null,
@@ -490,11 +490,11 @@ protected boolean commitAfterLock() {
                                     if (log.isDebugEnabled()) {
                                         log.debug("Set value loaded from store into entry [" +
                                             "oldVer=" + ver +
-                                            ", newVer=" + verVal.get2() +
+                                            ", newVer=" + verVal.version() +
                                             ", entry=" + entry + ']');
                                     }
 
-                                    ver = verVal.get2();
+                                    ver = verVal.version();
 
                                     break;
                                 }
@@ -1212,7 +1212,8 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                             assert ver != null;
                         }
 
-                        cacheCtx.addResult(map, key, val, skipVals, keepCacheObjects, deserializeBinary, false, ver);
+                        cacheCtx.addResult(map, key, val, skipVals, keepCacheObjects, deserializeBinary, false,
+                            ver, 0, 0);
                     }
                 }
                 else {
@@ -1221,6 +1222,7 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                     while (true) {
                         try {
                             GridCacheVersion readVer = null;
+                            EntryGetResult getRes = null;
 
                             Object transformClo =
                                 (txEntry.op() == TRANSFORM &&
@@ -1228,7 +1230,7 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                     F.first(txEntry.entryProcessors()) : null;
 
                             if (needVer) {
-                                EntryGetResult res = txEntry.cached().innerGetVersioned(
+                                getRes = txEntry.cached().innerGetVersioned(
                                     null,
                                     this,
                                     /*swap*/true,
@@ -1242,9 +1244,9 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                     txEntry.keepBinary(),
                                     null);
 
-                                if (res != null) {
-                                    val = res.value();
-                                    readVer = res.version();
+                                if (getRes != null) {
+                                    val = getRes.value();
+                                    readVer = getRes.version();
                                 }
                             }
                             else {
@@ -1277,7 +1279,11 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                     keepCacheObjects,
                                     deserializeBinary,
                                     false,
-                                    readVer);
+                                    getRes,
+                                    readVer,
+                                    0,
+                                    0,
+                                    needVer);
                             }
                             else
                                 missed.put(key, txEntry.cached().version());
@@ -1306,13 +1312,14 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
 
                         CacheObject val = null;
                         GridCacheVersion readVer = null;
+                        EntryGetResult getRes = null;
 
                         if (!pessimistic() || readCommitted() && !skipVals) {
                             IgniteCacheExpiryPolicy accessPlc =
                                 optimistic() ? accessPolicy(cacheCtx, txKey, expiryPlc) : null;
 
                             if (needReadVer) {
-                                EntryGetResult res = primaryLocal(entry) ?
+                                getRes = primaryLocal(entry) ?
                                     entry.innerGetVersioned(
                                         null,
                                         this,
@@ -1327,9 +1334,9 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                         !deserializeBinary,
                                         null) : null;
 
-                                if (res != null) {
-                                    val = res.value();
-                                    readVer = res.version();
+                                if (getRes != null) {
+                                    val = getRes.value();
+                                    readVer = getRes.version();
                                 }
                             }
                             else {
@@ -1356,7 +1363,11 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                                     keepCacheObjects,
                                     deserializeBinary,
                                     false,
-                                    needVer ? readVer : null);
+                                    getRes,
+                                    readVer,
+                                    0,
+                                    0,
+                                    needVer);
                             }
                             else
                                 missed.put(key, ver);
@@ -1534,7 +1545,9 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                                     keepCacheObjects,
                                     deserializeBinary,
                                     false,
-                                    needVer ? loadVer : null);
+                                    needVer ? loadVer : null,
+                                    0,
+                                    0);
                             }
                         }
                         else {
@@ -1556,7 +1569,9 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                                     keepCacheObjects,
                                     deserializeBinary,
                                     false,
-                                    needVer ? loadVer : null);
+                                    needVer ? loadVer : null,
+                                    0,
+                                    0);
                             }
                         }
                     }
@@ -1663,6 +1678,7 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
 
                                 CacheObject val = null;
                                 GridCacheVersion readVer = null;
+                                EntryGetResult getRes = null;
 
                                 try {
                                     Object transformClo =
@@ -1671,7 +1687,7 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                                             F.first(txEntry.entryProcessors()) : null;
 
                                     if (needVer) {
-                                        EntryGetResult res = cached.innerGetVersioned(
+                                        getRes = cached.innerGetVersioned(
                                             null,
                                             IgniteTxLocalAdapter.this,
                                             /*swap*/cacheCtx.isSwapOrOffheapEnabled(),
@@ -1685,9 +1701,9 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                                             txEntry.keepBinary(),
                                             null);
 
-                                        if (res != null) {
-                                            val = res.value();
-                                            readVer = res.version();
+                                        if (getRes != null) {
+                                            val = getRes.value();
+                                            readVer = getRes.version();
                                         }
                                     }
                                     else{
@@ -1722,7 +1738,11 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                                             keepCacheObjects,
                                             deserializeBinary,
                                             false,
-                                            readVer);
+                                            getRes,
+                                            readVer,
+                                            0,
+                                            0,
+                                            needVer);
 
                                         if (readVer != null)
                                             txEntry.entryReadVersion(readVer);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 2954bdb413d16..d46dee03b3a27 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -686,7 +686,7 @@ void recheckLock() {
     }
 
     /** @inheritDoc */
-    @Override public T2<CacheObject, GridCacheVersion> versionedValue(CacheObject val,
+    @Override public EntryGetResult versionedValue(CacheObject val,
         GridCacheVersion curVer,
         GridCacheVersion newVer,
         @Nullable ReaderArguments readerArgs,
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index f22ca6d30b9c4..b234631e4002a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -36,12 +36,14 @@
 import javax.cache.expiry.ModifiedExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheMemoryMode;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
@@ -54,6 +56,7 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.PAX;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
@@ -1014,6 +1017,45 @@ public void testNearAccess() throws Exception {
         checkTtl(key, 62_000L, true);
     }
 
+    /**
+     * Put entry to server node and check how its expires in client NearCache.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNearExpiresOnClient() throws Exception {
+        if(cacheMode() != PARTITIONED)
+            return;
+
+        factory =  CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS,1));
+
+        nearCache = true;
+
+        startGrids();
+
+        IgniteConfiguration clientCfg = getConfiguration("client").setClientMode(true);
+
+        ((TcpDiscoverySpi)clientCfg.getDiscoverySpi()).setForceServerMode(false);
+
+        Ignite client = startGrid("client", clientCfg);
+
+        IgniteCache<Object, Object> cache = client.cache(null);
+
+        Integer key = 1;
+
+        // Put on server node.
+        jcache(0).put(key, 1);
+
+        // Make entry cached in client NearCache.
+        assertEquals(1, cache.get(key));
+
+        assertEquals(1, cache.localPeek(key, CachePeekMode.NEAR));
+
+        waitExpired(key);
+
+        // Check client NearCache.
+        assertNull(cache.localPeek(key, CachePeekMode.NEAR));
+    }
+
     /**
      * @return Test keys.
      * @throws Exception If failed.
@@ -1270,4 +1312,4 @@ private static class TestPolicy implements ExpiryPolicy, Serializable {
             return S.toString(TestPolicy.class, this);
         }
     }
-}
\ No newline at end of file
+}

From 898fa150c7e2a663685abf37c6d1bcd547910f0e Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 27 Jan 2017 11:30:49 +0700
Subject: [PATCH 129/446] IGNITE-4622 Fixed generation in domain model for
 cache store.

(cherry picked from commit 43007d5)
---
 .../configuration/generator/ConfigurationGenerator.js    | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 8770bf6063c73..abe361cde29d8 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -48,6 +48,10 @@ export default class IgniteConfigurationGenerator {
         return new Bean('org.apache.ignite.cache.QueryEntity', 'qryEntity', domain, cacheDflts);
     }
 
+    static domainJdbcTypeBean(domain) {
+        return new Bean('org.apache.ignite.cache.store.jdbc.JdbcType', 'type', domain);
+    }
+
     static discoveryConfigurationBean(discovery) {
         return new Bean('org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi', 'discovery', discovery, clusterDflts.discovery);
     }
@@ -1383,7 +1387,7 @@ export default class IgniteConfigurationGenerator {
     }
 
     // Generate domain model for store group.
-    static domainStore(domain, cfg = this.domainConfigurationBean(domain)) {
+    static domainStore(domain, cfg = this.domainJdbcTypeBean(domain)) {
         cfg.stringProperty('databaseSchema')
             .stringProperty('databaseTable');
 
@@ -1527,8 +1531,7 @@ export default class IgniteConfigurationGenerator {
                         if (_.isNil(domain.databaseTable))
                             return acc;
 
-                        const typeBean = new Bean('org.apache.ignite.cache.store.jdbc.JdbcType', 'type',
-                            _.merge({}, domain, {cacheName: cache.name}))
+                        const typeBean = this.domainJdbcTypeBean(_.merge({}, domain, {cacheName: cache.name}))
                             .stringProperty('cacheName');
 
                         setType(typeBean, 'keyType');

From 1b3bbdb1758eb19c755dabcb5fe329529fa0f378 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 27 Jan 2017 11:30:49 +0700
Subject: [PATCH 130/446] IGNITE-4622 Fixed generation in domain model for
 cache store.

(cherry picked from commit 43007d5)
---
 .../configuration/generator/ConfigurationGenerator.js    | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index de2b750675fd8..670b8284f14c1 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -48,6 +48,10 @@ export default class IgniteConfigurationGenerator {
         return new Bean('org.apache.ignite.cache.QueryEntity', 'qryEntity', domain, cacheDflts);
     }
 
+    static domainJdbcTypeBean(domain) {
+        return new Bean('org.apache.ignite.cache.store.jdbc.JdbcType', 'type', domain);
+    }
+
     static discoveryConfigurationBean(discovery) {
         return new Bean('org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi', 'discovery', discovery, clusterDflts.discovery);
     }
@@ -1383,7 +1387,7 @@ export default class IgniteConfigurationGenerator {
     }
 
     // Generate domain model for store group.
-    static domainStore(domain, cfg = this.domainConfigurationBean(domain)) {
+    static domainStore(domain, cfg = this.domainJdbcTypeBean(domain)) {
         cfg.stringProperty('databaseSchema')
             .stringProperty('databaseTable');
 
@@ -1562,8 +1566,7 @@ export default class IgniteConfigurationGenerator {
                         if (_.isNil(domain.databaseTable))
                             return acc;
 
-                        const typeBean = new Bean('org.apache.ignite.cache.store.jdbc.JdbcType', 'type',
-                            _.merge({}, domain, {cacheName: cache.name}))
+                        const typeBean = this.domainJdbcTypeBean(_.merge({}, domain, {cacheName: cache.name}))
                             .stringProperty('cacheName');
 
                         setType(typeBean, 'keyType');

From 83ad0943edf6a0e07d1b07c889492bce4932f9d1 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Fri, 27 Jan 2017 17:44:57 +0300
Subject: [PATCH 131/446] Minor javadoc fix

---
 .../processors/cache/distributed/near/GridNearTxLocal.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index ae9edcdddc0b1..5aa091d7cc062 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -1169,7 +1169,7 @@ public IgniteInternalFuture<IgniteInternalTx> rollbackAsyncLocal() {
      * @param keys Keys.
      * @param retval Return value flag.
      * @param read Read flag.
-     * @param accessTtl Create ttl.
+     * @param createTtl Create ttl.
      * @param accessTtl Access ttl.
      * @param <K> Key type.
      * @param skipStore Skip store flag.

From 92ceb7f71e3ec1856c69fa464c8a8c848314a27e Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Fri, 20 Jan 2017 16:22:24 +0700
Subject: [PATCH 132/446] IGNITE-4548 CacheJdbcStore: support mapping of enum
 types.

(cherry picked from commit f1fca3a)
---
 .../store/jdbc/CacheAbstractJdbcStore.java    | 12 +++++-
 .../jdbc/JdbcTypesDefaultTransformer.java     | 19 +++++++++
 .../store/jdbc/JdbcTypesTransformer.java      | 17 ++++++++
 .../CacheJdbcPojoStoreAbstractSelfTest.java   | 23 +++++++----
 .../store/jdbc/CacheJdbcPojoStoreTest.java    |  3 ++
 ...dbcStoreAbstractMultithreadedSelfTest.java | 17 ++++----
 .../ignite/cache/store/jdbc/model/Gender.java | 41 +++++++++++++++++++
 .../ignite/cache/store/jdbc/model/Person.java | 31 +++++++++++++-
 .../test/config/jdbc-pojo-store-builtin.xml   |  8 ++++
 .../src/test/config/jdbc-pojo-store-obj.xml   |  8 ++++
 10 files changed, 162 insertions(+), 17 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Gender.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
index 4bfd92b836268..e7ce52651a868 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
@@ -80,6 +80,8 @@
 import static org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory.DFLT_BATCH_SIZE;
 import static org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory.DFLT_PARALLEL_LOAD_CACHE_MINIMUM_THRESHOLD;
 import static org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory.DFLT_WRITE_ATTEMPTS;
+import static org.apache.ignite.cache.store.jdbc.JdbcTypesTransformer.NUMERIC_TYPES;
+import static org.apache.ignite.cache.store.jdbc.JdbcTypesTransformer.NUMERIC_TYPES;
 
 /**
  * Implementation of {@link CacheStore} backed by JDBC.
@@ -1393,8 +1395,15 @@ protected void fillParameter(PreparedStatement stmt, int idx, JdbcTypeField fiel
                             fieldVal = fieldVal.toString();
 
                             break;
+                        default:
+                            // No-op.
                     }
                 }
+                else if (field.getJavaFieldType().isEnum() && fieldVal instanceof Enum) {
+                    Enum val = (Enum)fieldVal;
+                    
+                    fieldVal = NUMERIC_TYPES.contains(field.getDatabaseFieldType()) ? val.ordinal() : val.name();
+                }
 
                 stmt.setObject(idx, fieldVal);
             }
@@ -2068,12 +2077,13 @@ private LoadWorker(Connection conn, EntryMapping em) {
 
                 int idx = 1;
 
-                for (Object key : keys)
+                for (Object key : keys) {
                     for (JdbcTypeField field : em.keyColumns()) {
                         Object fieldVal = extractParameter(em.cacheName, em.keyType(), em.keyKind(), field.getJavaFieldName(), key);
 
                         fillParameter(stmt, idx++, field, fieldVal);
                     }
+                }
 
                 ResultSet rs = stmt.executeQuery();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
index c32eaa227dfd0..c387b779e60c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
@@ -114,6 +114,25 @@ public class JdbcTypesDefaultTransformer implements JdbcTypesTransformer {
                 return UUID.fromString((String)res);
         }
 
+        if (type.isEnum()) {
+            if (NUMERIC_TYPES.contains(rs.getMetaData().getColumnType(colIdx))) {
+                int ordinal = rs.getInt(colIdx);
+
+                Object[] values = type.getEnumConstants();
+
+                return rs.wasNull() || ordinal >= values.length ? null : values[ordinal];
+            }
+
+            String str = rs.getString(colIdx);
+
+            try {
+                return rs.wasNull() ? null : Enum.valueOf((Class<? extends Enum>) type, str.trim());
+            }
+            catch (IllegalArgumentException ignore) {
+                return null;
+            }
+        }
+
         return rs.getObject(colIdx);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesTransformer.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesTransformer.java
index 76fb00b90b2ed..fc0bc883d0778 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesTransformer.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesTransformer.java
@@ -20,11 +20,28 @@
 import java.io.Serializable;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static java.sql.Types.BIGINT;
+import static java.sql.Types.DECIMAL;
+import static java.sql.Types.DOUBLE;
+import static java.sql.Types.FLOAT;
+import static java.sql.Types.INTEGER;
+import static java.sql.Types.NUMERIC;
+import static java.sql.Types.REAL;
+import static java.sql.Types.SMALLINT;
+import static java.sql.Types.TINYINT;
 
 /**
  * API for implementing custom mapping logic for loaded from store data.
  */
 public interface JdbcTypesTransformer extends Serializable {
+    /** Numeric types. */
+    public final List<Integer> NUMERIC_TYPES =
+        U.sealList(TINYINT, SMALLINT, INTEGER, BIGINT, REAL, FLOAT, DOUBLE, NUMERIC, DECIMAL);
+
+
     /**
      * Retrieves the value of the designated column in the current row of this <code>ResultSet</code> object and
      * will convert to the requested Java data type.
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
index 368a28e010c3b..1de44f7243fbb 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect;
 import org.apache.ignite.cache.store.jdbc.model.Person;
+import org.apache.ignite.cache.store.jdbc.model.Gender;
 import org.apache.ignite.cache.store.jdbc.model.PersonKey;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ConnectorConfiguration;
@@ -112,7 +113,8 @@ protected Connection getConnection() throws SQLException {
             " id INTEGER PRIMARY KEY," +
             " org_id INTEGER," +
             " birthday DATE," +
-            " name VARCHAR(50))");
+            " name VARCHAR(50)," +
+            " gender VARCHAR(50))");
 
         conn.commit();
 
@@ -201,7 +203,8 @@ protected JdbcType[] storeTypes() {
             new JdbcTypeField(Types.INTEGER, "ID", Integer.class, "id"),
             new JdbcTypeField(Types.INTEGER, "ORG_ID", Integer.class, "orgId"),
             new JdbcTypeField(Types.DATE, "BIRTHDAY", Date.class, "birthday"),
-            new JdbcTypeField(Types.VARCHAR, "NAME", String.class, "name"));
+            new JdbcTypeField(Types.VARCHAR, "NAME", String.class, "name"),
+            new JdbcTypeField(Types.VARCHAR, "GENDER", Gender.class, "gender"));
 
         return storeTypes;
     }
@@ -260,7 +263,7 @@ protected void fillSampleDatabase(Connection conn) throws SQLException {
         conn.commit();
 
         PreparedStatement prnStmt = conn.prepareStatement(
-            "INSERT INTO Person(id, org_id, birthday, name) VALUES (?, ?, ?, ?)");
+            "INSERT INTO Person(id, org_id, birthday, name, gender) VALUES (?, ?, ?, ?, ?)");
 
         Random rnd = new Random();
 
@@ -269,6 +272,7 @@ protected void fillSampleDatabase(Connection conn) throws SQLException {
             prnStmt.setInt(2, i % 100);
             prnStmt.setDate(3, Date.valueOf(String.format("%d-%d-%d", 1970 + rnd.nextInt(50), 1 + rnd.nextInt(11), 1 + rnd.nextInt(27))));
             prnStmt.setString(4, "name" + i);
+            prnStmt.setString(5, Gender.random().toString());
 
             prnStmt.addBatch();
         }
@@ -319,7 +323,7 @@ protected void checkCacheLoad() {
     protected void checkCacheLoadWithSql() {
         IgniteCache<Object, Object> c1 = grid().cache(CACHE_NAME);
 
-        c1.loadCache(null, "org.apache.ignite.cache.store.jdbc.model.PersonKey", "select id, org_id, name, birthday from Person");
+        c1.loadCache(null, "org.apache.ignite.cache.store.jdbc.model.PersonKey", "select id, org_id, name, birthday, gender from Person");
 
         assertEquals(PERSON_CNT, c1.size());
     }
@@ -397,7 +401,9 @@ private void checkPutRemove() throws Exception {
 
         Connection conn = getConnection();
         try {
-            PreparedStatement stmt = conn.prepareStatement("SELECT ID, ORG_ID, BIRTHDAY, NAME FROM PERSON WHERE ID = ?");
+            Random rnd = new Random();
+
+            PreparedStatement stmt = conn.prepareStatement("SELECT ID, ORG_ID, BIRTHDAY, NAME, GENDER FROM PERSON WHERE ID = ?");
 
             stmt.setInt(1, -1);
 
@@ -408,8 +414,9 @@ private void checkPutRemove() throws Exception {
             U.closeQuiet(rs);
 
             Date testDate = Date.valueOf("2001-05-05");
+            Gender testGender = Gender.random();
 
-            Person val = new Person(-1, -2, testDate, "Person-to-test-put-insert", 999);
+            Person val = new Person(-1, -2, testDate, "Person-to-test-put-insert", 999, testGender);
 
             Object key = builtinKeys ? Integer.valueOf(-1) : new PersonKey(-1);
 
@@ -424,6 +431,7 @@ private void checkPutRemove() throws Exception {
             assertEquals(-2, rs.getInt(2));
             assertEquals(testDate, rs.getDate(3));
             assertEquals("Person-to-test-put-insert", rs.getString(4));
+            assertEquals(testGender.toString(), rs.getString(5));
 
             assertFalse("Unexpected more data in result set", rs.next());
 
@@ -432,7 +440,7 @@ private void checkPutRemove() throws Exception {
             // Test put-update.
             testDate = Date.valueOf("2016-04-04");
 
-            c1.put(key, new Person(-1, -3, testDate, "Person-to-test-put-update", 999));
+            c1.put(key, new Person(-1, -3, testDate, "Person-to-test-put-update", 999, testGender));
 
             rs = stmt.executeQuery();
 
@@ -442,6 +450,7 @@ private void checkPutRemove() throws Exception {
             assertEquals(-3, rs.getInt(2));
             assertEquals(testDate, rs.getDate(3));
             assertEquals("Person-to-test-put-update", rs.getString(4));
+            assertEquals(testGender.toString(), rs.getString(5));
 
             assertFalse("Unexpected more data in result set", rs.next());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
index 4a0b1daf646d1..849cab79a03be 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
@@ -25,6 +25,7 @@
 import java.sql.Types;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import javax.cache.integration.CacheWriterException;
@@ -233,6 +234,8 @@ public CacheJdbcPojoStoreTest() throws Exception {
     public void testLoadCache() throws Exception {
         Connection conn = store.openConnection(false);
 
+        Random rnd = new Random();
+
         PreparedStatement orgStmt = conn.prepareStatement("INSERT INTO Organization(id, name, city) VALUES (?, ?, ?)");
 
         for (int i = 0; i < ORGANIZATION_CNT; i++) {
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreAbstractMultithreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreAbstractMultithreadedSelfTest.java
index e831445c896da..f1a321ba01216 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreAbstractMultithreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreAbstractMultithreadedSelfTest.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheTypeMetadata;
+import org.apache.ignite.cache.store.jdbc.model.Gender;
 import org.apache.ignite.cache.store.jdbc.model.Organization;
 import org.apache.ignite.cache.store.jdbc.model.OrganizationKey;
 import org.apache.ignite.cache.store.jdbc.model.Person;
@@ -208,7 +209,7 @@ public void testMultithreadedPut() throws Exception {
                         cache.put(new OrganizationKey(id), new Organization(id, "Name" + id, "City" + id));
                     else
                         cache.put(new PersonKey(id), new Person(id, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + id, 1));
+                            new Date(System.currentTimeMillis()), "Name" + id, 1, Gender.random()));
                 }
 
                 return null;
@@ -228,7 +229,7 @@ public void testMultithreadedPut() throws Exception {
                         cache.putIfAbsent(new OrganizationKey(id), new Organization(id, "Name" + id, "City" + id));
                     else
                         cache.putIfAbsent(new PersonKey(id), new Person(id, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + id, i));
+                            new Date(System.currentTimeMillis()), "Name" + id, i, Gender.random()));
                 }
 
                 return null;
@@ -268,7 +269,7 @@ public void testMultithreadedPutAll() throws Exception {
                             map.put(new OrganizationKey(id), new Organization(id, "Name" + id, "City" + id));
                         else
                             map.put(new PersonKey(id), new Person(id, rnd.nextInt(),
-                                new Date(System.currentTimeMillis()), "Name" + id, 1));
+                                new Date(System.currentTimeMillis()), "Name" + id, 1, Gender.random()));
                     }
 
                     IgniteCache<Object, Object> cache = jcache();
@@ -294,11 +295,11 @@ public void testMultithreadedExplicitTx() throws Exception {
 
                     try (Transaction tx = grid().transactions().txStart()) {
                         cache.put(new PersonKey(1), new Person(1, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + 1, 1));
+                            new Date(System.currentTimeMillis()), "Name" + 1, 1, Gender.random()));
                         cache.put(new PersonKey(2), new Person(2, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + 2, 2));
+                            new Date(System.currentTimeMillis()), "Name" + 2, 2, Gender.random()));
                         cache.put(new PersonKey(3), new Person(3, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + 3, 3));
+                            new Date(System.currentTimeMillis()), "Name" + 3, 3, Gender.random()));
 
                         cache.get(new PersonKey(1));
                         cache.get(new PersonKey(4));
@@ -306,9 +307,9 @@ public void testMultithreadedExplicitTx() throws Exception {
                         Map<PersonKey, Person> map =  U.newHashMap(2);
 
                         map.put(new PersonKey(5), new Person(5, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + 5, 5));
+                            new Date(System.currentTimeMillis()), "Name" + 5, 5, Gender.random()));
                         map.put(new PersonKey(6), new Person(6, rnd.nextInt(),
-                            new Date(System.currentTimeMillis()), "Name" + 6, 6));
+                            new Date(System.currentTimeMillis()), "Name" + 6, 6, Gender.random()));
 
                         cache.putAll(map);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Gender.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Gender.java
new file mode 100644
index 0000000000000..8ddb0e2a3670e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Gender.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.jdbc.model;
+
+import java.io.Serializable;
+import java.util.Random;
+
+/**
+ * Person gender enum.
+ */
+public enum Gender implements Serializable {
+    /** */
+    MALE,
+    /** */
+    FEMALE;
+
+    /** */
+    private static final Random RAND = new Random();
+
+    /**
+     * Used for testing purposes.
+     */
+    public static Gender random() {
+        return values()[RAND.nextInt(values().length)];
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java
index 52ddfc8ecdcbb..89258b403de23 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/Person.java
@@ -44,6 +44,9 @@ public class Person implements Serializable {
     /** Value for salary. */
     private Integer salary;
 
+    /** Value of person gender. */
+    private Gender gender;
+
     /**
      * Empty constructor.
      */
@@ -59,13 +62,15 @@ public Person(
         Integer orgId,
         Date birthday,
         String name,
-        Integer salary
+        Integer salary,
+        Gender gender
     ) {
         this.id = id;
         this.orgId = orgId;
         this.birthday = birthday;
         this.name = name;
         this.salary = salary;
+        this.gender = gender;
     }
 
     /**
@@ -159,6 +164,24 @@ public void setSalary(Integer salary) {
         this.salary = salary;
     }
 
+    /**
+     * Gets gender.
+     *
+     * @return Gender.
+     */
+    public Gender getGender() {
+        return gender;
+    }
+
+    /**
+     * Sets gender.
+     *
+     * @param gender New value for gender.
+     */
+    public void setGender(Gender gender) {
+        this.gender = gender;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean equals(Object o) {
         if (this == o)
@@ -178,6 +201,9 @@ public void setSalary(Integer salary) {
         if (name != null ? !name.equals(that.name) : that.name != null)
             return false;
 
+        if (gender != null ? !gender.equals(that.gender) : that.gender != null)
+            return false;
+
         return true;
     }
 
@@ -189,6 +215,8 @@ public void setSalary(Integer salary) {
 
         res = 31 * res + (name != null ? name.hashCode() : 0);
 
+        res = 31 * res + (gender != null ? gender.hashCode() : 0);
+
         return res;
     }
 
@@ -198,6 +226,7 @@ public void setSalary(Integer salary) {
             ", orgId=" + orgId +
             ", birthday=" + (birthday == null ? null : birthday.getTime()) +
             ", name=" + name +
+            ", gender=" + gender +
             "]";
     }
 }
diff --git a/modules/spring/src/test/config/jdbc-pojo-store-builtin.xml b/modules/spring/src/test/config/jdbc-pojo-store-builtin.xml
index dfaf828fa8d22..bfb109c035d94 100644
--- a/modules/spring/src/test/config/jdbc-pojo-store-builtin.xml
+++ b/modules/spring/src/test/config/jdbc-pojo-store-builtin.xml
@@ -151,6 +151,14 @@
                                                     <property name="javaFieldName" value="orgId"/>
                                                     <property name="javaFieldType" value="java.lang.Integer"/>
                                                 </bean>
+                                                <bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
+                                                    <property name="databaseFieldName" value="GENDER"/>
+                                                    <property name="databaseFieldType">
+                                                        <util:constant static-field="java.sql.Types.VARCHAR"/>
+                                                    </property>
+                                                    <property name="javaFieldName" value="gender"/>
+                                                    <property name="javaFieldType" value="org.apache.ignite.cache.store.jdbc.model.Gender"/>
+                                                </bean>
                                             </list>
                                         </property>
                                     </bean>
diff --git a/modules/spring/src/test/config/jdbc-pojo-store-obj.xml b/modules/spring/src/test/config/jdbc-pojo-store-obj.xml
index 9bc99776a5bec..40a14dc4f4fda 100644
--- a/modules/spring/src/test/config/jdbc-pojo-store-obj.xml
+++ b/modules/spring/src/test/config/jdbc-pojo-store-obj.xml
@@ -151,6 +151,14 @@
                                                     <property name="javaFieldName" value="orgId"/>
                                                     <property name="javaFieldType" value="java.lang.Integer"/>
                                                 </bean>
+                                                <bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
+                                                    <property name="databaseFieldName" value="GENDER"/>
+                                                    <property name="databaseFieldType">
+                                                        <util:constant static-field="java.sql.Types.VARCHAR"/>
+                                                    </property>
+                                                    <property name="javaFieldName" value="gender"/>
+                                                    <property name="javaFieldType" value="org.apache.ignite.cache.store.jdbc.model.Gender"/>
+                                                </bean>
                                             </list>
                                         </property>
                                     </bean>

From 4d72bf8bd47d87f846178f3c0e4e8df8b17d3094 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 25 Jan 2017 13:58:57 +0700
Subject: [PATCH 133/446] IGNITE-4520 Added credential request for
 authentication on proxy.

(cherry picked from commit ef04f35)
---
 .../web-agent/bin/ignite-web-agent.bat        |  4 +-
 .../web-agent/bin/ignite-web-agent.sh         |  2 +
 .../ignite/console/agent/AgentLauncher.java   | 90 +++++++++++++++++--
 3 files changed, 88 insertions(+), 8 deletions(-)

diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.bat b/modules/web-console/web-agent/bin/ignite-web-agent.bat
index 8291b5531da50..1f1b52dc83d23 100644
--- a/modules/web-console/web-agent/bin/ignite-web-agent.bat
+++ b/modules/web-console/web-agent/bin/ignite-web-agent.bat
@@ -60,7 +60,9 @@ if %ERRORLEVEL% equ 0 (
     if "%JVM_OPTS%" == "" set JVM_OPTS=-Xms1g -Xmx1g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=256m
 )
 
-"%JAVA_HOME%\bin\java.exe" %JVM_OPTS% -cp "*" org.apache.ignite.console.agent.AgentLauncher  %*
+set JVM_OPTS=%JVM_OPTS% -Djava.net.useSystemProxies=true
+
+"%JAVA_HOME%\bin\java.exe" %JVM_OPTS% -cp "*" org.apache.ignite.console.agent.AgentLauncher %*
 
 set JAVA_ERRORLEVEL=%ERRORLEVEL%
 
diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.sh b/modules/web-console/web-agent/bin/ignite-web-agent.sh
index 2e9f041f8ca7c..c2958fc0c7c9a 100644
--- a/modules/web-console/web-agent/bin/ignite-web-agent.sh
+++ b/modules/web-console/web-agent/bin/ignite-web-agent.sh
@@ -88,4 +88,6 @@ if [ -z "$JVM_OPTS" ] ; then
     fi
 fi
 
+JVM_OPTS="${JVM_OPTS} -Djava.net.useSystemProxies=true"
+
 "$JAVA" ${JVM_OPTS} -cp "*" org.apache.ignite.console.agent.AgentLauncher "$@"
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 0c03d7787b15d..049791f9107dc 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -25,11 +25,15 @@
 import io.socket.emitter.Emitter;
 import java.io.File;
 import java.io.IOException;
+import java.net.Authenticator;
 import java.net.ConnectException;
+import java.net.PasswordAuthentication;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.UnknownHostException;
 import java.util.Arrays;
+import java.util.Scanner;
 import java.util.concurrent.CountDownLatch;
 import java.util.jar.Attributes;
 import java.util.jar.Manifest;
@@ -76,9 +80,6 @@ public class AgentLauncher {
     /** */
     private static final String EVENT_AGENT_CLOSE = "agent:close";
 
-    /** */
-    private static final int RECONNECT_INTERVAL = 3000;
-
     /**
      * Create a trust manager that trusts all certificates It is not using a particular keyStore
      */
@@ -121,6 +122,15 @@ public void checkServerTrusted(
                     System.exit(1);
                 }
 
+                ignore = X.cause(e, UnknownHostException.class);
+
+                if (ignore != null) {
+                    log.error("Failed to establish connection to server, due to errors with DNS or missing proxy settings.");
+                    log.error("Documentation for proxy configuration can be found here: http://apacheignite.readme.io/docs/web-agent#section-proxy-configuration");
+
+                    System.exit(1);
+                }
+
                 ignore = X.cause(e, IOException.class);
 
                 if (ignore != null && "404".equals(ignore.getMessage())) {
@@ -129,6 +139,29 @@ public void checkServerTrusted(
                     return;
                 }
 
+                if (ignore != null && "407".equals(ignore.getMessage())) {
+                    log.error("Failed to establish connection to server, due to proxy requires authentication.");
+
+                    String userName = System.getProperty("https.proxyUsername", System.getProperty("http.proxyUsername"));
+
+                    if (userName == null || userName.trim().isEmpty())
+                        userName = readLine("Enter proxy user name: ");
+                    else
+                        System.out.println("Read username from system properties: " + userName);
+
+                    char[] pwd = readPassword("Enter proxy password: ");
+
+                    final PasswordAuthentication pwdAuth = new PasswordAuthentication(userName, pwd);
+
+                    Authenticator.setDefault(new Authenticator() {
+                        @Override protected PasswordAuthentication getPasswordAuthentication() {
+                            return pwdAuth;
+                        }
+                    });
+
+                    return;
+                }
+
                 log.error("Connection error.", e);
             }
         }
@@ -143,6 +176,32 @@ public void checkServerTrusted(
         }
     };
 
+    /**
+     * @param fmt Format string.
+     * @param args Arguments.
+     */
+    private static String readLine(String fmt, Object ... args) {
+        if (System.console() != null)
+            return System.console().readLine(fmt, args);
+
+        System.out.print(String.format(fmt, args));
+
+        return new Scanner(System.in).nextLine();
+    }
+
+    /**
+     * @param fmt Format string.
+     * @param args Arguments.
+     */
+    private static char[] readPassword(String fmt, Object ... args) {
+        if (System.console() != null)
+            return System.console().readPassword(fmt, args);
+
+        System.out.print(String.format(fmt, args));
+
+        return new Scanner(System.in).nextLine().toCharArray();
+    }
+
     /**
      * @param args Args.
      */
@@ -214,9 +273,9 @@ public static void main(String[] args) throws Exception {
             System.out.println("Security token is required to establish connection to the web console.");
             System.out.println(String.format("It is available on the Profile page: https://%s/profile", webHost));
 
-            System.out.print("Enter security tokens separated by comma: ");
+            String tokens = String.valueOf(readPassword("Enter security tokens separated by comma: "));
 
-            cfg.tokens(Arrays.asList(System.console().readLine().trim().split(",")));
+            cfg.tokens(Arrays.asList(tokens.trim().split(",")));
         }
 
         final RestHandler restHnd = new RestHandler(cfg);
@@ -226,12 +285,29 @@ public static void main(String[] args) throws Exception {
 
             URI uri = URI.create(cfg.serverUri());
 
+            // Create proxy authenticator using passed properties.
+            switch (uri.getScheme()) {
+                case "http":
+                case "https":
+                    final String username = System.getProperty(uri.getScheme() + ".proxyUsername");
+                    final char[] pwd = System.getProperty(uri.getScheme() +  ".proxyPassword", "").toCharArray();
+
+                    Authenticator.setDefault(new Authenticator() {
+                        @Override protected PasswordAuthentication getPasswordAuthentication() {
+                            return new PasswordAuthentication(username, pwd);
+                        }
+                    });
+
+                    break;
+
+                default:
+                    // No-op.
+            }
+
             IO.Options opts = new IO.Options();
 
             opts.path = "/agents";
 
-            opts.reconnectionDelay = RECONNECT_INTERVAL;
-
             // Workaround for use self-signed certificate
             if (Boolean.getBoolean("trust.all")) {
                 SSLContext ctx = SSLContext.getInstance("TLS");

From cfbe6e5d4ab1ead118da474d86262ab614da88ad Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 25 Jan 2017 16:48:05 +0700
Subject: [PATCH 134/446] IGNITE-1596 Fixed version sort.

(cherry picked from commit 128ba07)
---
 modules/web-console/backend/app/agent.js | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index 791ea50ed58b9..961253f148a70 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -650,14 +650,14 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo)
                         const bParts = b.split('.');
 
                         for (let i = 0; i < aParts.length; ++i) {
-                            if (bParts.length === i)
-                                return 1;
+                            if (aParts[i] !== bParts[i])
+                                return aParts[i] < bParts[i] ? 1 : -1;
+                        }
 
-                            if (aParts[i] === aParts[i])
-                                continue;
+                        if (aParts.length === bParts.length)
+                            return 0;
 
-                            return aParts[i] > bParts[i] ? 1 : -1;
-                        }
+                        return aParts.length < bParts.length ? 1 : -1;
                     }));
 
                     // Latest version of agent distribution.

From 6667f3bf572b05a51c2d576f7208d7c0a20d2e9a Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 27 Jan 2017 11:30:49 +0700
Subject: [PATCH 135/446] IGNITE-4622 Fixed generation in domain model for
 cache store.

(cherry picked from commit 43007d5)
---
 .../configuration/generator/ConfigurationGenerator.js    | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 8770bf6063c73..abe361cde29d8 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -48,6 +48,10 @@ export default class IgniteConfigurationGenerator {
         return new Bean('org.apache.ignite.cache.QueryEntity', 'qryEntity', domain, cacheDflts);
     }
 
+    static domainJdbcTypeBean(domain) {
+        return new Bean('org.apache.ignite.cache.store.jdbc.JdbcType', 'type', domain);
+    }
+
     static discoveryConfigurationBean(discovery) {
         return new Bean('org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi', 'discovery', discovery, clusterDflts.discovery);
     }
@@ -1383,7 +1387,7 @@ export default class IgniteConfigurationGenerator {
     }
 
     // Generate domain model for store group.
-    static domainStore(domain, cfg = this.domainConfigurationBean(domain)) {
+    static domainStore(domain, cfg = this.domainJdbcTypeBean(domain)) {
         cfg.stringProperty('databaseSchema')
             .stringProperty('databaseTable');
 
@@ -1527,8 +1531,7 @@ export default class IgniteConfigurationGenerator {
                         if (_.isNil(domain.databaseTable))
                             return acc;
 
-                        const typeBean = new Bean('org.apache.ignite.cache.store.jdbc.JdbcType', 'type',
-                            _.merge({}, domain, {cacheName: cache.name}))
+                        const typeBean = this.domainJdbcTypeBean(_.merge({}, domain, {cacheName: cache.name}))
                             .stringProperty('cacheName');
 
                         setType(typeBean, 'keyType');

From f505e2095ce509d738b06e9442556e793fa05ab5 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Mon, 30 Jan 2017 17:18:03 +0300
Subject: [PATCH 136/446] IGNITE-4621 Hang on broadcast when
 BinaryUtils.FIELDS_SORTED_ORDER == true

---
 .../ignite/internal/binary/BinaryContext.java |  3 +
 .../ignite/internal/binary/BinaryTreeMap.java |  6 +-
 .../resources/META-INF/classnames.properties  | 62 ++++++++++++++-----
 3 files changed, 54 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 2237c27cb846a..1ad2a0f072ade 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -191,6 +191,9 @@ public class BinaryContext {
         sysClss.add(GridClosureProcessor.C4V2.class.getName());
         sysClss.add(GridClosureProcessor.C4MLAV2.class.getName());
 
+        // BinaryUtils.FIELDS_SORTED_ORDER support, since it uses TreeMap at BinaryMetadata.
+        sysClss.add(BinaryTreeMap.class.getName());
+
         if (BinaryUtils.wrapTrees()) {
             sysClss.add(TreeMap.class.getName());
             sysClss.add(TreeSet.class.getName());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java
index 6a7cf9b3db50d..3dae8ce74fa27 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.binary;
 
+import java.io.Serializable;
 import org.apache.ignite.binary.BinaryObjectException;
 import org.apache.ignite.binary.BinaryRawReader;
 import org.apache.ignite.binary.BinaryRawWriter;
@@ -32,7 +33,10 @@
 /**
  * Binary {@link TreeMap} wrapper.
  */
-public class BinaryTreeMap implements Binarylizable {
+public class BinaryTreeMap implements Binarylizable, Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
     /** Original map. */
     private TreeMap map;
 
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 8a6dc6603c4f1..df605dafb2b34 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -241,6 +241,7 @@ org.apache.ignite.internal.binary.BinaryObjectOffheapImpl
 org.apache.ignite.internal.binary.BinaryReaderExImpl$Flag
 org.apache.ignite.internal.binary.BinarySchema
 org.apache.ignite.internal.binary.BinarySchema$Confirmation
+org.apache.ignite.internal.binary.BinaryTreeMap
 org.apache.ignite.internal.binary.BinaryWriteMode
 org.apache.ignite.internal.binary.builder.BinaryLazyMap$1$1$1
 org.apache.ignite.internal.client.GridClientAuthenticationException
@@ -294,12 +295,17 @@ org.apache.ignite.internal.jdbc2.JdbcDatabaseMetadata$UpdateMetadataTask
 org.apache.ignite.internal.jdbc2.JdbcQueryTask
 org.apache.ignite.internal.jdbc2.JdbcQueryTask$1
 org.apache.ignite.internal.jdbc2.JdbcQueryTask$QueryResult
+org.apache.ignite.internal.jdbc2.JdbcQueryTaskV2
+org.apache.ignite.internal.jdbc2.JdbcQueryTaskV2$1
+org.apache.ignite.internal.jdbc2.JdbcQueryTaskV2$QueryResult
+org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery
 org.apache.ignite.internal.managers.GridManagerAdapter$1$1
 org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager$CheckpointSet
 org.apache.ignite.internal.managers.checkpoint.GridCheckpointRequest
 org.apache.ignite.internal.managers.communication.GridIoManager$ConcurrentHashMap0
 org.apache.ignite.internal.managers.communication.GridIoMessage
 org.apache.ignite.internal.managers.communication.GridIoUserMessage
+org.apache.ignite.internal.managers.communication.IgniteIoTestMessage
 org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean
 org.apache.ignite.internal.managers.deployment.GridDeploymentPerVersionStore$2
 org.apache.ignite.internal.managers.deployment.GridDeploymentRequest
@@ -376,9 +382,9 @@ org.apache.ignite.internal.processors.cache.GridCacheAdapter$12
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$13
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$14
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$15
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$16$1
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$16
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$17
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$18
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$18$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$2
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$26$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$28
@@ -387,20 +393,20 @@ org.apache.ignite.internal.processors.cache.GridCacheAdapter$3
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$30
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$32
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$4
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$48
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$49
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$50
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$51
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$52
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$53
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$54
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$55
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$57
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$58
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$58$1
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$59
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$6
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$60
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$61
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$62
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$63
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$64
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$65
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$66
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$67
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$69
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$70
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$70$1
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$71
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$72
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$9
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$AsyncOp$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$AsyncOp$1$1
@@ -1111,6 +1117,12 @@ org.apache.ignite.internal.processors.hadoop.HadoopJobStatus
 org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan
 org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo
 org.apache.ignite.internal.processors.hadoop.HadoopTaskType
+org.apache.ignite.internal.processors.hadoop.message.HadoopMessage
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopDirectShuffleMessage
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleAck
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishRequest
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishResponse
+org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleMessage
 org.apache.ignite.internal.processors.igfs.IgfsAckMessage
 org.apache.ignite.internal.processors.igfs.IgfsAttributes
 org.apache.ignite.internal.processors.igfs.IgfsBlockKey
@@ -1208,6 +1220,7 @@ org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryProcessor
 org.apache.ignite.internal.processors.platform.cache.PlatformCachePartialUpdateException
 org.apache.ignite.internal.processors.platform.cache.affinity.PlatformAffinity$1
 org.apache.ignite.internal.processors.platform.cache.affinity.PlatformAffinityFunction
+org.apache.ignite.internal.processors.platform.cache.expiry.PlatformExpiryPolicyFactory
 org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQuery
 org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQueryFilter
 org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQueryImpl
@@ -1244,6 +1257,9 @@ org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore$9
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetConfigurationClosure
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetService
 org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetServiceImpl
+org.apache.ignite.internal.processors.platform.entityframework.PlatformDotNetEntityFrameworkCacheExtension$CleanupCompletionListener
+org.apache.ignite.internal.processors.platform.entityframework.PlatformDotNetEntityFrameworkCacheExtension$RemoveOldEntriesRunnable
+org.apache.ignite.internal.processors.platform.entityframework.PlatformDotNetEntityFrameworkIncreaseVersionProcessor
 org.apache.ignite.internal.processors.platform.events.PlatformEventFilterListenerImpl
 org.apache.ignite.internal.processors.platform.message.PlatformMessageFilter
 org.apache.ignite.internal.processors.platform.messaging.PlatformMessageFilterImpl
@@ -1266,6 +1282,7 @@ org.apache.ignite.internal.processors.query.GridQueryProcessor$6
 org.apache.ignite.internal.processors.query.GridQueryProcessor$7
 org.apache.ignite.internal.processors.query.GridQueryProcessor$8
 org.apache.ignite.internal.processors.query.GridQueryProcessor$IndexType
+org.apache.ignite.internal.processors.query.IgniteSQLException
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse
 org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageRequest
@@ -1331,6 +1348,9 @@ org.apache.ignite.internal.processors.rest.handlers.datastructures.DataStructure
 org.apache.ignite.internal.processors.rest.handlers.query.CacheQueryFieldsMetaResult
 org.apache.ignite.internal.processors.rest.handlers.query.CacheQueryResult
 org.apache.ignite.internal.processors.rest.handlers.query.QueryCommandHandler$QueryCursorIterator
+org.apache.ignite.internal.processors.rest.handlers.redis.GridRedisRestCommandHandler$1
+org.apache.ignite.internal.processors.rest.handlers.redis.exception.GridRedisGenericException
+org.apache.ignite.internal.processors.rest.handlers.redis.exception.GridRedisTypeException
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler$2
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler$ExeCallable
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultRequest
@@ -1342,6 +1362,9 @@ org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpMemcachedNioList
 org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpMemcachedNioListener$2
 org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpRestNioListener$1
 org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpRestNioListener$1$1
+org.apache.ignite.internal.processors.rest.protocols.tcp.redis.GridRedisCommand
+org.apache.ignite.internal.processors.rest.protocols.tcp.redis.GridRedisMessage
+org.apache.ignite.internal.processors.rest.protocols.tcp.redis.GridRedisNioListener$1
 org.apache.ignite.internal.processors.rest.request.RestQueryRequest$QueryType
 org.apache.ignite.internal.processors.service.GridServiceAssignments
 org.apache.ignite.internal.processors.service.GridServiceAssignmentsKey
@@ -1586,10 +1609,14 @@ org.apache.ignite.internal.util.lang.IgniteReducer2X
 org.apache.ignite.internal.util.lang.IgniteReducer3
 org.apache.ignite.internal.util.lang.IgniteReducer3X
 org.apache.ignite.internal.util.lang.IgniteReducerX
+org.apache.ignite.internal.util.lang.IgniteSingletonIterator
 org.apache.ignite.internal.util.nio.GridNioEmbeddedFuture$1
 org.apache.ignite.internal.util.nio.GridNioException
 org.apache.ignite.internal.util.nio.GridNioMessageTracker
 org.apache.ignite.internal.util.nio.GridNioServer$NioOperation
+org.apache.ignite.internal.util.nio.GridNioServer$RandomBalancer
+org.apache.ignite.internal.util.nio.GridNioServer$ReadWriteSizeBasedBalancer
+org.apache.ignite.internal.util.nio.GridNioServer$SizeBasedBalancer
 org.apache.ignite.internal.util.nio.GridNioSessionMetaKey
 org.apache.ignite.internal.util.nio.ssl.GridNioSslHandler
 org.apache.ignite.internal.util.offheap.GridOffHeapEvent
@@ -1864,12 +1891,15 @@ org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointData
 org.apache.ignite.spi.collision.jobstealing.JobStealingRequest
 org.apache.ignite.spi.collision.priorityqueue.PriorityQueueCollisionSpi$PriorityGridCollisionJobContextComparator
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$1
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$10
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$11
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure$1
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$8
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$9
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosureNew
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosureNew$1
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeClosure
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeMessage
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeMessage2
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeTimeoutException
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$NodeIdMessage
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$RecoveryLastReceivedMessage

From 607f4eb7cea1e7392fac584772b3b86851eaf57c Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Thu, 15 Dec 2016 12:31:56 +0300
Subject: [PATCH 137/446] ignite-4429 Deadlock in IgniteAtomicSequence when
 used inside transaction

(cherry picked from commit 9905ff3)
---
 .../GridCacheAtomicSequenceImpl.java          |  12 +-
 ...hePartitionedAtomicSequenceTxSelfTest.java | 169 ++++++++++++++++++
 ...gniteCacheDataStructuresSelfTestSuite.java |   2 +
 3 files changed, 181 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSequenceTxSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
index 4f660b6115525..754d8f50a1ca9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
@@ -29,6 +29,7 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -253,8 +254,15 @@ private long internalUpdate(long l, @Nullable Callable<Long> updateCall, boolean
         while (true) {
             if (updateGuard.compareAndSet(false, true)) {
                 try {
-                    // This call must be outside lock.
-                    return CU.outTx(updateCall, ctx);
+                    try {
+                        return updateCall.call();
+                    }
+                    catch (IgniteCheckedException | IgniteException | IllegalStateException e) {
+                        throw e;
+                    }
+                    catch (Exception e) {
+                        throw new IgniteCheckedException(e);
+                    }
                 }
                 finally {
                     lock.lock();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSequenceTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSequenceTxSelfTest.java
new file mode 100644
index 0000000000000..d04a68afdb53e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSequenceTxSelfTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.datastructures.partitioned;
+
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteAtomicSequence;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.AtomicConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+
+/**
+ * Tests {@link IgniteAtomicSequence} operations inside started user transaction.
+ */
+public class GridCachePartitionedAtomicSequenceTxSelfTest extends GridCommonAbstractTest {
+    /** Number of threads. */
+    private static final int THREAD_NUM = 8;
+
+    /** Sequence cache size. */
+    private static final int SEQ_CACHE_SIZE = 10;
+
+    /** Iterations. */
+    private static final int ITERATIONS = 100;
+
+    /** Sequence name. */
+    private static final String SEQ_NAME = "seq";
+
+    /** Latch. */
+    private static CountDownLatch latch;
+
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi spi = new TcpDiscoverySpi();
+
+        spi.setIpFinder(ipFinder);
+
+        cfg.setDiscoverySpi(spi);
+
+        cfg.setPublicThreadPoolSize(THREAD_NUM);
+
+        AtomicConfiguration atomicCfg = atomicConfiguration();
+
+        assertNotNull(atomicCfg);
+
+        cfg.setAtomicConfiguration(atomicCfg);
+
+        return cfg;
+    }
+
+    /**
+     * @return Atomic config for test.
+     */
+    protected AtomicConfiguration atomicConfiguration() {
+        AtomicConfiguration cfg = new AtomicConfiguration();
+
+        cfg.setBackups(1);
+        cfg.setAtomicSequenceReserveSize(SEQ_CACHE_SIZE);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        latch = new CountDownLatch(THREAD_NUM);
+
+        startGridsMultiThreaded(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * Tests sequence calls inside transactions.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTransactionIncrement() throws Exception {
+        ignite(0).atomicSequence(SEQ_NAME, 0, true);
+
+        for (int i = 0; i < THREAD_NUM; i++) {
+            multithreaded(new Runnable() {
+                @Override public void run() {
+                    ignite(0).compute().run(new IncrementClosure());
+
+                }
+            }, THREAD_NUM);
+        }
+    }
+
+    /**
+     * Tests isolation of system and user transactions.
+     */
+    public void testIsolation() {
+        IgniteAtomicSequence seq = ignite(0).atomicSequence(SEQ_NAME, 0, true);
+
+        CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>();
+        ccfg.setAtomicityMode(TRANSACTIONAL);
+
+        IgniteCache<Object, Object> cache = ignite(0).getOrCreateCache(ccfg);
+
+        try (Transaction tx = ignite(0).transactions().txStart()) {
+            seq.getAndIncrement();
+
+            cache.put(1, 1);
+
+            tx.rollback();
+        }
+
+        assertEquals(0, cache.size());
+        assertEquals(new Long(1L), U.field(seq, "locVal"));
+        assertEquals(new Long(SEQ_CACHE_SIZE - 1), U.field(seq, "upBound"));
+    }
+
+    /**
+     * Closure which does sequence increment.
+     */
+    private static class IncrementClosure implements IgniteRunnable {
+        /** Ignite instance. */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            IgniteAtomicSequence seq = ignite.atomicSequence(SEQ_NAME, 0, false);
+
+            latch.countDown();
+
+            U.awaitQuiet(latch);
+
+            for (int i = 0; i < ITERATIONS; i++)
+                try (Transaction ignored = ignite.transactions().txStart()) {
+                    seq.incrementAndGet();
+                }
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
index 45a49bf899599..267128b6055ef 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
@@ -46,6 +46,7 @@
 import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedAtomicQueueRotativeMultiNodeTest;
 import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedAtomicReferenceApiSelfTest;
 import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedAtomicSequenceMultiThreadedTest;
+import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedAtomicSequenceTxSelfTest;
 import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedAtomicSetFailoverSelfTest;
 import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedAtomicSetSelfTest;
 import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedAtomicStampedApiSelfTest;
@@ -168,6 +169,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(IgniteReplicatedAtomicLongApiSelfTest.class));
 
         suite.addTest(new TestSuite(GridCachePartitionedAtomicSequenceMultiThreadedTest.class));
+        suite.addTest(new TestSuite(GridCachePartitionedAtomicSequenceTxSelfTest.class));
 
         suite.addTest(new TestSuite(GridCachePartitionedAtomicStampedApiSelfTest.class));
         suite.addTest(new TestSuite(GridCacheReplicatedAtomicStampedApiSelfTest.class));

From d22bcf6c72db5122b62e80f7719b22de67190e6a Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Mon, 30 Jan 2017 17:18:03 +0300
Subject: [PATCH 138/446] IGNITE-4621 Hang on broadcast when
 BinaryUtils.FIELDS_SORTED_ORDER == true

---
 .../org/apache/ignite/internal/binary/BinaryContext.java    | 3 +++
 .../org/apache/ignite/internal/binary/BinaryTreeMap.java    | 6 +++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 4030ef02c799c..7b21dfbed51f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -190,6 +190,9 @@ public class BinaryContext {
         sysClss.add(GridClosureProcessor.C4V2.class.getName());
         sysClss.add(GridClosureProcessor.C4MLAV2.class.getName());
 
+        // BinaryUtils.FIELDS_SORTED_ORDER support, since it uses TreeMap at BinaryMetadata.
+        sysClss.add(BinaryTreeMap.class.getName());
+
         if (BinaryUtils.wrapTrees()) {
             sysClss.add(TreeMap.class.getName());
             sysClss.add(TreeSet.class.getName());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java
index 6a7cf9b3db50d..3dae8ce74fa27 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.binary;
 
+import java.io.Serializable;
 import org.apache.ignite.binary.BinaryObjectException;
 import org.apache.ignite.binary.BinaryRawReader;
 import org.apache.ignite.binary.BinaryRawWriter;
@@ -32,7 +33,10 @@
 /**
  * Binary {@link TreeMap} wrapper.
  */
-public class BinaryTreeMap implements Binarylizable {
+public class BinaryTreeMap implements Binarylizable, Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
     /** Original map. */
     private TreeMap map;
 

From a8355feef630a5c8c4edbe02224c8325dd5952d3 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 3 Feb 2017 11:58:43 +0700
Subject: [PATCH 139/446] IGNITE-4610 Added more informative message.

(cherry picked from commit 9ff2a8f)
---
 .../web-console/frontend/views/templates/agent-download.jade    | 2 +-
 modules/web-console/web-agent/README.txt                        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/frontend/views/templates/agent-download.jade b/modules/web-console/frontend/views/templates/agent-download.jade
index 20b6b03d058f3..f57bf1d87e21c 100644
--- a/modules/web-console/frontend/views/templates/agent-download.jade
+++ b/modules/web-console/frontend/views/templates/agent-download.jade
@@ -41,7 +41,7 @@
                 p Connection to Ignite Web Agent is established, but agent failed to connect to Ignite Node
                 p Please check the following:
                 ul
-                    li Ignite Grid is up and running
+                    li Ignite Grid is up and Ignite REST server started (copy "ignite-rest-http" folder from libs/optional/ to libs/)
                     li In agent settings check URI for connect to Ignite REST server
                     li Check agent logs for errors
                     li Refer to #[b README.txt] in agent folder for more information
diff --git a/modules/web-console/web-agent/README.txt b/modules/web-console/web-agent/README.txt
index cc0c80fc99adb..81df1cb4f0234 100644
--- a/modules/web-console/web-agent/README.txt
+++ b/modules/web-console/web-agent/README.txt
@@ -32,7 +32,7 @@ Security tokens:
   3) One may specify several comma separated tokens using configuration file or command line arguments of web agent.
 
 Ignite Web agent requirements:
-  1) In order to communicate with web agent Ignite node should be started with REST server (move ignite-rest-http folder from lib/optional/ to lib/).
+  1) In order to communicate with web agent Ignite node should be started with REST server (copy "ignite-rest-http" folder from "libs/optional/" to "libs/").
   2) Configure web agent serverURI property by Ignite node REST server URI.
 
 Options:

From ad91eac2d67d594f4a3def7fdfa2590ef5721b63 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 3 Feb 2017 11:58:43 +0700
Subject: [PATCH 140/446] IGNITE-4610 Added more informative message.

(cherry picked from commit 9ff2a8f)
---
 .../web-console/frontend/views/templates/agent-download.jade    | 2 +-
 modules/web-console/web-agent/README.txt                        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/frontend/views/templates/agent-download.jade b/modules/web-console/frontend/views/templates/agent-download.jade
index 20b6b03d058f3..f57bf1d87e21c 100644
--- a/modules/web-console/frontend/views/templates/agent-download.jade
+++ b/modules/web-console/frontend/views/templates/agent-download.jade
@@ -41,7 +41,7 @@
                 p Connection to Ignite Web Agent is established, but agent failed to connect to Ignite Node
                 p Please check the following:
                 ul
-                    li Ignite Grid is up and running
+                    li Ignite Grid is up and Ignite REST server started (copy "ignite-rest-http" folder from libs/optional/ to libs/)
                     li In agent settings check URI for connect to Ignite REST server
                     li Check agent logs for errors
                     li Refer to #[b README.txt] in agent folder for more information
diff --git a/modules/web-console/web-agent/README.txt b/modules/web-console/web-agent/README.txt
index cc0c80fc99adb..81df1cb4f0234 100644
--- a/modules/web-console/web-agent/README.txt
+++ b/modules/web-console/web-agent/README.txt
@@ -32,7 +32,7 @@ Security tokens:
   3) One may specify several comma separated tokens using configuration file or command line arguments of web agent.
 
 Ignite Web agent requirements:
-  1) In order to communicate with web agent Ignite node should be started with REST server (move ignite-rest-http folder from lib/optional/ to lib/).
+  1) In order to communicate with web agent Ignite node should be started with REST server (copy "ignite-rest-http" folder from "libs/optional/" to "libs/").
   2) Configure web agent serverURI property by Ignite node REST server URI.
 
 Options:

From 9c401b0dcfc6edb5cc83f6b35bc0d24db6f10347 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 24 Jan 2017 16:40:54 +0300
Subject: [PATCH 141/446] IGNITE-4537 - Filter possible sensitive user data
 that might be set in system properties to send in update notifier.

---
 .../apache/ignite/IgniteSystemProperties.java | 32 +++++++++++++++++++
 .../cluster/GridUpdateNotifier.java           |  2 +-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index bf51c342a2954..f0270986fc4b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Properties;
 import javax.net.ssl.HostnameVerifier;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.jetbrains.annotations.Nullable;
 
@@ -526,6 +527,15 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI = "IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI";
 
+    /** Returns true for system properties only avoiding sending sensitive information. */
+    private static final IgnitePredicate<Map.Entry<String, String>> PROPS_FILTER = new IgnitePredicate<Map.Entry<String, String>>() {
+        @Override public boolean apply(final Map.Entry<String, String> entry) {
+            final String key = entry.getKey();
+
+            return key.startsWith("java.") || key.startsWith("os.") || key.startsWith("user.");
+        }
+    };
+
     /**
      * Enforces singleton.
      */
@@ -699,4 +709,26 @@ public static Properties snapshot() {
 
         return sysProps;
     }
+
+    /**
+     * Does the same as {@link #snapshot()} but filters out
+     * possible sensitive user data.
+     *
+     * @return Snapshot of system properties.
+     */
+    @SuppressWarnings("unchecked")
+    public static Properties safeSnapshot() {
+        final Properties props = snapshot();
+
+        final Iterator<Map.Entry<Object, Object>> iter = props.entrySet().iterator();
+
+        while (iter.hasNext()) {
+            final Map.Entry entry = iter.next();
+
+            if (!PROPS_FILTER.apply(entry))
+                iter.remove();
+        }
+
+        return props;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
index 592fdd10bde73..5b2edcd94bab3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
@@ -169,7 +169,7 @@ private static String getSystemProperties() {
             StringWriter sw = new StringWriter();
 
             try {
-                IgniteSystemProperties.snapshot().store(new PrintWriter(sw), "");
+                IgniteSystemProperties.safeSnapshot().store(new PrintWriter(sw), "");
             }
             catch (IOException ignore) {
                 return null;

From 6e68f64d484371c1d7e43dbe0b95906faf15242c Mon Sep 17 00:00:00 2001
From: Ignite Teamcity <ignite@apache.org>
Date: Fri, 3 Feb 2017 13:01:27 +0300
Subject: [PATCH 142/446] 1.9.0-SNAPSHOT

---
 examples/pom.xml                                     |  2 +-
 examples/schema-import/pom.xml                       |  2 +-
 modules/aop/pom.xml                                  |  2 +-
 modules/apache-license-gen/pom.xml                   |  2 +-
 modules/aws/pom.xml                                  |  2 +-
 modules/benchmarks/pom.xml                           |  2 +-
 modules/camel/pom.xml                                |  2 +-
 modules/cassandra/pom.xml                            |  2 +-
 modules/cassandra/serializers/pom.xml                |  4 ++--
 modules/cassandra/store/pom.xml                      |  4 ++--
 modules/clients/pom.xml                              | 12 +++++-------
 modules/cloud/pom.xml                                |  2 +-
 modules/codegen/pom.xml                              |  2 +-
 modules/core/pom.xml                                 |  2 +-
 modules/core/src/main/resources/ignite.properties    |  2 +-
 modules/extdata/p2p/pom.xml                          |  2 +-
 modules/extdata/uri/modules/uri-dependency/pom.xml   |  2 +-
 modules/extdata/uri/pom.xml                          |  2 +-
 modules/flink/pom.xml                                |  2 +-
 modules/flume/pom.xml                                |  2 +-
 modules/gce/pom.xml                                  |  2 +-
 modules/geospatial/pom.xml                           |  2 +-
 modules/hadoop/pom.xml                               |  2 +-
 modules/hibernate/pom.xml                            |  2 +-
 modules/indexing/pom.xml                             |  2 +-
 modules/jcl/pom.xml                                  |  2 +-
 modules/jms11/pom.xml                                |  2 +-
 modules/jta/pom.xml                                  |  2 +-
 modules/kafka/pom.xml                                |  2 +-
 modules/log4j/pom.xml                                |  2 +-
 modules/log4j2/pom.xml                               |  2 +-
 modules/mesos/pom.xml                                |  2 +-
 modules/mqtt/pom.xml                                 |  2 +-
 modules/osgi-karaf/pom.xml                           |  2 +-
 modules/osgi-paxlogging/pom.xml                      |  2 +-
 modules/osgi/pom.xml                                 |  2 +-
 modules/platforms/cpp/configure.ac                   |  2 +-
 modules/platforms/cpp/configure.acrel                |  2 +-
 modules/platforms/cpp/examples/configure.ac          |  2 +-
 .../platforms/cpp/odbc/install/ignite-odbc-amd64.wxs |  2 +-
 .../platforms/cpp/odbc/install/ignite-odbc-x86.wxs   |  2 +-
 .../Properties/AssemblyInfo.cs                       |  6 +++---
 .../Apache.Ignite.AspNet/Properties/AssemblyInfo.cs  |  6 +++---
 .../Properties/AssemblyInfo.cs                       |  6 +++---
 .../Properties/AssemblyInfo.cs                       |  6 +++---
 .../Properties/AssemblyInfo.cs                       |  6 +++---
 .../Properties/AssemblyInfo.cs                       |  6 +++---
 .../Apache.Ignite.Core/Properties/AssemblyInfo.cs    |  6 +++---
 .../Properties/AssemblyInfo.cs                       |  6 +++---
 .../Properties/AssemblyInfo.cs                       |  6 +++---
 .../Apache.Ignite.Linq/Properties/AssemblyInfo.cs    |  6 +++---
 .../Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs |  6 +++---
 .../Apache.Ignite.NLog/Properties/AssemblyInfo.cs    |  6 +++---
 .../dotnet/Apache.Ignite/Properties/AssemblyInfo.cs  |  6 +++---
 .../Properties/AssemblyInfo.cs                       |  6 +++---
 .../Properties/AssemblyInfo.cs                       |  6 +++---
 modules/rest-http/pom.xml                            |  2 +-
 modules/scalar-2.10/pom.xml                          |  2 +-
 modules/scalar/pom.xml                               |  2 +-
 modules/schedule/pom.xml                             |  2 +-
 modules/schema-import-db/pom.xml                     |  2 +-
 modules/schema-import/pom.xml                        |  2 +-
 modules/slf4j/pom.xml                                |  2 +-
 modules/spark-2.10/pom.xml                           |  2 +-
 modules/spark/pom.xml                                |  2 +-
 modules/spring/pom.xml                               |  2 +-
 modules/ssh/pom.xml                                  |  2 +-
 modules/storm/pom.xml                                |  2 +-
 modules/tools/pom.xml                                |  2 +-
 modules/twitter/pom.xml                              |  2 +-
 modules/urideploy/pom.xml                            |  2 +-
 modules/visor-console-2.10/pom.xml                   |  2 +-
 modules/visor-console/pom.xml                        |  2 +-
 modules/visor-plugins/pom.xml                        |  2 +-
 modules/web-console/pom.xml                          |  2 +-
 modules/web-console/web-agent/pom.xml                |  2 +-
 modules/web/ignite-appserver-test/pom.xml            |  2 +-
 modules/web/ignite-websphere-test/pom.xml            |  2 +-
 modules/web/pom.xml                                  |  2 +-
 modules/yardstick/pom.xml                            |  2 +-
 modules/yarn/pom.xml                                 |  2 +-
 modules/zookeeper/pom.xml                            |  2 +-
 pom.xml                                              |  2 +-
 83 files changed, 119 insertions(+), 121 deletions(-)

diff --git a/examples/pom.xml b/examples/pom.xml
index 6f5556015e827..255ce019fd147 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -28,7 +28,7 @@
     </parent>
 
     <artifactId>ignite-examples</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/examples/schema-import/pom.xml b/examples/schema-import/pom.xml
index 8ca92dd70dbe2..93ad1a53c7525 100644
--- a/examples/schema-import/pom.xml
+++ b/examples/schema-import/pom.xml
@@ -35,7 +35,7 @@
     </properties>
 
     <artifactId>ignite-schema-import-demo</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/aop/pom.xml b/modules/aop/pom.xml
index d0314af927805..46649c2d9b797 100644
--- a/modules/aop/pom.xml
+++ b/modules/aop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aop</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/apache-license-gen/pom.xml b/modules/apache-license-gen/pom.xml
index 74cf35e22bb59..fdd2a6e533a32 100644
--- a/modules/apache-license-gen/pom.xml
+++ b/modules/apache-license-gen/pom.xml
@@ -31,7 +31,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>ignite-apache-license-gen</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <build>
diff --git a/modules/aws/pom.xml b/modules/aws/pom.xml
index ee300d4acc2fd..91ad5935b16db 100644
--- a/modules/aws/pom.xml
+++ b/modules/aws/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aws</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/benchmarks/pom.xml b/modules/benchmarks/pom.xml
index 4a18c1890d0f9..6bdac8cb7beb4 100644
--- a/modules/benchmarks/pom.xml
+++ b/modules/benchmarks/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-benchmarks</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/camel/pom.xml b/modules/camel/pom.xml
index 841cfd92ff9e3..8f1feb4b823ed 100644
--- a/modules/camel/pom.xml
+++ b/modules/camel/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-camel</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/pom.xml b/modules/cassandra/pom.xml
index a665538fc5567..7ec01b542094e 100644
--- a/modules/cassandra/pom.xml
+++ b/modules/cassandra/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-cassandra</artifactId>
     <packaging>pom</packaging>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencyManagement>
diff --git a/modules/cassandra/serializers/pom.xml b/modules/cassandra/serializers/pom.xml
index 33be131ce7b3c..3c593c0e50258 100644
--- a/modules/cassandra/serializers/pom.xml
+++ b/modules/cassandra/serializers/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>1.8.0-SNAPSHOT</version>
+        <version>1.9.0-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-serializers</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/store/pom.xml b/modules/cassandra/store/pom.xml
index 0b233fa105623..a81800a5f06c3 100644
--- a/modules/cassandra/store/pom.xml
+++ b/modules/cassandra/store/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>1.8.0-SNAPSHOT</version>
+        <version>1.9.0-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-store</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/clients/pom.xml b/modules/clients/pom.xml
index 2e29cc068f949..d6cbbc1f93fc2 100644
--- a/modules/clients/pom.xml
+++ b/modules/clients/pom.xml
@@ -20,8 +20,7 @@
 <!--
     POM file.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
@@ -32,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-clients</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
@@ -168,11 +167,10 @@
                                 <configuration>
                                     <target>
                                         <exec executable="${doxygen.exec}" searchpath="true" dir="../platforms/dotnet/">
-                                            <arg value="Apache.Ignite.dxg"/>
+                                            <arg value="Apache.Ignite.dxg" />
                                         </exec>
 
-                                        <copy file="../../assembly/docfiles/ignite_logo.png"
-                                              todir="target/dotnetdoc/html"/>
+                                        <copy file="../../assembly/docfiles/ignite_logo.png" todir="target/dotnetdoc/html" />
                                     </target>
                                 </configuration>
                             </execution>
@@ -186,7 +184,7 @@
                                 <configuration>
                                     <target>
                                         <exec executable="${doxygen.exec}" searchpath="true" dir="../platforms/cpp/">
-                                            <arg value="cpp.dxg"/>
+                                            <arg value="cpp.dxg" />
                                         </exec>
                                     </target>
                                 </configuration>
diff --git a/modules/cloud/pom.xml b/modules/cloud/pom.xml
index 1bcc4bfb1a147..8f59524515b0a 100644
--- a/modules/cloud/pom.xml
+++ b/modules/cloud/pom.xml
@@ -29,7 +29,7 @@
     </parent>
 
     <artifactId>ignite-cloud</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/codegen/pom.xml b/modules/codegen/pom.xml
index be287e373088b..2390e14cba566 100644
--- a/modules/codegen/pom.xml
+++ b/modules/codegen/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-codegen</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 4c4343a9c41a1..f12cdfa3d510d 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-core</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <repositories>
diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index dcebbe4b5b4fc..75859adb5941f 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=1.8.0-SNAPSHOT
+ignite.version=1.9.0-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970
diff --git a/modules/extdata/p2p/pom.xml b/modules/extdata/p2p/pom.xml
index 73a80cf607db7..eb20e280b68a7 100644
--- a/modules/extdata/p2p/pom.xml
+++ b/modules/extdata/p2p/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-extdata-p2p</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/extdata/uri/modules/uri-dependency/pom.xml b/modules/extdata/uri/modules/uri-dependency/pom.xml
index 40518fb117a0b..625cbad3a1061 100644
--- a/modules/extdata/uri/modules/uri-dependency/pom.xml
+++ b/modules/extdata/uri/modules/uri-dependency/pom.xml
@@ -27,7 +27,7 @@
     <artifactId>ignite-extdata-uri-dep</artifactId>
     <packaging>jar</packaging>
 
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <modelVersion>4.0.0</modelVersion>
 
     <dependencies>
diff --git a/modules/extdata/uri/pom.xml b/modules/extdata/uri/pom.xml
index ff30d0aa258e7..15f8524dde1d3 100644
--- a/modules/extdata/uri/pom.xml
+++ b/modules/extdata/uri/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-uri</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/flink/pom.xml b/modules/flink/pom.xml
index cd80f165be7a0..27c8928ec2e81 100644
--- a/modules/flink/pom.xml
+++ b/modules/flink/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flink</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/flume/pom.xml b/modules/flume/pom.xml
index 5e6fa6427710e..eafa6ce863667 100644
--- a/modules/flume/pom.xml
+++ b/modules/flume/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flume</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/gce/pom.xml b/modules/gce/pom.xml
index 8340512e876ee..a6c0a4992765c 100644
--- a/modules/gce/pom.xml
+++ b/modules/gce/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-gce</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/geospatial/pom.xml b/modules/geospatial/pom.xml
index e65339bafd715..662411589a2b0 100644
--- a/modules/geospatial/pom.xml
+++ b/modules/geospatial/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-geospatial</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index db302d7f13278..98291c6848f89 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hadoop</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate/pom.xml b/modules/hibernate/pom.xml
index 40a3c95586139..96e71477077d1 100644
--- a/modules/hibernate/pom.xml
+++ b/modules/hibernate/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml
index 58e82ed278dd6..aaaced3528954 100644
--- a/modules/indexing/pom.xml
+++ b/modules/indexing/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-indexing</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jcl/pom.xml b/modules/jcl/pom.xml
index 37f475be4d1ba..41ec2d5daeaa4 100644
--- a/modules/jcl/pom.xml
+++ b/modules/jcl/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jcl</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jms11/pom.xml b/modules/jms11/pom.xml
index eddeb707a55e1..e7a8e4bb3d550 100644
--- a/modules/jms11/pom.xml
+++ b/modules/jms11/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jms11</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jta/pom.xml b/modules/jta/pom.xml
index 1f22f0276e29b..f6fd4259a434e 100644
--- a/modules/jta/pom.xml
+++ b/modules/jta/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jta</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kafka/pom.xml b/modules/kafka/pom.xml
index 96e3d0decc949..9aff624f36c5d 100644
--- a/modules/kafka/pom.xml
+++ b/modules/kafka/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kafka</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j/pom.xml b/modules/log4j/pom.xml
index 24f31076ef0ea..e4e4c33902a50 100644
--- a/modules/log4j/pom.xml
+++ b/modules/log4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j2/pom.xml b/modules/log4j2/pom.xml
index 99174ea2448b4..e0a15170082cd 100644
--- a/modules/log4j2/pom.xml
+++ b/modules/log4j2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j2</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml
index f407a3f4e84ac..4c717c24091ac 100644
--- a/modules/mesos/pom.xml
+++ b/modules/mesos/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mesos</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/mqtt/pom.xml b/modules/mqtt/pom.xml
index 4b0e1a9635aa7..d475d1828bc48 100644
--- a/modules/mqtt/pom.xml
+++ b/modules/mqtt/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mqtt</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/osgi-karaf/pom.xml b/modules/osgi-karaf/pom.xml
index c2d455379b157..98bafe72a1345 100644
--- a/modules/osgi-karaf/pom.xml
+++ b/modules/osgi-karaf/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-karaf</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <build>
diff --git a/modules/osgi-paxlogging/pom.xml b/modules/osgi-paxlogging/pom.xml
index 3cb2a88a49188..1b5c5f883d97c 100644
--- a/modules/osgi-paxlogging/pom.xml
+++ b/modules/osgi-paxlogging/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-paxlogging</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <dependencies>
diff --git a/modules/osgi/pom.xml b/modules/osgi/pom.xml
index 5232a10a6cf6c..63fbc405fe9e9 100644
--- a/modules/osgi/pom.xml
+++ b/modules/osgi/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/platforms/cpp/configure.ac b/modules/platforms/cpp/configure.ac
index 500bdbff2cdc2..283097b291568 100644
--- a/modules/platforms/cpp/configure.ac
+++ b/modules/platforms/cpp/configure.ac
@@ -19,7 +19,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([Apache Ignite C++], [1.8.0.16695], [dev@ignite.apache.org], [ignite], [ignite.apache.org])
+AC_INIT([Apache Ignite C++], [1.9.0.17605], [dev@ignite.apache.org], [ignite], [ignite.apache.org])
 
 AC_CANONICAL_HOST
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/modules/platforms/cpp/configure.acrel b/modules/platforms/cpp/configure.acrel
index 984aa38c89d1f..3d1f885152ebc 100644
--- a/modules/platforms/cpp/configure.acrel
+++ b/modules/platforms/cpp/configure.acrel
@@ -19,7 +19,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([Apache Ignite C++], [1.8.0.16695], [dev@ignite.apache.org], [ignite], [ignite.apache.org])
+AC_INIT([Apache Ignite C++], [1.9.0.17605], [dev@ignite.apache.org], [ignite], [ignite.apache.org])
 
 AC_CANONICAL_HOST
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/modules/platforms/cpp/examples/configure.ac b/modules/platforms/cpp/examples/configure.ac
index 6f08490aa08ef..094db24612503 100644
--- a/modules/platforms/cpp/examples/configure.ac
+++ b/modules/platforms/cpp/examples/configure.ac
@@ -19,7 +19,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([Apache Ignite C++ Examples], [1.8.0.16695], [dev@ignite.apache.org], [ignite-examples], [ignite.apache.org])
+AC_INIT([Apache Ignite C++ Examples], [1.9.0.17605], [dev@ignite.apache.org], [ignite-examples], [ignite.apache.org])
 
 AC_CANONICAL_HOST
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/modules/platforms/cpp/odbc/install/ignite-odbc-amd64.wxs b/modules/platforms/cpp/odbc/install/ignite-odbc-amd64.wxs
index 2c12d9a2eac08..532f2867aba00 100644
--- a/modules/platforms/cpp/odbc/install/ignite-odbc-amd64.wxs
+++ b/modules/platforms/cpp/odbc/install/ignite-odbc-amd64.wxs
@@ -21,7 +21,7 @@
 	<Product Name='Apache Ignite ODBC 64-bit Driver' Manufacturer='The Apache Software Foundation'
 		Id='F3E308E4-910C-4AF5-82DE-2ACF4D64830E' 
 		UpgradeCode='1D7AEFDF-6CD2-4FB5-88F2-811A89832D6D'
-		Language='1033' Codepage='1252' Version='1.8.0.16695'>
+		Language='1033' Codepage='1252' Version='1.9.0.17605'>
 		
 		<Package Id='*' Keywords='Installer' Description="Apache Ignite ODBC 64-bit Driver Installer"
 			Comments='Apache, Apache Ignite, the Apache feather and the Apache Ignite logo are trademarks of The Apache Software Foundation.'
diff --git a/modules/platforms/cpp/odbc/install/ignite-odbc-x86.wxs b/modules/platforms/cpp/odbc/install/ignite-odbc-x86.wxs
index 9e77e2ad14337..3cc6fa68e4b1b 100644
--- a/modules/platforms/cpp/odbc/install/ignite-odbc-x86.wxs
+++ b/modules/platforms/cpp/odbc/install/ignite-odbc-x86.wxs
@@ -21,7 +21,7 @@
 	<Product Name='Apache Ignite ODBC 32-bit Driver' Manufacturer='The Apache Software Foundation'
 		Id='D39CBABA-1E21-4701-AA5C-91EDA07B383B' 
 		UpgradeCode='743902A4-365C-424E-B226-5B2898A3941E'
-		Language='1033' Codepage='1252' Version='1.8.0.16695'>
+		Language='1033' Codepage='1252' Version='1.9.0.17605'>
 		
 		<Package Id='*' Keywords='Installer' Description="Apache Ignite ODBC 32-bit Driver Installer"
 			Comments='Apache, Apache Ignite, the Apache feather and the Apache Ignite logo are trademarks of The Apache Software Foundation.'
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
index d6cb3df14b072..0c7f8312a828e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
@@ -35,8 +35,8 @@
 
 [assembly: Guid("18ea4c71-a11d-4ab1-8042-418f7559d84f")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
 
 [assembly: CLSCompliant(true)]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
index 56e67ffae3a17..a1c19d31bdedd 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
@@ -33,9 +33,9 @@
 
 [assembly: Guid("13ea96fc-cc83-4164-a7c0-4f30ed797460")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
 
 [assembly: CLSCompliant(true)]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
index 9537860bde5dd..49a039533e019 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
@@ -31,6 +31,6 @@
 
 [assembly: Guid("8fae8395-7e91-411a-a78f-44d6d3fed0fc")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
\ No newline at end of file
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
index 88e619a6c0282..b123427808c9c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
@@ -30,6 +30,6 @@
 [assembly: ComVisible(false)]
 [assembly: Guid("134707f6-155d-47f6-9eb2-c67abbf3c009")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
\ No newline at end of file
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
index d93861922dedf..0b8ce1ed3182b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
@@ -45,6 +45,6 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
index b236907ecf25f..463ed3e391c32 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
@@ -31,6 +31,6 @@
 
 [assembly: Guid("de8dd5cc-7c7f-4a09-80d5-7086d9416a7b")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
\ No newline at end of file
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
index 91e569ace4880..2a14a2583e751 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
@@ -33,9 +33,9 @@
 
 [assembly: Guid("97db45a8-f922-456a-a819-7b3c6e5e03ba")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
 
 [assembly: CLSCompliant(true)]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.EntityFramework.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.EntityFramework.Tests/Properties/AssemblyInfo.cs
index fe5e7ce24dcbf..67b1f88c80097 100644
--- a/modules/platforms/dotnet/Apache.Ignite.EntityFramework.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.EntityFramework.Tests/Properties/AssemblyInfo.cs
@@ -32,8 +32,8 @@
 
 [assembly: Guid("cda5700e-78f3-4a9e-a9b0-704cbe94651c")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
 
 [assembly: CLSCompliant(true)]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs
index 7ce4c5ff692af..acf46b0a873e8 100644
--- a/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs
@@ -32,9 +32,9 @@
 [assembly: ComVisible(false)]
 [assembly: Guid("c558518a-c1a0-4224-aaa9-a8688474b4dc")]
 
-[assembly: AssemblyVersion("1.8.0.14218")]
-[assembly: AssemblyFileVersion("1.8.0.14218")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
 
 [assembly: CLSCompliant(true)]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
index a44235564d752..8e07c7d16cefc 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
@@ -33,8 +33,8 @@
 // The following GUID is for the ID of the typelib if this project is exposed to COM
 [assembly: Guid("5b571661-17f4-4f29-8c7d-0edb38ca9b55")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
 
 [assembly: CLSCompliant(true)]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
index 824252e90449d..59d31088df94d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
@@ -33,8 +33,8 @@
 // The following GUID is for the ID of the typelib if this project is exposed to COM
 [assembly: Guid("6f82d669-382e-4435-8092-68c4440146d8")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
 
 [assembly: CLSCompliant(true)]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
index f28278c55723b..d40e9817a6079 100644
--- a/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
@@ -33,8 +33,8 @@
 // The following GUID is for the ID of the typelib if this project is exposed to COM
 [assembly: Guid("c6b58e4a-a2e9-4554-ad02-68ce6da5cfb7")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
 
 [assembly: CLSCompliant(true)]
diff --git a/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
index 17d065f771c6a..d805afb66ca25 100644
--- a/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
@@ -31,6 +31,6 @@
 
 [assembly: Guid("0f9702ec-da7d-4ce5-b4b7-73310c885355")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
\ No newline at end of file
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
index 224efd8d769ca..9082035b247f4 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
@@ -31,6 +31,6 @@
 
 [assembly: Guid("41a0cb95-3435-4c78-b867-900b28e2c9ee")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
\ No newline at end of file
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
\ No newline at end of file
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
index 8e6b8b51ab52f..3e55f91cc68e5 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
@@ -31,6 +31,6 @@
 
 [assembly: Guid("ce65ec7c-d3cf-41ad-8f45-f90d5af68d77")]
 
-[assembly: AssemblyVersion("1.8.0.16695")]
-[assembly: AssemblyFileVersion("1.8.0.16695")]
-[assembly: AssemblyInformationalVersion("1.8.0")]
\ No newline at end of file
+[assembly: AssemblyVersion("1.9.0.17605")]
+[assembly: AssemblyFileVersion("1.9.0.17605")]
+[assembly: AssemblyInformationalVersion("1.9.0")]
\ No newline at end of file
diff --git a/modules/rest-http/pom.xml b/modules/rest-http/pom.xml
index 14c70eb46c913..fffa8024b53ef 100644
--- a/modules/rest-http/pom.xml
+++ b/modules/rest-http/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-rest-http</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/scalar-2.10/pom.xml b/modules/scalar-2.10/pom.xml
index 3920e0a8ad3ec..d1410f4f15864 100644
--- a/modules/scalar-2.10/pom.xml
+++ b/modules/scalar-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar_2.10</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml
index eb8a24cfd0790..1e803adec1fab 100644
--- a/modules/scalar/pom.xml
+++ b/modules/scalar/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/schedule/pom.xml b/modules/schedule/pom.xml
index e4e0005e527ad..9f213fc14624b 100644
--- a/modules/schedule/pom.xml
+++ b/modules/schedule/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-schedule</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/schema-import-db/pom.xml b/modules/schema-import-db/pom.xml
index ba6b80ec86e29..8907da215010f 100644
--- a/modules/schema-import-db/pom.xml
+++ b/modules/schema-import-db/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-schema-import-db</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/schema-import/pom.xml b/modules/schema-import/pom.xml
index c8f341f670d0f..fd7de0eebd17c 100644
--- a/modules/schema-import/pom.xml
+++ b/modules/schema-import/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-schema-import</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/slf4j/pom.xml b/modules/slf4j/pom.xml
index e9733e0c41456..9f7845a4a9b3e 100644
--- a/modules/slf4j/pom.xml
+++ b/modules/slf4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-slf4j</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark-2.10/pom.xml b/modules/spark-2.10/pom.xml
index b39012c458682..7ae6386be74e5 100644
--- a/modules/spark-2.10/pom.xml
+++ b/modules/spark-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark_2.10</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index 25da109ce4a02..e549567f0a2c0 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring/pom.xml b/modules/spring/pom.xml
index a746c93252566..ec72f09d9b617 100644
--- a/modules/spring/pom.xml
+++ b/modules/spring/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ssh/pom.xml b/modules/ssh/pom.xml
index 9b1a6330d6310..e46692b223497 100644
--- a/modules/ssh/pom.xml
+++ b/modules/ssh/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ssh</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/storm/pom.xml b/modules/storm/pom.xml
index 19165be897937..a6d701c90b319 100644
--- a/modules/storm/pom.xml
+++ b/modules/storm/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-storm</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/tools/pom.xml b/modules/tools/pom.xml
index 232a1c2f11965..c96bb07f0e387 100644
--- a/modules/tools/pom.xml
+++ b/modules/tools/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-tools</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/twitter/pom.xml b/modules/twitter/pom.xml
index 4afd8bf3cc43b..331af742ebdce 100644
--- a/modules/twitter/pom.xml
+++ b/modules/twitter/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-twitter</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/urideploy/pom.xml b/modules/urideploy/pom.xml
index 2042d03f2201b..af4b800a969b8 100644
--- a/modules/urideploy/pom.xml
+++ b/modules/urideploy/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-urideploy</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console-2.10/pom.xml b/modules/visor-console-2.10/pom.xml
index 71599bada490c..0d25eeaf16b24 100644
--- a/modules/visor-console-2.10/pom.xml
+++ b/modules/visor-console-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console_2.10</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console/pom.xml b/modules/visor-console/pom.xml
index c8a7c6ebe9668..c4821899c1b52 100644
--- a/modules/visor-console/pom.xml
+++ b/modules/visor-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-plugins/pom.xml b/modules/visor-plugins/pom.xml
index ddbb474017703..3f33ed3bbe898 100644
--- a/modules/visor-plugins/pom.xml
+++ b/modules/visor-plugins/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-plugins</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web-console/pom.xml b/modules/web-console/pom.xml
index 18821b45ccd45..5a6d3ca9338d3 100644
--- a/modules/web-console/pom.xml
+++ b/modules/web-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web-console</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <build>
diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml
index 736136ab97301..7c9d4598e7033 100644
--- a/modules/web-console/web-agent/pom.xml
+++ b/modules/web-console/web-agent/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-web-agent</artifactId>
     <packaging>jar</packaging>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web/ignite-appserver-test/pom.xml b/modules/web/ignite-appserver-test/pom.xml
index 30b6e1583bb81..307e35aadc188 100644
--- a/modules/web/ignite-appserver-test/pom.xml
+++ b/modules/web/ignite-appserver-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-appserver-test</artifactId>
     <packaging>jar</packaging>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/ignite-websphere-test/pom.xml b/modules/web/ignite-websphere-test/pom.xml
index 5e4c4150916d5..b4eac95d0c6d0 100644
--- a/modules/web/ignite-websphere-test/pom.xml
+++ b/modules/web/ignite-websphere-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-websphere-test</artifactId>
     <packaging>war</packaging>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/pom.xml b/modules/web/pom.xml
index 8235b5bfc64cd..c85e7caada97b 100644
--- a/modules/web/pom.xml
+++ b/modules/web/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index cf404a700f1b0..8643156339593 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yardstick</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml
index 322924406b454..6e5fd6dcf49c1 100644
--- a/modules/yarn/pom.xml
+++ b/modules/yarn/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yarn</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index ec9ff7b0d1ec3..2db5cacad89db 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zookeeper</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/pom.xml b/pom.xml
index ac23b3837c02f..d83bb1b969b9e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>apache-ignite</artifactId>
-    <version>1.8.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <properties>

From 296bc413fcc4ea5a541d027788fa458bfa480be7 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Mon, 6 Feb 2017 18:07:28 +0300
Subject: [PATCH 143/446] .NET: Extract exceptions tests in CacheStoreTest and
 ignore due to IGNITE-4657

---
 .../Cache/Store/CacheStoreTest.cs             | 30 +++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs
index d39ccde391a96..2a235aaef6dd1 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs
@@ -296,12 +296,38 @@ public void TestPutLoad()
             Assert.AreEqual("val", cache.Get(1));
 
             Assert.AreEqual(1, cache.GetSize());
+        }
+
+        [Test]
+        public void TestExceptions()
+        {
+            var cache = GetCache();
+
+            cache.Put(1, "val");
 
-            // Test errors
             CacheTestStore.ThrowError = true;
             CheckCustomStoreError(Assert.Throws<CacheStoreException>(() => cache.Put(-2, "fail")).InnerException);
 
-            cache.LocalEvict(new[] { 1 });
+            cache.LocalEvict(new[] {1});
+            CheckCustomStoreError(Assert.Throws<CacheStoreException>(() => cache.Get(1)).InnerException);
+
+            CacheTestStore.ThrowError = false;
+
+            cache.Remove(1);
+        }
+
+        [Test]
+        [Ignore("IGNITE-4657")]
+        public void TestExceptionsNoRemove()
+        {
+            var cache = GetCache();
+
+            cache.Put(1, "val");
+
+            CacheTestStore.ThrowError = true;
+            CheckCustomStoreError(Assert.Throws<CacheStoreException>(() => cache.Put(-2, "fail")).InnerException);
+
+            cache.LocalEvict(new[] {1});
             CheckCustomStoreError(Assert.Throws<CacheStoreException>(() => cache.Get(1)).InnerException);
         }
 

From 03f6822319b9c3d4ca7e76daef5a671097b913d9 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Fri, 20 Jan 2017 12:56:26 +0300
Subject: [PATCH 144/446] IGNITE-4563 .NET: Fix ICache.LoadCache failures on
 non-primitive arguments

---
 .../platform/cache/PlatformCache.java         |  11 +-
 .../Apache.Ignite.Core.Tests.csproj           |   1 +
 .../Cache/Store/CacheParallelLoadStoreTest.cs |   9 +-
 .../Cache/Store/CacheStoreSessionTest.cs      |  22 +-
 .../Cache/Store/CacheStoreTest.cs             | 333 +++++++++++-------
 .../Cache/Store/CacheTestStore.cs             |  14 +
 .../Cache/Store/NamedNodeCacheStoreTest.cs    |  34 ++
 .../Impl/Cache/CacheImpl.cs                   |  14 +-
 8 files changed, 294 insertions(+), 144 deletions(-)
 create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/NamedNodeCacheStoreTest.cs

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
index aec3703b2169b..643907d52f3e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
@@ -815,7 +815,16 @@ private void loadCache0(BinaryRawReaderEx reader, boolean loc, IgniteCache cache
         if (pred != null)
             filter = platformCtx.createCacheEntryFilter(pred, 0);
 
-        Object[] args = reader.readObjectArray();
+        Object[] args = null;
+
+        int argCnt = reader.readInt();
+
+        if (argCnt > 0) {
+            args = new Object[argCnt];
+
+            for (int i = 0; i < argCnt; i++)
+                args[i] = reader.readObjectDetached();
+        }
 
         if (loc)
             cache.localLoadCache(filter, args);
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
index f440c25a41d3c..179d67fede287 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
@@ -72,6 +72,7 @@
     <Compile Include="Cache\CacheResultTest.cs" />
     <Compile Include="Cache\CacheSwapSpaceTest.cs" />
     <Compile Include="Cache\Store\CacheStoreAdapterTest.cs" />
+    <Compile Include="Cache\Store\NamedNodeCacheStoreTest.cs" />
     <Compile Include="Collections\MultiValueDictionaryTest.cs" />
     <Compile Include="Collections\ReadOnlyCollectionTest.cs" />
     <Compile Include="Collections\ReadOnlyDictionaryTest.cs" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheParallelLoadStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheParallelLoadStoreTest.cs
index 105dea23f11cb..2e74b3f308796 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheParallelLoadStoreTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheParallelLoadStoreTest.cs
@@ -25,7 +25,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Store
     /// <summary>
     /// Tests for GridCacheParallelLoadStoreAdapter.
     /// </summary>
-    public class CacheParallelLoadStoreTest
+    public sealed class CacheParallelLoadStoreTest
     {
         // object store name
         private const string ObjectStoreCacheName = "object_store_parallel";
@@ -34,11 +34,8 @@ public class CacheParallelLoadStoreTest
         /// Set up test class.
         /// </summary>
         [TestFixtureSetUp]
-        public virtual void BeforeTests()
+        public void BeforeTests()
         {
-            TestUtils.KillProcesses();
-            TestUtils.JvmDebug = true;
-
             Ignition.Start(new IgniteConfiguration
             {
                 JvmClasspath = TestUtils.CreateTestClasspath(),
@@ -55,7 +52,7 @@ public virtual void BeforeTests()
         /// Tear down test class.
         /// </summary>
         [TestFixtureTearDown]
-        public virtual void AfterTests()
+        public void AfterTests()
         {
             Ignition.StopAll(true);
         }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs
index 5cc0849e88cd6..54e0414c23f47 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs
@@ -22,14 +22,13 @@ namespace Apache.Ignite.Core.Tests.Cache.Store
     using System.Collections.Generic;
     using System.Linq;
     using Apache.Ignite.Core.Cache.Store;
-    using Apache.Ignite.Core.Impl;
     using Apache.Ignite.Core.Resource;
     using NUnit.Framework;
 
     /// <summary>
     /// Tests for store session.
     /// </summary>
-    public class CacheStoreSessionTest
+    public sealed class CacheStoreSessionTest
     {
         /** Grid name. */
         private const string IgniteName = "grid";
@@ -47,7 +46,7 @@ public class CacheStoreSessionTest
         /// Set up routine.
         /// </summary>
         [TestFixtureSetUp]
-        public virtual void BeforeTests()
+        public void BeforeTests()
         {
             //TestUtils.JVM_DEBUG = true;
 
@@ -71,7 +70,7 @@ public virtual void BeforeTests()
         /// Tear down routine.
         /// </summary>
         [TestFixtureTearDown]
-        public virtual void AfterTests()
+        public void AfterTests()
         {
             Ignition.StopAll(true);
         }
@@ -147,7 +146,7 @@ public void TestSession()
         /// Dump operations.
         /// </summary>
         /// <param name="dump">Dump.</param>
-        internal static void DumpOperations(ICollection<Operation> dump)
+        private static void DumpOperations(ICollection<Operation> dump)
         {
             _dumps.Add(dump);
         }
@@ -155,6 +154,7 @@ internal static void DumpOperations(ICollection<Operation> dump)
         /// <summary>
         /// Test store implementation.
         /// </summary>
+        // ReSharper disable once UnusedMember.Global
         public class Store : CacheStoreAdapter
         {
             /** Store session. */
@@ -215,7 +215,7 @@ private ICollection<Operation> GetOperations()
         /// <summary>
         /// Logged operation.
         /// </summary>
-        internal class Operation
+        private class Operation
         {
             /// <summary>
             /// Constructor.
@@ -244,22 +244,22 @@ public Operation(string cacheName, OperationType type, int key, int val) : this(
             /// <summary>
             /// Cache name.
             /// </summary>
-            public string CacheName { get; set; }
+            public string CacheName { get; private set; }
             
             /// <summary>
             /// Operation type.
             /// </summary>
-            public OperationType Type { get; set; }
+            public OperationType Type { get; private set; }
 
             /// <summary>
             /// Key.
             /// </summary>
-            public int Key { get; set; }
+            public int Key { get; private set; }
 
             /// <summary>
             /// Value.
             /// </summary>
-            public int Value { get; set; }
+            public int Value { get; private set; }
 
             /// <summary>
             /// Commit flag.
@@ -270,7 +270,7 @@ public Operation(string cacheName, OperationType type, int key, int val) : this(
         /// <summary>
         /// Operation types.
         /// </summary>
-        internal enum OperationType
+        private enum OperationType
         {
             /** Write. */
             Write,
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs
index 2a235aaef6dd1..a66aea81201ca 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs
@@ -28,92 +28,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Store
     using NUnit.Framework;
 
     /// <summary>
-    ///
-    /// </summary>
-    class Key
-    {
-        private readonly int _idx;
-
-        public Key(int idx)
-        {
-            _idx = idx;
-        }
-
-        public int Index()
-        {
-            return _idx;
-        }
-
-        public override bool Equals(object obj)
-        {
-            if (obj == null || obj.GetType() != GetType())
-                return false;
-
-            Key key = (Key)obj;
-
-            return key._idx == _idx;
-        }
-
-        public override int GetHashCode()
-        {
-            return _idx;
-        }
-    }
-
-    /// <summary>
-    ///
-    /// </summary>
-    class Value
-    {
-        private int _idx;
-
-        public Value(int idx)
-        {
-            _idx = idx;
-        }
-
-        public int Index()
-        {
-            return _idx;
-        }
-    }
-
-    /// <summary>
-    /// Cache entry predicate.
-    /// </summary>
-    [Serializable]
-    public class CacheEntryFilter : ICacheEntryFilter<int, string>
-    {
-        /** <inheritdoc /> */
-        public bool Invoke(ICacheEntry<int, string> entry)
-        {
-            return entry.Key >= 105;
-        }
-    }
-
-    /// <summary>
-    /// Cache entry predicate that throws an exception.
-    /// </summary>
-    [Serializable]
-    public class ExceptionalEntryFilter : ICacheEntryFilter<int, string>
-    {
-        /** <inheritdoc /> */
-        public bool Invoke(ICacheEntry<int, string> entry)
-        {
-            throw new Exception("Expected exception in ExceptionalEntryFilter");
-        }
-    }
-
-    /// <summary>
-    /// Filter that can't be serialized.
-    /// </summary>
-    public class InvalidCacheEntryFilter : CacheEntryFilter
-    {
-        // No-op.
-    }
-
-    /// <summary>
-    ///
+    /// Tests cache store functionality.
     /// </summary>
     public class CacheStoreTest
     {
@@ -129,19 +44,12 @@ public class CacheStoreTest
         /** */
         private const string TemplateStoreCacheName = "template_store*";
 
-        /** */
-        private volatile int _storeCount = 3;
-
         /// <summary>
-        ///
+        /// Fixture set up.
         /// </summary>
         [TestFixtureSetUp]
         public virtual void BeforeTests()
         {
-            TestUtils.KillProcesses();
-
-            TestUtils.JvmDebug = true;
-
             var cfg = new IgniteConfiguration
             {
                 GridName = GridName,
@@ -155,7 +63,7 @@ public virtual void BeforeTests()
         }
 
         /// <summary>
-        ///
+        /// Fixture tear down.
         /// </summary>
         [TestFixtureTearDown]
         public void AfterTests()
@@ -164,16 +72,7 @@ public void AfterTests()
         }
 
         /// <summary>
-        ///
-        /// </summary>
-        [SetUp]
-        public void BeforeTest()
-        {
-            Console.WriteLine("Test started: " + TestContext.CurrentContext.Test.Name);
-        }
-
-        /// <summary>
-        ///
+        /// Test tear down.
         /// </summary>
         [TearDown]
         public void AfterTest()
@@ -188,11 +87,12 @@ public void AfterTest()
                 "Cache is not empty: " +
                 string.Join(", ", cache.Select(x => string.Format("[{0}:{1}]", x.Key, x.Value))));
 
-            TestUtils.AssertHandleRegistryHasItems(300, _storeCount, Ignition.GetIgnite(GridName));
-
-            Console.WriteLine("Test finished: " + TestContext.CurrentContext.Test.Name);
+            TestUtils.AssertHandleRegistryHasItems(300, 3, Ignition.GetIgnite(GridName));
         }
 
+        /// <summary>
+        /// Tests that simple cache loading works and exceptions are propagated properly.
+        /// </summary>
         [Test]
         public void TestLoadCache()
         {
@@ -219,6 +119,9 @@ public void TestLoadCache()
                 cache.LoadCache(new CacheEntryFilter(), 100, 10)).InnerException);
         }
 
+        /// <summary>
+        /// Tests cache loading in local mode.
+        /// </summary>
         [Test]
         public void TestLocalLoadCache()
         {
@@ -234,6 +137,9 @@ public void TestLocalLoadCache()
                 Assert.AreEqual("val_" + i, cache.Get(i));
         }
 
+        /// <summary>
+        /// Tests that object metadata propagates properly during cache loading.
+        /// </summary>
         [Test]
         public void TestLoadCacheMetadata()
         {
@@ -254,6 +160,9 @@ public void TestLoadCacheMetadata()
             Assert.AreEqual("Value", meta.TypeName);
         }
 
+        /// <summary>
+        /// Tests asynchronous cache load.
+        /// </summary>
         [Test]
         public void TestLoadCacheAsync()
         {
@@ -278,6 +187,9 @@ public void TestLoadCacheAsync()
                     .InnerException);
         }
 
+        /// <summary>
+        /// Tests write-through and read-through behavior.
+        /// </summary>
         [Test]
         public void TestPutLoad()
         {
@@ -285,7 +197,7 @@ public void TestPutLoad()
 
             cache.Put(1, "val");
 
-            IDictionary map = StoreMap();
+            IDictionary map = GetStoreMap();
 
             Assert.AreEqual(1, map.Count);
 
@@ -331,6 +243,9 @@ public void TestExceptionsNoRemove()
             CheckCustomStoreError(Assert.Throws<CacheStoreException>(() => cache.Get(1)).InnerException);
         }
 
+        /// <summary>
+        /// Tests write-through and read-through behavior with binarizable values.
+        /// </summary>
         [Test]
         public void TestPutLoadBinarizable()
         {
@@ -338,7 +253,7 @@ public void TestPutLoadBinarizable()
 
             cache.Put(1, new Value(1));
 
-            IDictionary map = StoreMap();
+            IDictionary map = GetStoreMap();
 
             Assert.AreEqual(1, map.Count);
 
@@ -350,11 +265,14 @@ public void TestPutLoadBinarizable()
 
             Assert.AreEqual(0, cache.GetSize());
 
-            Assert.AreEqual(1, cache.Get(1).Index());
+            Assert.AreEqual(1, cache.Get(1).Index);
 
             Assert.AreEqual(1, cache.GetSize());
         }
 
+        /// <summary>
+        /// Tests write-through and read-through behavior with storeKeepBinary=false.
+        /// </summary>
         [Test]
         public void TestPutLoadObjects()
         {
@@ -362,23 +280,26 @@ public void TestPutLoadObjects()
 
             cache.Put(1, new Value(1));
 
-            IDictionary map = StoreMap();
+            IDictionary map = GetStoreMap();
 
             Assert.AreEqual(1, map.Count);
 
             Value v = (Value)map[1];
 
-            Assert.AreEqual(1, v.Index());
+            Assert.AreEqual(1, v.Index);
 
             cache.LocalEvict(new[] { 1 });
 
             Assert.AreEqual(0, cache.GetSize());
 
-            Assert.AreEqual(1, cache.Get(1).Index());
+            Assert.AreEqual(1, cache.Get(1).Index);
 
             Assert.AreEqual(1, cache.GetSize());
         }
 
+        /// <summary>
+        /// Tests cache store LoadAll functionality.
+        /// </summary>
         [Test]
         public void TestPutLoadAll()
         {
@@ -391,7 +312,7 @@ public void TestPutLoadAll()
 
             cache.PutAll(putMap);
 
-            IDictionary map = StoreMap();
+            IDictionary map = GetStoreMap();
 
             Assert.AreEqual(10, map.Count);
 
@@ -417,6 +338,9 @@ public void TestPutLoadAll()
             Assert.AreEqual(10, cache.GetSize());
         }
 
+        /// <summary>
+        /// Tests cache store removal.
+        /// </summary>
         [Test]
         public void TestRemove()
         {
@@ -425,7 +349,7 @@ public void TestRemove()
             for (int i = 0; i < 10; i++)
                 cache.Put(i, "val_" + i);
 
-            IDictionary map = StoreMap();
+            IDictionary map = GetStoreMap();
 
             Assert.AreEqual(10, map.Count);
 
@@ -438,6 +362,9 @@ public void TestRemove()
                 Assert.AreEqual("val_" + i, map[i]);
         }
 
+        /// <summary>
+        /// Tests cache store removal.
+        /// </summary>
         [Test]
         public void TestRemoveAll()
         {
@@ -446,7 +373,7 @@ public void TestRemoveAll()
             for (int i = 0; i < 10; i++)
                 cache.Put(i, "val_" + i);
 
-            IDictionary map = StoreMap();
+            IDictionary map = GetStoreMap();
 
             Assert.AreEqual(10, map.Count);
 
@@ -458,6 +385,9 @@ public void TestRemoveAll()
                 Assert.AreEqual("val_" + i, map[i]);
         }
 
+        /// <summary>
+        /// Tests cache store with transactions.
+        /// </summary>
         [Test]
         public void TestTx()
         {
@@ -472,13 +402,16 @@ public void TestTx()
                 tx.Commit();
             }
 
-            IDictionary map = StoreMap();
+            IDictionary map = GetStoreMap();
 
             Assert.AreEqual(1, map.Count);
 
             Assert.AreEqual("val", map[1]);
         }
 
+        /// <summary>
+        /// Tests multithreaded cache loading.
+        /// </summary>
         [Test]
         public void TestLoadCacheMultithreaded()
         {
@@ -496,6 +429,9 @@ public void TestLoadCacheMultithreaded()
                 Assert.AreEqual("val_" + i, cache.Get(i));
         }
 
+        /// <summary>
+        /// Tests that cache store property values are propagated from Spring XML.
+        /// </summary>
         [Test]
         public void TestCustomStoreProperties()
         {
@@ -506,6 +442,9 @@ public void TestCustomStoreProperties()
             Assert.AreEqual("String value", CacheTestStore.stringProperty);
         }
 
+        /// <summary>
+        /// Tests cache store with dynamically started cache.
+        /// </summary>
         [Test]
         public void TestDynamicStoreStart()
         {
@@ -524,6 +463,9 @@ public void TestDynamicStoreStart()
             Assert.AreEqual(handleCount, reg.Count);
         }
 
+        /// <summary>
+        /// Tests the load all.
+        /// </summary>
         [Test]
         public void TestLoadAll([Values(true, false)] bool isAsync)
         {
@@ -554,6 +496,49 @@ public void TestLoadAll([Values(true, false)] bool isAsync)
             Assert.AreEqual("val_106", cache[106]);
         }
 
+        /// <summary>
+        /// Tests the argument passing to LoadCache method.
+        /// </summary>
+        [Test]
+        public void TestArgumentPassing()
+        {
+            var cache = GetBinaryStoreCache<object, object>();
+
+            Action<object> checkValue = o =>
+            {
+                cache.Clear();
+                Assert.AreEqual(0, cache.GetSize());
+                cache.LoadCache(null, null, 1, o);
+                Assert.AreEqual(o, cache[1]);
+            };
+
+            // Null.
+            cache.LoadCache(null, null);
+            Assert.AreEqual(0, cache.GetSize());
+
+            // Empty args array.
+            cache.LoadCache(null);
+            Assert.AreEqual(0, cache.GetSize());
+
+            // Simple types.
+            checkValue(1);
+            checkValue(new[] {1, 2, 3});
+
+            checkValue("1");
+            checkValue(new[] {"1", "2"});
+
+            checkValue(Guid.NewGuid());
+            checkValue(new[] {Guid.NewGuid(), Guid.NewGuid()});
+
+            checkValue(DateTime.Now);
+            checkValue(new[] {DateTime.Now, DateTime.UtcNow});
+
+            // Collections.
+            checkValue(new ArrayList {1, "2", 3.3});
+            checkValue(new List<int> {1, 2});
+            checkValue(new Dictionary<int, string> {{1, "foo"}});
+        }
+
         /// <summary>
         /// Get's grid name for this test.
         /// </summary>
@@ -563,31 +548,49 @@ protected virtual string GridName
             get { return null; }
         }
 
-        private IDictionary StoreMap()
+        /// <summary>
+        /// Gets the store map.
+        /// </summary>
+        private static IDictionary GetStoreMap()
         {
             return CacheTestStore.Map;
         }
 
+        /// <summary>
+        /// Gets the cache.
+        /// </summary>
         private ICache<int, string> GetCache()
         {
             return GetBinaryStoreCache<int, string>();
         }
 
+        /// <summary>
+        /// Gets the binary store cache.
+        /// </summary>
         private ICache<TK, TV> GetBinaryStoreCache<TK, TV>()
         {
             return Ignition.GetIgnite(GridName).GetCache<TK, TV>(BinaryStoreCacheName);
         }
 
+        /// <summary>
+        /// Gets the object store cache.
+        /// </summary>
         private ICache<TK, TV> GetObjectStoreCache<TK, TV>()
         {
             return Ignition.GetIgnite(GridName).GetCache<TK, TV>(ObjectStoreCacheName);
         }
 
+        /// <summary>
+        /// Gets the custom store cache.
+        /// </summary>
         private ICache<int, string> GetCustomStoreCache()
         {
             return Ignition.GetIgnite(GridName).GetCache<int, string>(CustomStoreCacheName);
         }
 
+        /// <summary>
+        /// Gets the template store cache.
+        /// </summary>
         private ICache<int, string> GetTemplateStoreCache()
         {
             var cacheName = TemplateStoreCacheName.Replace("*", Guid.NewGuid().ToString());
@@ -595,6 +598,9 @@ private IDictionary StoreMap()
             return Ignition.GetIgnite(GridName).GetOrCreateCache<int, string>(cacheName);
         }
 
+        /// <summary>
+        /// Checks the custom store error.
+        /// </summary>
         private static void CheckCustomStoreError(Exception err)
         {
             var customErr = err as CacheTestStore.CustomStoreException ??
@@ -607,14 +613,93 @@ private static void CheckCustomStoreError(Exception err)
     }
 
     /// <summary>
-    /// 
+    /// Cache key.
     /// </summary>
-    public class NamedNodeCacheStoreTest : CacheStoreTest
+    internal class Key
     {
-        /** <inheritDoc /> */
-        protected override string GridName
+        /** */
+        private readonly int _idx;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Key"/> class.
+        /// </summary>
+        public Key(int idx)
+        {
+            _idx = idx;
+        }
+
+        /** <inheritdoc /> */
+        public override bool Equals(object obj)
         {
-            get { return "name"; }
+            if (obj == null || obj.GetType() != GetType())
+                return false;
+
+            return ((Key)obj)._idx == _idx;
         }
+
+        /** <inheritdoc /> */
+        public override int GetHashCode()
+        {
+            return _idx;
+        }
+    }
+
+    /// <summary>
+    /// Cache value.
+    /// </summary>
+    internal class Value
+    {
+        /** */
+        private readonly int _idx;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Value"/> class.
+        /// </summary>
+        public Value(int idx)
+        {
+            _idx = idx;
+        }
+
+        /// <summary>
+        /// Gets the index.
+        /// </summary>
+        public int Index
+        {
+            get { return _idx; }
+        }
+    }
+
+    /// <summary>
+    /// Cache entry predicate.
+    /// </summary>
+    [Serializable]
+    public class CacheEntryFilter : ICacheEntryFilter<int, string>
+    {
+        /** <inheritdoc /> */
+        public bool Invoke(ICacheEntry<int, string> entry)
+        {
+            return entry.Key >= 105;
+        }
+    }
+
+    /// <summary>
+    /// Cache entry predicate that throws an exception.
+    /// </summary>
+    [Serializable]
+    public class ExceptionalEntryFilter : ICacheEntryFilter<int, string>
+    {
+        /** <inheritdoc /> */
+        public bool Invoke(ICacheEntry<int, string> entry)
+        {
+            throw new Exception("Expected exception in ExceptionalEntryFilter");
+        }
+    }
+
+    /// <summary>
+    /// Filter that can't be serialized.
+    /// </summary>
+    public class InvalidCacheEntryFilter : CacheEntryFilter
+    {
+        // No-op.
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheTestStore.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheTestStore.cs
index 4224835b96335..f80f5ce7f1dbc 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheTestStore.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheTestStore.cs
@@ -66,6 +66,20 @@ public void LoadCache(Action<object, object> act, params object[] args)
 
             Debug.Assert(_grid != null);
 
+            if (args == null || args.Length == 0)
+                return;
+
+            if (args.Length == 3 && args[0] == null)
+            {
+                // Testing arguments passing.
+                var key = args[1];
+                var val = args[2];
+
+                act(key, val);
+
+                return;
+            }
+
             if (LoadMultithreaded)
             {
                 int cnt = 0;
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/NamedNodeCacheStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/NamedNodeCacheStoreTest.cs
new file mode 100644
index 0000000000000..02e257fe3eed0
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/NamedNodeCacheStoreTest.cs
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Core.Tests.Cache.Store
+{
+    using NUnit.Framework;
+
+    /// <summary>
+    /// Cache store test with named node.
+    /// </summary>
+    [TestFixture]
+    public class NamedNodeCacheStoreTest : CacheStoreTest
+    {
+        /** <inheritDoc /> */
+        protected override string GridName
+        {
+            get { return "name"; }
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
index 186737c47906d..b8dc6cb71d290 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
@@ -225,7 +225,7 @@ public Task LocalLoadCacheAsync(ICacheEntryFilter<TK, TV> p, params object[] arg
         /// <summary>
         /// Writes the load cache data to the writer.
         /// </summary>
-        private void WriteLoadCacheData(IBinaryRawWriter writer, ICacheEntryFilter<TK, TV> p, object[] args)
+        private void WriteLoadCacheData(BinaryWriter writer, ICacheEntryFilter<TK, TV> p, object[] args)
         {
             if (p != null)
             {
@@ -237,7 +237,17 @@ private void WriteLoadCacheData(IBinaryRawWriter writer, ICacheEntryFilter<TK, T
             else
                 writer.WriteObject<CacheEntryFilterHolder>(null);
 
-            writer.WriteArray(args);
+            if (args != null && args.Length > 0)
+            {
+                writer.WriteInt(args.Length);
+
+                foreach (var o in args)
+                    writer.WriteObject(o);
+            }
+            else
+            {
+                writer.WriteInt(0);
+            }
         }
 
         /** <inheritDoc /> */

From 4bf7fdf40e50aa6aca8fb438fb314c35166802df Mon Sep 17 00:00:00 2001
From: nikolay tikhonov <ntikhonov@gridgain.com>
Date: Tue, 7 Feb 2017 16:18:56 +0300
Subject: [PATCH 145/446] IGNITE-4590 Fixed Lock/unlock operations are hanging
 when topology changed

---
 .../transactions/IgniteTxLocalAdapter.java    |   3 +
 .../cache/GridCacheTestEntryEx.java           | 102 +++++++++---------
 .../CacheLockReleaseNodeLeaveTest.java        |  65 ++++++++++-
 3 files changed, 118 insertions(+), 52 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 777489e026edb..1a9b082b3793f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -3785,6 +3785,9 @@ protected void updateExplicitVersion(IgniteTxEntry txEntry, GridCacheEntryEx ent
             // so it is safe to get acquired locks.
             GridCacheMvccCandidate explicitCand = entry.localOwner();
 
+            if (explicitCand == null)
+                explicitCand = cctx.mvcc().explicitLock(threadId(), entry.txKey());
+
             if (explicitCand != null) {
                 GridCacheVersion explicitVer = explicitCand.version();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index d46dee03b3a27..8f0d9b13d127d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -280,7 +280,7 @@ void recheckLock() {
         return true;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public KeyCacheObject key() {
         return key;
     }
@@ -290,7 +290,7 @@ void recheckLock() {
         return new IgniteTxKey(key, 0);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public CacheObject rawGet() {
         return val;
     }
@@ -305,7 +305,7 @@ void recheckLock() {
         return val != null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public CacheObject rawPut(CacheObject val, long ttl) {
         CacheObject old = this.val;
 
@@ -315,14 +315,14 @@ void recheckLock() {
         return old;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public Cache.Entry wrap() {
         assert false;
 
         return null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public Cache.Entry wrapLazyValue(boolean keepBinary) {
         assert false;
 
@@ -336,19 +336,19 @@ void recheckLock() {
         return null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Nullable @Override public CacheObject peekVisibleValue() {
         assert false;
 
         return null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public GridCacheVersion obsoleteVersion() {
         return obsoleteVer;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean obsolete() {
         return obsoleteVer != null;
     }
@@ -358,7 +358,7 @@ void recheckLock() {
         return obsoleteVer != null && !obsoleteVer.equals(exclude);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean invalidate(@Nullable GridCacheVersion curVer, GridCacheVersion newVer)
         throws IgniteCheckedException {
         assert false;
@@ -366,7 +366,7 @@ void recheckLock() {
         return false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean invalidate(@Nullable CacheEntryPredicate[] filter)
         throws GridCacheEntryRemovedException, IgniteCheckedException {
         assert false;
@@ -374,7 +374,7 @@ void recheckLock() {
         return false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean evictInternal(boolean swap, GridCacheVersion obsoleteVer,
         @Nullable CacheEntryPredicate[] filter) {
         assert false;
@@ -390,7 +390,7 @@ void recheckLock() {
         return null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean isNew() {
         assert false; return false;
     }
@@ -400,7 +400,7 @@ void recheckLock() {
         assert false; return false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public CacheObject innerGet(
         @Nullable GridCacheVersion ver,
         @Nullable IgniteInternalTx tx,
@@ -417,12 +417,12 @@ void recheckLock() {
         return val;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public void clearReserveForLoad(GridCacheVersion ver) {
         assert false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public EntryGetResult innerGetAndReserveForLoad(
         boolean readSwap,
         boolean updateMetrics,
@@ -437,7 +437,7 @@ void recheckLock() {
         return null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Nullable @Override public EntryGetResult innerGetVersioned(
         @Nullable GridCacheVersion ver,
         IgniteInternalTx tx,
@@ -456,12 +456,12 @@ void recheckLock() {
         return null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public CacheObject innerReload() {
         return val;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public GridCacheUpdateTxResult innerSet(@Nullable IgniteInternalTx tx,
         UUID evtNodeId,
         UUID affNodeId,
@@ -544,7 +544,7 @@ void recheckLock() {
         return null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public GridCacheUpdateTxResult innerRemove(
         @Nullable IgniteInternalTx tx,
         UUID evtNodeId,
@@ -573,7 +573,7 @@ void recheckLock() {
         return new GridCacheUpdateTxResult(true, old);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean clear(GridCacheVersion ver, boolean readers) throws IgniteCheckedException {
         if (ver == null || ver.equals(this.ver)) {
             val = null;
@@ -584,7 +584,7 @@ void recheckLock() {
         return false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean tmLock(IgniteInternalTx tx,
         long timeout,
         @Nullable GridCacheVersion serOrder,
@@ -594,12 +594,12 @@ void recheckLock() {
         return false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public void txUnlock(IgniteInternalTx tx) {
         assert false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean removeLock(GridCacheVersion ver) {
         GridCacheMvccCandidate doomed = mvcc.candidate(ver);
 
@@ -608,7 +608,7 @@ void recheckLock() {
         return doomed != null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean markObsolete(GridCacheVersion ver) {
         if (ver == null || ver.equals(obsoleteVer)) {
             obsoleteVer = ver;
@@ -645,19 +645,19 @@ void recheckLock() {
         return false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public GridCacheVersion version() {
         return ver;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean checkSerializableReadVersion(GridCacheVersion ver) {
         assert false;
 
         return false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean initialValue(
         CacheObject val,
         GridCacheVersion ver,
@@ -673,19 +673,19 @@ void recheckLock() {
         return false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean initialValue(KeyCacheObject key, GridCacheSwapEntry unswapped) {
         assert false;
 
         return false;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public GridCacheVersionedEntryEx versionedEntry(final boolean keepBinary) throws IgniteCheckedException {
         return null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public EntryGetResult versionedValue(CacheObject val,
         GridCacheVersion curVer,
         GridCacheVersion newVer,
@@ -696,22 +696,22 @@ void recheckLock() {
         return null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean hasLockCandidate(GridCacheVersion ver) {
         return mvcc.hasCandidate(ver);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean lockedByAny(GridCacheVersion... exclude) {
         return !mvcc.isEmpty(exclude);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean lockedByThread()  {
         return lockedByThread(Thread.currentThread().getId());
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean lockedLocally(GridCacheVersion lockVer) {
         return mvcc.isLocallyOwned(lockVer);
     }
@@ -722,52 +722,52 @@ void recheckLock() {
         return lockedLocally(lockVer) || lockedByThread(threadId);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean lockedByThread(long threadId, GridCacheVersion exclude) {
         return mvcc.isLocallyOwnedByThread(threadId, false, exclude);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean lockedByThread(long threadId) {
         return mvcc.isLocallyOwnedByThread(threadId, true);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean lockedBy(GridCacheVersion ver) {
         return mvcc.isOwnedBy(ver);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean lockedByThreadUnsafe(long threadId) {
         return mvcc.isLocallyOwnedByThread(threadId, true);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean lockedByUnsafe(GridCacheVersion ver) {
         return mvcc.isOwnedBy(ver);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean lockedLocallyUnsafe(GridCacheVersion lockVer) {
         return mvcc.isLocallyOwned(lockVer);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public boolean hasLockCandidateUnsafe(GridCacheVersion ver) {
         return mvcc.hasCandidate(ver);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public Collection<GridCacheMvccCandidate> localCandidates(GridCacheVersion... exclude) {
         return mvcc.localCandidates(exclude);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     Collection<GridCacheMvccCandidate> localCandidates(boolean reentries, GridCacheVersion... exclude) {
         return mvcc.localCandidates(reentries, exclude);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public Collection<GridCacheMvccCandidate> remoteMvccSnapshot(GridCacheVersion... exclude) {
         return mvcc.remoteCandidates(exclude);
     }
@@ -777,7 +777,7 @@ Collection<GridCacheMvccCandidate> localCandidates(boolean reentries, GridCacheV
         return mvcc.localCandidate(threadId);
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public GridCacheMvccCandidate candidate(GridCacheVersion ver) {
         return mvcc.candidate(ver);
     }
@@ -795,19 +795,19 @@ GridCacheMvccCandidate anyOwner() {
         return mvcc.anyOwner();
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public GridCacheMvccCandidate localOwner() {
         return mvcc.localOwner();
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public CacheObject valueBytes() {
         assert false;
 
         return null;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public CacheObject valueBytes(GridCacheVersion ver) {
         assert false;
 
@@ -819,7 +819,7 @@ GridCacheMvccCandidate anyOwner() {
         return 0;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public long expireTime() {
         return 0;
     }
@@ -839,12 +839,12 @@ GridCacheMvccCandidate anyOwner() {
         return ttl;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public long ttl() {
         return ttl;
     }
 
-    /** @inheritDoc */
+    /** {@inheritDoc} */
     @Override public void updateTtl(GridCacheVersion ver, long ttl) {
         throw new UnsupportedOperationException();
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java
index e84fd3fa125d0..687fe0bb6fa1a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java
@@ -17,9 +17,13 @@
 
 package org.apache.ignite.internal.processors.cache.distributed;
 
+import java.util.ArrayDeque;
+import java.util.Queue;
 import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.Lock;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -33,6 +37,7 @@
 
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
 import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
@@ -43,6 +48,9 @@ public class CacheLockReleaseNodeLeaveTest extends GridCommonAbstractTest {
     /** */
     private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
 
+    /** */
+    private static final String REPLICATED_TEST_CACHE = "REPLICATED_TEST_CACHE";
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -53,7 +61,12 @@ public class CacheLockReleaseNodeLeaveTest extends GridCommonAbstractTest {
 
         ccfg.setAtomicityMode(TRANSACTIONAL);
 
-        cfg.setCacheConfiguration(ccfg);
+        CacheConfiguration ccfg1 = new CacheConfiguration(REPLICATED_TEST_CACHE)
+            .setCacheMode(REPLICATED)
+            .setAtomicityMode(TRANSACTIONAL)
+            .setReadFromBackup(false);
+
+        cfg.setCacheConfiguration(ccfg, ccfg1);
 
         return cfg;
     }
@@ -111,6 +124,56 @@ public void testLockRelease() throws Exception {
         fut2.get(5, SECONDS);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLockTopologyChange() throws Exception {
+        final int nodeCnt = 5;
+        int threadCnt = 8;
+        final int keys = 100;
+
+        try {
+            final AtomicBoolean stop = new AtomicBoolean(false);
+
+            Queue<IgniteInternalFuture<Long>> q = new ArrayDeque<>(nodeCnt);
+
+            for (int i = 0; i < nodeCnt; i++) {
+                final Ignite ignite = startGrid(i);
+
+                IgniteInternalFuture<Long> f = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+                    @Override public void run() {
+                        while (!Thread.currentThread().isInterrupted() && !stop.get()) {
+                            IgniteCache<Integer, Integer> cache = ignite.cache(REPLICATED_TEST_CACHE);
+
+                            for (int i = 0; i < keys; i++) {
+                                Lock lock = cache.lock(i);
+                                lock.lock();
+
+                                cache.put(i, i);
+
+                                lock.unlock();
+                            }
+                        }
+                    }
+                }, threadCnt, "test-lock-thread");
+
+                q.add(f);
+
+                U.sleep(1_000);
+            }
+
+            stop.set(true);
+
+            IgniteInternalFuture<Long> f;
+
+            while ((f = q.poll()) != null)
+                f.get(2_000);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From 94d8a6fc0a61bb3046e2ebd8b3cbffb8917c2b6a Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 8 Feb 2017 11:43:22 +0700
Subject: [PATCH 146/446] IGNITE-4472 Added user activities in Web Console.

(cherry picked from commit 26ee9c2)
---
 modules/web-console/backend/app/agent.js      |  10 +-
 modules/web-console/backend/app/mongo.js      |  49 ++--
 modules/web-console/backend/app/routes.js     |   5 +-
 .../web-console/backend/routes/activities.js  |  52 ++++
 modules/web-console/backend/routes/admin.js   |   2 +-
 modules/web-console/backend/routes/agent.js   |  10 +-
 modules/web-console/backend/routes/public.js  |   1 -
 .../backend/services/activities.js            | 136 ++++++++++
 modules/web-console/backend/services/users.js |  13 +-
 .../web-console/frontend/app/app.config.js    |   9 +
 modules/web-console/frontend/app/app.js       |  29 ++-
 .../activities-user-dialog.controller.js      |  60 +++++
 .../activities-user-dialog.jade               |  36 +++
 .../activities-user-dialog/index.js           |  36 +++
 .../form-field-datepicker.jade                |  55 ++++
 .../form-field-datepicker.scss                |  20 ++
 .../list-of-registered-users/index.js         |  28 +++
 .../list-of-registered-users.categories.js    |  30 +++
 .../list-of-registered-users.column-defs.js   |  80 ++++++
 .../list-of-registered-users.controller.js    | 207 ++++++++++++++++
 .../list-of-registered-users.jade             |  54 ++++
 .../ui-grid-header/ui-grid-header.jade        |  27 ++
 .../ui-grid-header/ui-grid-header.scss        |  84 +++++++
 .../ui-grid-settings/ui-grid-settings.jade    |   0
 .../ui-grid-settings/ui-grid-settings.scss    |  32 +++
 .../app/core/activities/Activities.data.js    |  39 +++
 .../frontend/app/core/admin/Admin.data.js     |  77 ++++++
 .../web-console/frontend/app/core/index.js    |  25 ++
 modules/web-console/frontend/app/data/i18n.js |  38 +++
 .../app/filters/uiGridSubcategories.filter.js |  24 ++
 .../app/modules/sql/sql.controller.js         |  14 +-
 .../frontend/app/modules/sql/sql.module.js    |   2 +-
 .../app/modules/states/admin.state.js         |   2 +-
 .../summary/summary.controller.js             |   6 +-
 .../app/modules/user/AclRoute.provider.js     |  31 ++-
 .../frontend/app/modules/user/Auth.service.js |   2 +-
 .../frontend/app/modules/user/permissions.js  |   2 +-
 .../frontend/app/modules/user/user.module.js  |   6 +-
 modules/web-console/frontend/app/vendor.js    |   1 +
 .../frontend/controllers/admin-controller.js  | 234 ------------------
 .../controllers/domains-controller.js         |  12 +-
 modules/web-console/frontend/package.json     |   1 +
 .../stylesheets/_font-awesome-custom.scss     |  28 +++
 .../frontend/public/stylesheets/style.scss    |  39 ++-
 .../public/stylesheets/variables.scss         |   1 +
 .../frontend/views/settings/admin.jade        |  32 +--
 .../web-console/frontend/views/sql/sql.jade   |   4 +-
 47 files changed, 1360 insertions(+), 325 deletions(-)
 create mode 100644 modules/web-console/backend/routes/activities.js
 create mode 100644 modules/web-console/backend/services/activities.js
 create mode 100644 modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
 create mode 100644 modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
 create mode 100644 modules/web-console/frontend/app/components/activities-user-dialog/index.js
 create mode 100644 modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
 create mode 100644 modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.scss
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/index.js
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.categories.js
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
 create mode 100644 modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
 create mode 100644 modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
 rename modules/web-console/frontend/app/{directives => components}/ui-grid-settings/ui-grid-settings.jade (100%)
 rename modules/web-console/frontend/app/{directives => components}/ui-grid-settings/ui-grid-settings.scss (68%)
 create mode 100644 modules/web-console/frontend/app/core/activities/Activities.data.js
 create mode 100644 modules/web-console/frontend/app/core/admin/Admin.data.js
 create mode 100644 modules/web-console/frontend/app/core/index.js
 create mode 100644 modules/web-console/frontend/app/data/i18n.js
 create mode 100644 modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js
 delete mode 100644 modules/web-console/frontend/controllers/admin-controller.js

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index 961253f148a70..817028059f50f 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -24,7 +24,7 @@
  */
 module.exports = {
     implements: 'agent-manager',
-    inject: ['require(lodash)', 'require(fs)', 'require(path)', 'require(jszip)', 'require(socket.io)', 'settings', 'mongo']
+    inject: ['require(lodash)', 'require(fs)', 'require(path)', 'require(jszip)', 'require(socket.io)', 'settings', 'mongo', 'services/activities']
 };
 
 /**
@@ -35,9 +35,10 @@ module.exports = {
  * @param socketio
  * @param settings
  * @param mongo
+ * @param {ActivitiesService} activitiesService
  * @returns {AgentManager}
  */
-module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo) {
+module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo, activitiesService) {
     /**
      *
      */
@@ -823,6 +824,11 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo)
                 const sockets = this._browsers[accountId];
 
                 _.forEach(sockets, (socket) => socket.emit('agent:count', {count: agents.length}));
+
+                activitiesService.merge(accountId, {
+                    group: 'agent',
+                    action: '/agent/start'
+                });
             });
         }
 
diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index dd71f3a8bf57e..2d252b9b8eccc 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -48,6 +48,7 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
         company: String,
         country: String,
         lastLogin: Date,
+        lastActivity: Date,
         admin: Boolean,
         token: String,
         resetPasswordToken: String
@@ -59,22 +60,26 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
         usernameLowerCase: true
     });
 
+    const transform = (doc, ret) => {
+        return {
+            _id: ret._id,
+            email: ret.email,
+            firstName: ret.firstName,
+            lastName: ret.lastName,
+            company: ret.company,
+            country: ret.country,
+            admin: ret.admin,
+            token: ret.token,
+            lastLogin: ret.lastLogin,
+            lastActivity: ret.lastActivity
+        };
+    };
+
     // Configure transformation to JSON.
-    AccountSchema.set('toJSON', {
-        transform: (doc, ret) => {
-            return {
-                _id: ret._id,
-                email: ret.email,
-                firstName: ret.firstName,
-                lastName: ret.lastName,
-                company: ret.company,
-                country: ret.country,
-                admin: ret.admin,
-                token: ret.token,
-                lastLogin: ret.lastLogin
-            };
-        }
-    });
+    AccountSchema.set('toJSON', { transform });
+
+    // Configure transformation to JSON.
+    AccountSchema.set('toObject', { transform });
 
     result.errCodes = {
         DUPLICATE_KEY_ERROR: 11000,
@@ -902,6 +907,20 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
         res.status(err.code || 500).send(err.message);
     };
 
+    // Define Activities schema.
+    const ActivitiesSchema = new Schema({
+        owner: {type: ObjectId, ref: 'Account'},
+        date: Date,
+        group: String,
+        action: String,
+        amount: { type: Number, default: 1 }
+    });
+
+    ActivitiesSchema.index({ owner: 1, group: 1, action: 1, date: 1}, { unique: true });
+
+    // Define Activities model.
+    result.Activities = mongoose.model('Activities', ActivitiesSchema);
+
     // Registering the routes of all plugin modules
     for (const name in pluginMongo) {
         if (pluginMongo.hasOwnProperty(name))
diff --git a/modules/web-console/backend/app/routes.js b/modules/web-console/backend/app/routes.js
index 6961173d03eb9..6b5d052d4d765 100644
--- a/modules/web-console/backend/app/routes.js
+++ b/modules/web-console/backend/app/routes.js
@@ -22,11 +22,11 @@
 module.exports = {
     implements: 'routes',
     inject: ['routes/public', 'routes/admin', 'routes/profiles', 'routes/demo', 'routes/clusters', 'routes/domains',
-        'routes/caches', 'routes/igfss', 'routes/notebooks', 'routes/agents', 'routes/configurations']
+        'routes/caches', 'routes/igfss', 'routes/notebooks', 'routes/agents', 'routes/configurations', 'routes/activities']
 };
 
 module.exports.factory = function(publicRoute, adminRoute, profilesRoute, demoRoute,
-    clustersRoute, domainsRoute, cachesRoute, igfssRoute, notebooksRoute, agentsRoute, configurationsRoute) {
+    clustersRoute, domainsRoute, cachesRoute, igfssRoute, notebooksRoute, agentsRoute, configurationsRoute, activitiesRoute) {
     return {
         register: (app) => {
             const _mustAuthenticated = (req, res, next) => {
@@ -59,6 +59,7 @@ module.exports.factory = function(publicRoute, adminRoute, profilesRoute, demoRo
 
             app.use('/notebooks', _mustAuthenticated, notebooksRoute);
             app.use('/agent', _mustAuthenticated, agentsRoute);
+            app.use('/activities', _mustAuthenticated, activitiesRoute);
         }
     };
 };
diff --git a/modules/web-console/backend/routes/activities.js b/modules/web-console/backend/routes/activities.js
new file mode 100644
index 0000000000000..08c27cf1cdf8c
--- /dev/null
+++ b/modules/web-console/backend/routes/activities.js
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+// Fire me up!
+
+module.exports = {
+    implements: 'routes/activities',
+    inject: ['require(express)', 'services/activities']
+};
+
+/**
+ * @param express
+ * @param {ActivitiesService} activitiesService
+ * @returns {Promise}
+ */
+module.exports.factory = function(express, activitiesService) {
+    return new Promise((factoryResolve) => {
+        const router = new express.Router();
+
+        // Get user activities.
+        router.get('/user/:userId', (req, res) => {
+            activitiesService.listByUser(req.params.userId, req.query)
+                .then(res.api.ok)
+                .catch(res.api.error);
+        });
+
+        // Post user activities to page.
+        router.post('/page', (req, res) => {
+            activitiesService.merge(req.user._id, req.body)
+                .then(res.api.ok)
+                .catch(res.api.error);
+        });
+
+        factoryResolve(router);
+    });
+};
diff --git a/modules/web-console/backend/routes/admin.js b/modules/web-console/backend/routes/admin.js
index 70736d00ea43a..5b0896ac69437 100644
--- a/modules/web-console/backend/routes/admin.js
+++ b/modules/web-console/backend/routes/admin.js
@@ -43,7 +43,7 @@ module.exports.factory = function(_, express, settings, mongo, spacesService, ma
          * Get list of user accounts.
          */
         router.post('/list', (req, res) => {
-            usersService.list()
+            usersService.list(req.body)
                 .then(res.api.ok)
                 .catch(res.api.error);
         });
diff --git a/modules/web-console/backend/routes/agent.js b/modules/web-console/backend/routes/agent.js
index 477363f9f5266..5ae807b5e88c1 100644
--- a/modules/web-console/backend/routes/agent.js
+++ b/modules/web-console/backend/routes/agent.js
@@ -21,21 +21,27 @@
 
 module.exports = {
     implements: 'routes/agents',
-    inject: ['require(lodash)', 'require(express)', 'services/agents']
+    inject: ['require(lodash)', 'require(express)', 'services/agents', 'services/activities']
 };
 
 /**
  * @param _
  * @param express
  * @param {AgentsService} agentsService
+ * @param {ActivitiesService} activitiesService
  * @returns {Promise}
  */
-module.exports.factory = function(_, express, agentsService) {
+module.exports.factory = function(_, express, agentsService, activitiesService) {
     return new Promise((resolveFactory) => {
         const router = new express.Router();
 
         /* Get grid topology. */
         router.get('/download/zip', (req, res) => {
+            activitiesService.merge(req.user._id, {
+                group: 'agent',
+                action: '/agent/download'
+            });
+
             agentsService.getArchive(req.origin(), req.user.token)
                 .then(({fileName, buffer}) => {
                     // Set the archive name.
diff --git a/modules/web-console/backend/routes/public.js b/modules/web-console/backend/routes/public.js
index 590d395179abd..860e2678c2fff 100644
--- a/modules/web-console/backend/routes/public.js
+++ b/modules/web-console/backend/routes/public.js
@@ -25,7 +25,6 @@ module.exports = {
 };
 
 /**
- *
  * @param express
  * @param passport
  * @param mongo
diff --git a/modules/web-console/backend/services/activities.js b/modules/web-console/backend/services/activities.js
new file mode 100644
index 0000000000000..7f3a7775f83d2
--- /dev/null
+++ b/modules/web-console/backend/services/activities.js
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+// Fire me up!
+
+module.exports = {
+    implements: 'services/activities',
+    inject: ['require(lodash)', 'mongo']
+};
+
+/**
+ * @param _
+ * @param mongo
+ * @returns {ActivitiesService}
+ */
+module.exports.factory = (_, mongo) => {
+    class ActivitiesService {
+        /**
+         * Update page activities.
+         *
+         * @param {String} owner - User ID
+         * @param {Object} page - The page
+         * @returns {Promise.<mongo.ObjectId>} that resolve activity
+         */
+        static merge(owner, {action, group}) {
+            mongo.Account.findById(owner)
+                .then((user) => {
+                    user.lastActivity = new Date();
+
+                    return user.save();
+                });
+
+            const date = new Date();
+
+            date.setDate(1);
+            date.setHours(0, 0, 0, 0);
+
+            return mongo.Activities.findOne({owner, action, date}).exec()
+                .then((activity) => {
+                    if (activity) {
+                        activity.amount++;
+
+                        return activity.save();
+                    }
+
+                    return mongo.Activities.create({owner, action, group, date});
+                });
+        }
+
+        /**
+         * Get user activities
+         * @param {String} owner - User ID
+         * @returns {Promise.<mongo.ObjectId>} that resolve activities
+         */
+        static listByUser(owner, {startDate, endDate}) {
+            const $match = {owner};
+
+            if (startDate)
+                $match.date = {$gte: new Date(startDate)};
+
+            if (endDate) {
+                $match.date = $match.date || {};
+                $match.date.$lt = new Date(endDate);
+            }
+
+            return mongo.Activities.find($match);
+        }
+
+        static total({startDate, endDate}) {
+            const $match = {};
+
+            if (startDate)
+                $match.date = {$gte: new Date(startDate)};
+
+            if (endDate) {
+                $match.date = $match.date || {};
+                $match.date.$lt = new Date(endDate);
+            }
+
+            return mongo.Activities.aggregate([
+                {$match},
+                {$group: {
+                    _id: {owner: '$owner', group: '$group'},
+                    amount: {$sum: '$amount'}
+                }}
+            ]).exec().then((data) => {
+                return _.reduce(data, (acc, { _id, amount }) => {
+                    const {owner, group} = _id;
+                    acc[owner] = _.merge(acc[owner] || {}, { [group]: amount });
+                    return acc;
+                }, {});
+            });
+        }
+
+        static detail({startDate, endDate}) {
+            const $match = { };
+
+            if (startDate)
+                $match.date = {$gte: new Date(startDate)};
+
+            if (endDate) {
+                $match.date = $match.date || {};
+                $match.date.$lt = new Date(endDate);
+            }
+
+            return mongo.Activities.aggregate([
+                {$match},
+                {$group: {_id: {owner: '$owner', action: '$action'}, total: {$sum: '$amount'}}}
+            ]).exec().then((data) => {
+                return _.reduce(data, (acc, { _id, total }) => {
+                    const {owner, action} = _id;
+                    acc[owner] = _.merge(acc[owner] || {}, { [action]: total });
+                    return acc;
+                }, {});
+            });
+        }
+    }
+
+    return ActivitiesService;
+};
diff --git a/modules/web-console/backend/services/users.js b/modules/web-console/backend/services/users.js
index 8058b25f9fed6..2dd603f193382 100644
--- a/modules/web-console/backend/services/users.js
+++ b/modules/web-console/backend/services/users.js
@@ -21,7 +21,7 @@
 
 module.exports = {
     implements: 'services/users',
-    inject: ['require(lodash)', 'mongo', 'settings', 'services/spaces', 'services/mails', 'agent-manager', 'errors']
+    inject: ['require(lodash)', 'mongo', 'settings', 'services/spaces', 'services/mails', 'services/activities', 'agent-manager', 'errors']
 };
 
 /**
@@ -30,11 +30,12 @@ module.exports = {
  * @param settings
  * @param {SpacesService} spacesService
  * @param {MailsService} mailsService
+ * @param {ActivitiesService} activitiesService
  * @param agentMgr
  * @param errors
  * @returns {UsersService}
  */
-module.exports.factory = (_, mongo, settings, spacesService, mailsService, agentMgr, errors) => {
+module.exports.factory = (_, mongo, settings, spacesService, mailsService, activitiesService, agentMgr, errors) => {
     const _randomString = () => {
         const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
         const possibleLen = possible.length;
@@ -143,7 +144,7 @@ module.exports.factory = (_, mongo, settings, spacesService, mailsService, agent
          * Get list of user accounts and summary information.
          * @returns {mongo.Account[]} - returns all accounts with counters object
          */
-        static list() {
+        static list(params) {
             return Promise.all([
                 mongo.Space.aggregate([
                     {$match: {demo: false}},
@@ -161,13 +162,17 @@ module.exports.factory = (_, mongo, settings, spacesService, mailsService, agent
                         }
                     }
                 ]).exec(),
+                activitiesService.total(params),
+                activitiesService.detail(params),
                 mongo.Account.find({}).sort('firstName lastName').lean().exec()
             ])
-                .then(([counters, users]) => {
+                .then(([counters, activitiesTotal, activitiesDetail, users]) => {
                     const countersMap = _.keyBy(counters, 'owner');
 
                     _.forEach(users, (user) => {
                         user.counters = _.omit(countersMap[user._id], '_id', 'owner');
+                        user.activitiesTotal = activitiesTotal[user._id];
+                        user.activitiesDetail = activitiesDetail[user._id];
                     });
 
                     return users;
diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js
index 7416ce9e3a283..0e85711141799 100644
--- a/modules/web-console/frontend/app/app.config.js
+++ b/modules/web-console/frontend/app/app.config.js
@@ -94,3 +94,12 @@ igniteConsoleCfg.config(['$dropdownProvider', ($dropdownProvider) => {
         templateUrl: 'templates/dropdown.html'
     });
 }]);
+
+// AngularStrap dropdowns () configuration.
+igniteConsoleCfg.config(['$datepickerProvider', ($datepickerProvider) => {
+    angular.extend($datepickerProvider.defaults, {
+        autoclose: true,
+        iconLeft: 'icon-datepicker-left',
+        iconRight: 'icon-datepicker-right'
+    });
+}]);
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 4ecd9b5394483..9958cb53ae539 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -16,7 +16,9 @@
  */
 
 import '../public/stylesheets/style.scss';
-import '../app/directives/ui-grid-settings/ui-grid-settings.scss';
+import '../app/components/ui-grid-header/ui-grid-header.scss';
+import '../app/components/ui-grid-settings/ui-grid-settings.scss';
+import '../app/components/form-field-datepicker/form-field-datepicker.scss';
 import './helpers/jade/mixins.jade';
 
 import './app.config';
@@ -25,10 +27,10 @@ import './decorator/select';
 import './decorator/tooltip';
 
 import './modules/form/form.module';
-import './modules/agent/agent.module.js';
+import './modules/agent/agent.module';
 import './modules/sql/sql.module';
 import './modules/nodes/nodes.module';
-import './modules/Demo/Demo.module.js';
+import './modules/demo/Demo.module';
 
 import './modules/states/signin.state';
 import './modules/states/logout.state';
@@ -39,6 +41,7 @@ import './modules/states/admin.state';
 import './modules/states/errors.state';
 
 // ignite:modules
+import './core';
 import './modules/user/user.module';
 import './modules/branding/branding.module';
 import './modules/navbar/navbar.module';
@@ -50,6 +53,9 @@ import './modules/socket.module';
 import './modules/loading/loading.module';
 // endignite
 
+// Data
+import i18n from './data/i18n';
+
 // Directives.
 import igniteAutoFocus from './directives/auto-focus.directive.js';
 import igniteBsAffixUpdate from './directives/bs-affix-update.directive';
@@ -98,9 +104,9 @@ import defaultName from './filters/default-name.filter';
 import domainsValidation from './filters/domainsValidation.filter';
 import duration from './filters/duration.filter';
 import hasPojo from './filters/hasPojo.filter';
+import uiGridSubcategories from './filters/uiGridSubcategories.filter';
 
 // Controllers
-import admin from 'controllers/admin-controller';
 import caches from 'controllers/caches-controller';
 import clusters from 'controllers/clusters-controller';
 import domains from 'controllers/domains-controller';
@@ -109,6 +115,10 @@ import profile from 'controllers/profile-controller';
 import auth from './controllers/auth.controller';
 import resetPassword from './controllers/reset-password.controller';
 
+// Components
+import igniteListOfRegisteredUsers from './components/list-of-registered-users';
+import IgniteActivitiesUserDialog from './components/activities-user-dialog';
+
 // Inject external modules.
 import 'ignite_modules_temp/index';
 
@@ -129,6 +139,7 @@ angular
     'nvd3',
     'smart-table',
     'treeControl',
+    'pascalprecht.translate',
     'ui.grid',
     'ui.grid.saveState',
     'ui.grid.selection',
@@ -136,6 +147,7 @@ angular
     'ui.grid.autoResize',
     'ui.grid.exporter',
     // Base modules.
+    'ignite-console.core',
     'ignite-console.ace',
     'ignite-console.Form',
     'ignite-console.user',
@@ -186,6 +198,7 @@ angular
 .directive(...igniteRetainSelection)
 .directive('igniteOnFocusOut', igniteOnFocusOut)
 .directive('igniteRestoreInputFocus', igniteRestoreInputFocus)
+.directive('igniteListOfRegisteredUsers', igniteListOfRegisteredUsers)
 // Services.
 .service('IgniteErrorPopover', ErrorPopover)
 .service('JavaTypes', JavaTypes)
@@ -204,8 +217,8 @@ angular
 .service(...FormUtils)
 .service(...LegacyUtils)
 .service(...UnsavedChangesGuard)
+.service('IgniteActivitiesUserDialog', IgniteActivitiesUserDialog)
 // Controllers.
-.controller(...admin)
 .controller(...auth)
 .controller(...resetPassword)
 .controller(...caches)
@@ -219,7 +232,11 @@ angular
 .filter(...domainsValidation)
 .filter(...duration)
 .filter(...hasPojo)
-.config(['$stateProvider', '$locationProvider', '$urlRouterProvider', ($stateProvider, $locationProvider, $urlRouterProvider) => {
+.filter('uiGridSubcategories', uiGridSubcategories)
+.config(['$translateProvider', '$stateProvider', '$locationProvider', '$urlRouterProvider', ($translateProvider, $stateProvider, $locationProvider, $urlRouterProvider) => {
+    $translateProvider.translations('en', i18n);
+    $translateProvider.preferredLanguage('en');
+
     // Set up the states.
     $stateProvider
         .state('base', {
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
new file mode 100644
index 0000000000000..46853b262eaad
--- /dev/null
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const COLUMNS_DEFS = [
+    {displayName: 'Action', field: 'action', minWidth: 65 },
+    {displayName: 'Description', field: 'title', minWidth: 65 },
+    {displayName: 'Visited', field: 'amount', minWidth: 65 }
+];
+
+export default class ActivitiesCtrl {
+    static $inject = ['$state', 'user', 'params', 'IgniteActivitiesData'];
+
+    constructor($state, user, params, ActivitiesData) {
+        const $ctrl = this;
+        const userId = user._id;
+
+        $ctrl.user = user;
+
+        $ctrl.gridOptions = {
+            data: [],
+            columnVirtualizationThreshold: 30,
+            columnDefs: COLUMNS_DEFS,
+            categories: [
+                {name: 'Action', visible: true, selectable: true},
+                {name: 'Description', visible: true, selectable: true},
+                {name: 'Visited', visible: true, selectable: true}
+            ],
+            enableRowSelection: false,
+            enableRowHeaderSelection: false,
+            enableColumnMenus: false,
+            multiSelect: false,
+            modifierKeysToMultiSelect: true,
+            noUnselect: true,
+            flatEntityAccess: true,
+            fastWatch: true,
+            onRegisterApi: (api) => {
+                $ctrl.gridApi = api;
+            }
+        };
+
+        ActivitiesData.listByUser(userId, params)
+            .then((data) => {
+                $ctrl.data = data;
+            });
+    }
+}
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
new file mode 100644
index 0000000000000..2c55ebd6da29a
--- /dev/null
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
@@ -0,0 +1,36 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+.modal(tabindex='-1' role='dialog')
+    .modal-dialog
+        .modal-content
+            .modal-header
+                h4.modal-title 
+                    i.fa.fa-info-circle
+                    | Activities: {{ ctrl.user.userName }}
+            .modal-body.modal-body-with-scroll(id='activities-user-dialog')
+                table.table.table-striped.table-bordered.table-hover(scrollable-container='#activities-user-dialog' st-table='displayedRows' st-safe-src='ctrl.data')
+                    thead
+                        th.text-center(st-sort='title') Description
+                        th.text-center(st-sort='action') Action
+                        th.text-center(st-sort='amount') Visited
+                    tbody
+                        tr(ng-repeat='row in displayedRows')
+                            td.text-left {{ row.action | translate }}
+                            td.text-left {{ row.action }}
+                            td.text-left {{ row.amount }}
+            .modal-footer
+                button.btn.btn-primary(id='confirm-btn-confirm' ng-click='$hide()') Close
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/index.js b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
new file mode 100644
index 0000000000000..03d35852b147a
--- /dev/null
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ import controller from './activities-user-dialog.controller';
+ import templateUrl from './activities-user-dialog.jade';
+
+ export default ['$modal', ($modal) => ({ show = true, user, params }) => {
+     const ActivitiesUserDialog = $modal({
+         templateUrl,
+         show,
+         resolve: {
+             user: () => user,
+             params: () => params
+         },
+         placement: 'center',
+         controller,
+         controllerAs: 'ctrl'
+     });
+
+     return ActivitiesUserDialog.$promise
+         .then(() => ActivitiesUserDialog);
+ }];
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
new file mode 100644
index 0000000000000..6792977b65831
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
@@ -0,0 +1,55 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+mixin ignite-form-field-datepicker(label, model, name, disabled, required, placeholder, tip)
+    mixin form-field-input()
+        input.form-control(
+            id='{{ #{name} }}Input'
+            name='{{ #{name} }}'
+
+            placeholder=placeholder
+            
+            data-ng-model=model
+
+            data-ng-required=required && '#{required}'
+            data-ng-disabled=disabled && '#{disabled}'
+
+            bs-datepicker
+            data-date-format='MMM yyyy' 
+            data-start-view='1' 
+            data-min-view='1' 
+            data-max-date='today'
+
+            data-container='body > .wrapper'
+
+            tabindex='0'
+
+            onkeydown="return false"
+
+            data-ignite-form-panel-field=''
+        )&attributes(attributes.attributes)
+
+    .ignite-form-field
+        +ignite-form-field__label(label, name, required)
+        .ignite-form-field__control
+            if tip
+                i.tipField.icon-help(bs-tooltip='' data-title=tip)
+
+            if block
+                block
+
+            .input-tip
+                +form-field-input(attributes=attributes)
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.scss b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.scss
new file mode 100644
index 0000000000000..0f6fe6eb62519
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.scss
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.datepicker.dropdown-menu tbody button {
+    height: 100%;
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/index.js b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
new file mode 100644
index 0000000000000..32a34f41c02d6
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import templateUrl from './list-of-registered-users.jade';
+import controller from './list-of-registered-users.controller';
+
+export default [() => {
+    return {
+        scope: true,
+        templateUrl,
+        controller,
+        controllerAs: '$ctrl'
+    };
+}];
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.categories.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.categories.js
new file mode 100644
index 0000000000000..95edf8b092b66
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.categories.js
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default [
+    {name: 'Actions', visible: true, selectable: true},
+    {name: 'User', visible: true, selectable: true},
+    {name: 'Email', visible: true, selectable: true},
+    {name: 'Company', visible: true, selectable: true},
+    {name: 'Country', visible: true, selectable: true},
+    {name: 'Last login', visible: false, selectable: true},
+    {name: 'Last activity', visible: true, selectable: true},
+    {name: 'Configurations', visible: false, selectable: true},
+    {name: 'Total activities', visible: true, selectable: true},
+    {name: 'Configuration\'s activities', visible: false, selectable: true},
+    {name: 'Queries\' activities', visible: false, selectable: true}
+];
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
new file mode 100644
index 0000000000000..61e1bd8575b22
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const ICON_SORT = '<span ui-grid-one-bind-id-grid="col.uid + \'-sortdir-text\'" ui-grid-visible="col.sort.direction" aria-label="Sort Descending"><i ng-class="{ \'ui-grid-icon-up-dir\': col.sort.direction == asc, \'ui-grid-icon-down-dir\': col.sort.direction == desc, \'ui-grid-icon-blank\': !col.sort.direction }" title="" aria-hidden="true"></i></span>';
+
+const USER_TEMPLATE = '<div class="ui-grid-cell-contents"><i class="pull-left" ng-class="row.entity.admin ? \'icon-admin\' : \'icon-user\'"></i>{{ COL_FIELD }}</div>';
+
+const CLUSTER_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-sitemap'></i>${ICON_SORT}</div>`;
+const MODEL_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-object-group'></i>${ICON_SORT}</div>`;
+const CACHE_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-database'></i>${ICON_SORT}</div>`;
+const IGFS_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-folder-o'></i>${ICON_SORT}</div>`;
+
+const ACTIONS_TEMPLATE = `
+<div class='text-center ui-grid-cell-actions'>
+    <a class='btn btn-default dropdown-toggle' bs-dropdown='' data-placement='bottom-right' data-container='.panel'>
+        <i class='fa fa-gear'></i>&nbsp;
+        <span class='caret'></span>
+    </a>
+    <ul class='dropdown-menu' role='menu'>
+        <li ng-show='row.entity._id != $root.user._id'>
+            <a ng-click='grid.api.becomeUser(row.entity)'>Become this user</a>
+        </li>
+        <li ng-show='row.entity._id != $root.user._id'>
+            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='row.entity.admin && row.entity._id !== $root.user._id'>Revoke admin</a>
+            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='!row.entity.admin && row.entity._id !== $root.user._id'>Grant admin</a>
+        </li>
+        <li ng-show='row.entity._id != $root.user._id'>
+            <a ng-click='grid.api.removeUser(row.entity)'>Remove user</a>
+        </li>
+        <li>
+            <a ng-click='grid.api.showActivities(row.entity)'>Activity detail</a>
+        </li>
+</div>`;
+
+const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
+
+export default [
+    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'test', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, pinnedLeft: true},
+    {displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by name...' }, pinnedLeft: true},
+    {displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
+    {displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 160, enableFiltering: true},
+    {displayName: 'Country', categoryDisplayName: 'Country', field: 'countryCode', minWidth: 80, enableFiltering: true},
+    {displayName: 'Last login', categoryDisplayName: 'Last login', field: 'lastLogin', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 105, width: 105, enableFiltering: false, visible: false},
+    {displayName: 'Last activity', categoryDisplayName: 'Last activity', field: 'lastActivity', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 105, width: 105, enableFiltering: false, visible: true, sort: { direction: 'desc', priority: 0 }},
+    // Configurations
+    {displayName: 'Clusters count', categoryDisplayName: 'Configurations', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: 'counters.clusters', type: 'number', headerTooltip: 'Clusters count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
+    {displayName: 'Models count', categoryDisplayName: 'Configurations', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: 'counters.models', type: 'number', headerTooltip: 'Models count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
+    {displayName: 'Caches count', categoryDisplayName: 'Configurations', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: 'counters.caches', type: 'number', headerTooltip: 'Caches count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
+    {displayName: 'IGFS count', categoryDisplayName: 'Configurations', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: 'counters.igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
+    // Activities Total
+    {displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Configuration', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Queries', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Demo', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'AD', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Agent Download', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'AS', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Agent Start', minWidth: 50, width: 50, enableFiltering: false},
+    // Activities Configuration
+    {displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration Clusters', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration Model', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', headerTooltip: 'Configuration Caches', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'IGFS', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/igfs"] || 0', type: 'number', headerTooltip: 'Configuration IGFS', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', headerTooltip: 'Configuration Summary', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    // Activities Queries
+    {displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', headerTooltip: 'Query execute', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', headerTooltip: 'Query explain', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', headerTooltip: 'Scan', minWidth: 50, width: 80, enableFiltering: false, visible: false}
+];
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
new file mode 100644
index 0000000000000..19f7921ac7f17
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import headerTemplate from 'app/components/ui-grid-header/ui-grid-header.jade';
+
+import columnDefs from './list-of-registered-users.column-defs';
+import categories from './list-of-registered-users.categories';
+
+export default class IgniteListOfRegisteredUsersCtrl {
+    static $inject = ['$scope', '$state', '$templateCache', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
+
+    constructor($scope, $state, $templateCache, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
+        const $ctrl = this;
+
+        const companySelectOptions = [];
+        const countrySelectOptions = [];
+
+        $ctrl.params = {
+            startDate: new Date()
+        };
+
+        $ctrl.params.startDate.setDate(1);
+        $ctrl.params.startDate.setHours(0, 0, 0, 0);
+
+        const columnCompany = _.find(columnDefs, { displayName: 'Company' });
+        const columnCountry = _.find(columnDefs, { displayName: 'Country' });
+
+        columnCompany.filter = {
+            selectOptions: companySelectOptions,
+            type: uiGridConstants.filter.SELECT,
+            condition: uiGridConstants.filter.EXACT
+        };
+
+        columnCountry.filter = {
+            selectOptions: countrySelectOptions,
+            type: uiGridConstants.filter.SELECT,
+            condition: uiGridConstants.filter.EXACT
+        };
+
+        const becomeUser = (user) => {
+            AdminData.becomeUser(user._id)
+                .then(() => User.load())
+                .then(() => $state.go('base.configuration.clusters'))
+                .then(() => NotebookData.load());
+        };
+
+        const removeUser = (user) => {
+            Confirm.confirm(`Are you sure you want to remove user: "${user.userName}"?`)
+                .then(() => AdminData.removeUser(user))
+                .then(() => {
+                    const i = _.findIndex($ctrl.gridOptions.data, (u) => u._id === user._id);
+
+                    if (i >= 0)
+                        $ctrl.gridOptions.data.splice(i, 1);
+                })
+                .then(() => $ctrl.adjustHeight($ctrl.gridOptions.data.length));
+        };
+
+        const toggleAdmin = (user) => {
+            if (user.adminChanging)
+                return;
+
+            user.adminChanging = true;
+
+            AdminData.toggleAdmin(user)
+                .then(() => user.admin = !user.admin)
+                .finally(() => user.adminChanging = false);
+        };
+
+        const showActivities = (user) => {
+            return new ActivitiesUserDialog({ user, params: $ctrl.params });
+        };
+
+        $ctrl.gridOptions = {
+            data: [],
+            columnVirtualizationThreshold: 30,
+            columnDefs,
+            categories,
+            headerTemplate: $templateCache.get(headerTemplate),
+            enableFiltering: true,
+            enableRowSelection: false,
+            enableRowHeaderSelection: false,
+            enableColumnMenus: false,
+            multiSelect: false,
+            modifierKeysToMultiSelect: true,
+            noUnselect: true,
+            fastWatch: true,
+            onRegisterApi: (api) => {
+                $ctrl.gridApi = api;
+
+                api.becomeUser = becomeUser;
+                api.removeUser = removeUser;
+                api.toggleAdmin = toggleAdmin;
+                api.showActivities = showActivities;
+            }
+        };
+
+        const usersToFilterOptions = (column) => {
+            return _.sortBy(
+                _.map(
+                    _.groupBy($ctrl.gridOptions.data, (usr) => {
+                        const fld = usr[column];
+
+                        return _.isNil(fld) ? fld : fld.toUpperCase();
+                    }),
+                    (arr, value) => ({label: `${_.head(arr)[column] || 'Not set'} (${arr.length})`, value})
+                ),
+                'value');
+        };
+
+        /**
+         * @param {{startDate: Date, endDate: Date}} params
+         */
+        const reloadUsers = (params) => {
+            AdminData.loadUsers(params)
+                .then((data) => $ctrl.gridOptions.data = data)
+                .then((data) => {
+                    companySelectOptions.push(...usersToFilterOptions('company'));
+                    countrySelectOptions.push(...usersToFilterOptions('countryCode'));
+
+                    this.gridApi.grid.refresh();
+
+                    return data;
+                })
+                .then((data) => $ctrl.adjustHeight(data.length));
+        };
+
+        $scope.$watch(() => $ctrl.params.startDate, () => {
+            const endDate = new Date($ctrl.params.startDate);
+
+            endDate.setMonth(endDate.getMonth() + 1);
+
+            $ctrl.params.endDate = endDate;
+
+            reloadUsers($ctrl.params);
+        });
+    }
+
+    adjustHeight(rows) {
+        const height = Math.min(rows, 20) * 30 + 75;
+
+        // Remove header height.
+        this.gridApi.grid.element.css('height', height + 'px');
+
+        this.gridApi.core.handleWindowResize();
+    }
+
+    _enableColumns(_categories, visible) {
+        _.forEach(_categories, (cat) => {
+            cat.visible = visible;
+
+            _.forEach(this.gridOptions.columnDefs, (col) => {
+                if (col.categoryDisplayName === cat.name)
+                    col.visible = visible;
+            });
+        });
+
+        // Workaround for this.gridApi.grid.refresh() didn't return promise.
+        this.gridApi.grid.processColumnsProcessors(this.gridApi.grid.columns)
+            .then((renderableColumns) => this.gridApi.grid.setVisibleColumns(renderableColumns))
+            .then(() => this.gridApi.grid.redrawInPlace())
+            .then(() => this.gridApi.grid.refreshCanvas(true))
+            .then(() => {
+                if (visible) {
+                    const categoryDisplayName = _.last(_categories).name;
+
+                    const col = _.findLast(this.gridOptions.columnDefs, {categoryDisplayName});
+
+                    this.gridApi.grid.scrollTo(null, col);
+                }
+            });
+    }
+
+    _selectableColumns() {
+        return _.filter(this.gridOptions.categories, (cat) => cat.selectable);
+    }
+
+    toggleColumns(category, visible) {
+        this._enableColumns([category], visible);
+    }
+
+    selectAllColumns() {
+        this._enableColumns(this._selectableColumns(), true);
+    }
+
+    clearAllColumns() {
+        this._enableColumns(this._selectableColumns(), false);
+    }
+
+    exportCsv() {
+        this.gridApi.exporter.csvExport('all', 'visible');
+    }
+}
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
new file mode 100644
index 0000000000000..efed9c0edee16
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
@@ -0,0 +1,54 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins.jade
+include /app/components/form-field-datepicker/form-field-datepicker.jade
+
+mixin grid-settings()
+    i.fa.fa-bars(data-animation='am-flip-x' bs-dropdown='' aria-haspopup='true' aria-expanded='expanded' data-auto-close='1' data-trigger='click')
+    ul.select.dropdown-menu(role='menu')
+        li(ng-repeat='item in $ctrl.gridOptions.categories|filter:{selectable:true}')
+            a(ng-click='$ctrl.toggleColumns(item, !item.visible)')
+                i.fa.fa-check-square-o.pull-left(ng-if='item.visible')
+                i.fa.fa-square-o.pull-left(ng-if='!item.visible')
+                span {{::item.name}}
+        li.divider
+        li
+            a(ng-click='$ctrl.selectAllColumns()') Select all
+        li
+            a(ng-click='$ctrl.clearAllColumns()') Clear all
+        li.divider
+        li
+            a(ng-click='$hide()') Close
+
+.panel.panel-default
+    .panel-heading.ui-grid-settings
+        +grid-settings
+        label Total users: 
+            strong {{ $ctrl.gridOptions.data.length }}&nbsp;&nbsp;&nbsp;
+        label Showing users: 
+            strong {{ $ctrl.gridApi.grid.getVisibleRows().length }}
+            sub(ng-show='users.length === $ctrl.gridApi.grid.getVisibleRows().length') all
+        div.ui-grid-settings-dateperiod
+            form(ng-form=form novalidate)
+                -var form = 'admin'
+
+                +ignite-form-field-datepicker('Period:', '$ctrl.params.startDate', '"period"')
+                
+                button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
+
+    .panel-collapse
+        .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-exporter ui-grid-pinning)
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
new file mode 100644
index 0000000000000..7e44d94671452
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
@@ -0,0 +1,27 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+.ui-grid-header.ui-grid-header--subcategories(role='rowgroup')
+    .ui-grid-top-panel
+        .ui-grid-header-viewport
+            .ui-grid-header-canvas
+                .ui-grid-header-cell-wrapper(ng-style='colContainer.headerCellWrapperStyle()')
+                    .ui-grid-header-cell-row(role='row')
+                        .ui-grid-header-span.ui-grid-header-cell.ui-grid-clearfix(ng-repeat='cat in grid.options.categories')
+                            div(ng-show='(colContainer.renderedColumns|uiGridSubcategories:cat.name).length > 1')
+                                .ui-grid-cell-contents {{ cat.name }}
+                            .ui-grid-header-cell-row
+                                .ui-grid-header-cell.ui-grid-clearfix(ng-repeat='col in (colContainer.renderedColumns|uiGridSubcategories:cat.name) track by col.uid' ui-grid-header-cell='' col='col' render-index='$index')
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
new file mode 100644
index 0000000000000..c39050469db9c
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.ui-grid-header--subcategories {
+    .ui-grid-row:nth-child(even) .ui-grid-cell.cell-total {
+        background-color: rgba(102,175,233,.6);
+    }
+
+    .ui-grid-row:nth-child(odd) .ui-grid-cell.cell-total {
+        background-color: rgba(102,175,233,.3);
+    }
+
+    .ui-grid-header-cell-row {
+        height: 30px;
+    }
+
+    .ui-grid-header-cell [role="columnheader"] {
+        display: flex;
+        
+        flex-wrap: wrap;
+        align-items: center;
+        justify-content: center;
+
+        height: 100%;
+
+        & > div {
+            flex: 1 100%;
+            height: auto;
+        }
+
+        & > div[ui-grid-filter] {
+            flex: auto;
+        }
+    }
+
+    .ui-grid-header-span {
+        position: relative;
+        border-right: 0;
+
+        .ng-hide + .ui-grid-header-cell-row .ui-grid-header-cell {
+            height: 58px;
+        }
+
+        .ng-hide + .ui-grid-header-cell-row .ui-grid-cell-contents {
+            padding: 5px 5px;
+        }
+
+        .ui-grid-column-resizer.right {
+            top: -100px;
+        }
+        .ng-hide + .ui-grid-header-cell-row .ui-grid-column-resizer.right {
+            bottom: -100px;
+        }
+
+        &.ui-grid-header-cell .ui-grid-header-cell .ui-grid-column-resizer.right {
+            border-right-width: 0;
+        }
+        &.ui-grid-header-cell .ui-grid-header-cell:last-child .ui-grid-column-resizer.right {
+            border-right-width: 1px;
+        }
+
+        & > div > .ui-grid-cell-contents {
+            border-bottom: 1px solid #d4d4d4;
+        }
+    }
+
+    input {
+        line-height: 21px;
+    }
+}
diff --git a/modules/web-console/frontend/app/directives/ui-grid-settings/ui-grid-settings.jade b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.jade
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-grid-settings/ui-grid-settings.jade
rename to modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.jade
diff --git a/modules/web-console/frontend/app/directives/ui-grid-settings/ui-grid-settings.scss b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
similarity index 68%
rename from modules/web-console/frontend/app/directives/ui-grid-settings/ui-grid-settings.scss
rename to modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
index 6517a603e137a..3016488f6bb6e 100644
--- a/modules/web-console/frontend/app/directives/ui-grid-settings/ui-grid-settings.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
@@ -35,4 +35,36 @@
         padding-right: 8px;
         cursor: pointer;
     }
+
+    &-dateperiod {
+        float: right;
+
+        .ignite-form-field {
+            width: 160px;
+            margin-right: 10px;
+
+            &__label {
+            }
+
+            &__control {
+            }
+
+            &:nth-child(1) {
+                float: left;
+
+                .ignite-form-field__label {
+                    width: 40%;
+                }
+
+                .ignite-form-field__control {
+                    width: 60%;
+                }
+            }
+        }
+
+        .btn {
+            line-height: 20px;
+            margin-right: 0;
+        }
+    }
 }
diff --git a/modules/web-console/frontend/app/core/activities/Activities.data.js b/modules/web-console/frontend/app/core/activities/Activities.data.js
new file mode 100644
index 0000000000000..8a67a97a5dd43
--- /dev/null
+++ b/modules/web-console/frontend/app/core/activities/Activities.data.js
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class ActivitiesData {
+    static $inject = ['$http', '$state'];
+
+    constructor($http, $state) {
+        this.$http = $http;
+        this.$state = $state;
+    }
+
+    post(options = {}) {
+        let { group, action } = options;
+
+        action = action || this.$state.$current.url.source;
+        group = group || action.match(/^\/([^/]+)/)[1];
+
+        return this.$http.post('/api/v1/activities/page', { group, action });
+    }
+
+    listByUser(userId, params) {
+        return this.$http.get(`/api/v1/activities/user/${userId}`, { params })
+            .then(({ data }) => data);
+    }
+}
diff --git a/modules/web-console/frontend/app/core/admin/Admin.data.js b/modules/web-console/frontend/app/core/admin/Admin.data.js
new file mode 100644
index 0000000000000..66d82f0388518
--- /dev/null
+++ b/modules/web-console/frontend/app/core/admin/Admin.data.js
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class IgniteAdminData {
+    static $inject = ['$http', 'IgniteMessages', 'IgniteCountries'];
+
+    constructor($http, Messages, Countries) {
+        this.$http = $http;
+        this.Messages = Messages;
+        this.Countries = Countries;
+    }
+
+    becomeUser(viewedUserId) {
+        return this.$http.get('/api/v1/admin/become', {
+            params: {viewedUserId}
+        })
+        .catch(this.Messages.showError);
+    }
+
+    removeUser(user) {
+        return this.$http.post('/api/v1/admin/remove', {
+            userId: user._id
+        })
+        .then(() => {
+            this.Messages.showInfo(`User has been removed: "${user.userName}"`);
+        })
+        .catch(({data, status}) => {
+            if (status === 503)
+                this.Messages.showInfo(data);
+            else
+                this.Messages.showError('Failed to remove user: ', data);
+        });
+    }
+
+    toggleAdmin(user) {
+        return this.$http.post('/api/v1/admin/save', {
+            userId: user._id,
+            adminFlag: !user.admin
+        })
+        .then(() => {
+            this.Messages.showInfo(`Admin right was successfully toggled for user: "${user.userName}"`);
+        })
+        .catch((res) => {
+            this.Messages.showError('Failed to toggle admin right for user: ', res);
+        });
+    }
+
+    prepareUsers(user) {
+        const { Countries } = this;
+
+        user.userName = user.firstName + ' ' + user.lastName;
+        user.countryCode = Countries.getByName(user.country).code;
+
+        return user;
+    }
+
+    loadUsers(params) {
+        return this.$http.post('/api/v1/admin/list', params)
+            .then(({ data }) => data)
+            .then((users) => _.map(users, this.prepareUsers.bind(this)))
+            .catch(this.Messages.showError);
+    }
+}
diff --git a/modules/web-console/frontend/app/core/index.js b/modules/web-console/frontend/app/core/index.js
new file mode 100644
index 0000000000000..7f72ee32f811f
--- /dev/null
+++ b/modules/web-console/frontend/app/core/index.js
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import IgniteAdminData from './admin/Admin.data';
+import IgniteActivitiesData from './activities/Activities.data';
+
+angular.module('ignite-console.core', [])
+    .service('IgniteAdminData', IgniteAdminData)
+    .service('IgniteActivitiesData', IgniteActivitiesData);
diff --git a/modules/web-console/frontend/app/data/i18n.js b/modules/web-console/frontend/app/data/i18n.js
new file mode 100644
index 0000000000000..bc8c700d8bfa6
--- /dev/null
+++ b/modules/web-console/frontend/app/data/i18n.js
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default {
+    '/agent/start': 'Agent start',
+    '/agent/download': 'Agent download',
+    '/configuration/clusters': 'Configure clusters',
+    '/configuration/caches': 'Configure caches',
+    '/configuration/domains': 'Configure domain model',
+    '/configuration/igfs': 'Configure IGFS',
+    '/configuration/summary': 'Configurations summary',
+    '/demo/resume': 'Demo resume',
+    '/demo/reset': 'Demo reset',
+    '/queries/execute': 'Query execute',
+    '/queries/explain': 'Query explain',
+    '/queries/scan': 'Scan',
+    '/queries/add/query': 'Add query',
+    '/queries/add/scan': 'Add scan',
+    '/queries/demo': 'SQL demo',
+    '/queries/notebook/': 'Query notebook',
+    '/settings/profile': 'User profile',
+    '/settings/admin': 'Admin panel',
+    '/logout': 'Logout'
+};
diff --git a/modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js b/modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js
new file mode 100644
index 0000000000000..f36ae6e057023
--- /dev/null
+++ b/modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default [() => {
+    return (arr, category) => {
+        return _.filter(arr, (item) => {
+            return item.colDef.categoryDisplayName === category;
+        });
+    };
+}];
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 0d0b171df5595..4e972ef88ab6f 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -186,8 +186,8 @@ class Paragraph {
 }
 
 // Controller for SQL notebook screen.
-export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion',
-    function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version) {
+export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData',
+    function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData) {
         let stopTopology = null;
 
         const _tryStopRefresh = function(paragraph) {
@@ -965,6 +965,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
         $scope.addQuery = function() {
             const sz = $scope.notebook.paragraphs.length;
 
+            ActivitiesData.post({ action: '/queries/add/query' });
+
             const paragraph = new Paragraph($animate, $timeout, {
                 name: 'Query' + (sz === 0 ? '' : sz),
                 query: '',
@@ -991,6 +993,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
         $scope.addScan = function() {
             const sz = $scope.notebook.paragraphs.length;
 
+            ActivitiesData.post({ action: '/queries/add/scan' });
+
             const paragraph = new Paragraph($animate, $timeout, {
                 name: 'Scan' + (sz === 0 ? '' : sz),
                 query: '',
@@ -1379,6 +1383,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
                             const qry = args.maxPages ? addLimit(args.query, args.pageSize * args.maxPages) : paragraph.query;
 
+                            ActivitiesData.post({ action: '/queries/execute' });
+
                             return agentMonitor.query(nid, args.cacheName, qry, nonCollocatedJoins, local, args.pageSize);
                         })
                         .then((res) => {
@@ -1430,6 +1436,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                         pageSize: paragraph.pageSize
                     };
 
+                    ActivitiesData.post({ action: '/queries/explain' });
+
                     return agentMonitor.query(nid, args.cacheName, args.query, false, false, args.pageSize);
                 })
                 .then(_processQueryResult.bind(this, paragraph, true))
@@ -1466,6 +1474,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                                 localNid: local ? nid : null
                             };
 
+                            ActivitiesData.post({ action: '/queries/scan' });
+
                             return agentMonitor.query(nid, args.cacheName, query, false, local, args.pageSize);
                         })
                         .then((res) => _processQueryResult(paragraph, true, res))
diff --git a/modules/web-console/frontend/app/modules/sql/sql.module.js b/modules/web-console/frontend/app/modules/sql/sql.module.js
index a1ffde9646f99..5875961f0a896 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.module.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.module.js
@@ -30,7 +30,7 @@ angular.module('ignite-console.sql', [
             // set up the states
             $stateProvider
                 .state('base.sql', {
-                    url: '/sql',
+                    url: '/queries',
                     abstract: true,
                     template: '<ui-view></ui-view>'
                 })
diff --git a/modules/web-console/frontend/app/modules/states/admin.state.js b/modules/web-console/frontend/app/modules/states/admin.state.js
index c3151e1f442a8..35c6fbbfeeab0 100644
--- a/modules/web-console/frontend/app/modules/states/admin.state.js
+++ b/modules/web-console/frontend/app/modules/states/admin.state.js
@@ -29,7 +29,7 @@ angular
         templateUrl: '/settings/admin.html',
         onEnter: AclRoute.checkAccess('admin_page'),
         metaTags: {
-            title: 'List of registered users'
+            title: 'Admin panel'
         }
     });
 }]);
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
index cfc6df9c89538..16d2faef04335 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
@@ -21,8 +21,8 @@ import saver from 'file-saver';
 const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
 
 export default [
-    '$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteLoading', '$filter', 'IgniteConfigurationResource', 'JavaTypes', 'IgniteVersion', 'IgniteConfigurationGenerator', 'SpringTransformer', 'JavaTransformer', 'IgniteDockerGenerator', 'IgniteMavenGenerator', 'IgnitePropertiesGenerator', 'IgniteReadmeGenerator', 'IgniteFormUtils', 'IgniteSummaryZipper',
-    function($root, $scope, $http, LegacyUtils, Messages, Loading, $filter, Resource, JavaTypes, Version, generator, spring, java, docker, pom, propsGenerator, readme, FormUtils, SummaryZipper) {
+    '$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteLoading', '$filter', 'IgniteConfigurationResource', 'JavaTypes', 'IgniteVersion', 'IgniteConfigurationGenerator', 'SpringTransformer', 'JavaTransformer', 'IgniteDockerGenerator', 'IgniteMavenGenerator', 'IgnitePropertiesGenerator', 'IgniteReadmeGenerator', 'IgniteFormUtils', 'IgniteSummaryZipper', 'IgniteActivitiesData',
+    function($root, $scope, $http, LegacyUtils, Messages, Loading, $filter, Resource, JavaTypes, Version, generator, spring, java, docker, pom, propsGenerator, readme, FormUtils, SummaryZipper, ActivitiesData) {
         const ctrl = this;
 
         $scope.ui = {
@@ -304,6 +304,8 @@ export default [
 
             $scope.isPrepareDownloading = true;
 
+            ActivitiesData.post({ action: '/configuration/download' });
+
             return new SummaryZipper({ cluster, data: ctrl.data || {}, IgniteDemoMode: $root.IgniteDemoMode })
                 .then((data) => {
                     saver.saveAs(data, escapeFileName(cluster.name) + '-project.zip');
diff --git a/modules/web-console/frontend/app/modules/user/AclRoute.provider.js b/modules/web-console/frontend/app/modules/user/AclRoute.provider.js
index 40abea5ad872e..4225bc4ed3721 100644
--- a/modules/web-console/frontend/app/modules/user/AclRoute.provider.js
+++ b/modules/web-console/frontend/app/modules/user/AclRoute.provider.js
@@ -17,31 +17,36 @@
 
 export default [() => {
     class AclRoute {
-        static checkAccess = (permissions, failState) => {
+        static checkAccess(permissions, failState) {
             failState = failState || '403';
 
-            return ['$state', 'AclService', 'User', ($state, AclService, User) => {
-                User.read()
-                    .then(() => {
-                        if (AclService.can(permissions))
-                            return;
+            return ['$q', '$state', 'AclService', 'User', 'IgniteActivitiesData', function($q, $state, AclService, User, Activities) {
+                const action = this.name ? $state.href(this.name) : null;
 
-                        return $state.go(failState);
-                    })
+                return User.read()
                     .catch(() => {
                         User.clean();
 
                         if ($state.current.name !== 'signin')
                             $state.go('signin');
+
+                        return $q.reject('Failed to detect user');
+                    })
+                    .then(() => {
+                        if (AclService.can(permissions))
+                            return Activities.post({ action });
+
+                        $state.go(failState);
+
+                        return $q.reject('User are not authorized');
                     });
             }];
         }
-    }
 
-    return {
-        checkAccess: AclRoute.checkAccess,
-        $get: () => {
+        static $get() {
             return AclRoute;
         }
-    };
+    }
+
+    return AclRoute;
 }];
diff --git a/modules/web-console/frontend/app/modules/user/Auth.service.js b/modules/web-console/frontend/app/modules/user/Auth.service.js
index e0f905da8bf8b..95ff4c3e6dc4c 100644
--- a/modules/web-console/frontend/app/modules/user/Auth.service.js
+++ b/modules/web-console/frontend/app/modules/user/Auth.service.js
@@ -21,7 +21,7 @@ export default ['Auth', ['$http', '$rootScope', '$state', '$window', 'IgniteErro
             forgotPassword(userInfo) {
                 $http.post('/api/v1/password/forgot', userInfo)
                     .then(() => $state.go('password.send'))
-                    .cacth(({data}) => ErrorPopover.show('forgot_email', Messages.errorMessage(null, data)));
+                    .catch(({data}) => ErrorPopover.show('forgot_email', Messages.errorMessage(null, data)));
             },
             auth(action, userInfo) {
                 $http.post('/api/v1/' + action, userInfo)
diff --git a/modules/web-console/frontend/app/modules/user/permissions.js b/modules/web-console/frontend/app/modules/user/permissions.js
index e13509ceaf719..b6f7c3a30cf42 100644
--- a/modules/web-console/frontend/app/modules/user/permissions.js
+++ b/modules/web-console/frontend/app/modules/user/permissions.js
@@ -16,7 +16,7 @@
  */
 
 const guest = ['login'];
-const becomed = ['profile', 'configuration', 'query'];
+const becomed = ['profile', 'configuration', 'query', 'demo'];
 const user = becomed.concat(['logout']);
 const admin = user.concat(['admin_page']);
 
diff --git a/modules/web-console/frontend/app/modules/user/user.module.js b/modules/web-console/frontend/app/modules/user/user.module.js
index 11798d0746d62..b86a62e492066 100644
--- a/modules/web-console/frontend/app/modules/user/user.module.js
+++ b/modules/web-console/frontend/app/modules/user/user.module.js
@@ -22,10 +22,10 @@ import Auth from './Auth.service';
 import User from './User.service';
 import AclRouteProvider from './AclRoute.provider';
 
-angular
-.module('ignite-console.user', [
+angular.module('ignite-console.user', [
     'mm.acl',
-    'ignite-console.config'
+    'ignite-console.config',
+    'ignite-console.core'
 ])
 .factory('sessionRecoverer', ['$injector', '$q', ($injector, $q) => {
     return {
diff --git a/modules/web-console/frontend/app/vendor.js b/modules/web-console/frontend/app/vendor.js
index a9e88442cf0a3..3bbb3224ce3b2 100644
--- a/modules/web-console/frontend/app/vendor.js
+++ b/modules/web-console/frontend/app/vendor.js
@@ -25,6 +25,7 @@ import 'angular-strap/dist/angular-strap.tpl';
 import 'angular-socket-io';
 import 'angular-retina';
 import 'angular-ui-router';
+import 'angular-translate';
 import 'ui-router-metatags/dist/ui-router-metatags';
 import 'angular-smart-table';
 import 'angular-ui-grid/ui-grid';
diff --git a/modules/web-console/frontend/controllers/admin-controller.js b/modules/web-console/frontend/controllers/admin-controller.js
deleted file mode 100644
index cf7fd71adeec0..0000000000000
--- a/modules/web-console/frontend/controllers/admin-controller.js
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const ICON_SORT = '<span ui-grid-one-bind-id-grid="col.uid + \'-sortdir-text\'" ui-grid-visible="col.sort.direction" aria-label="Sort Descending"><i ng-class="{ \'ui-grid-icon-up-dir\': col.sort.direction == asc, \'ui-grid-icon-down-dir\': col.sort.direction == desc, \'ui-grid-icon-blank\': !col.sort.direction }" title="" aria-hidden="true"></i></span>';
-
-const CLUSTER_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-sitemap'></i>${ICON_SORT}</div>`;
-const MODEL_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-object-group'></i>${ICON_SORT}</div>`;
-const CACHE_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-database'></i>${ICON_SORT}</div>`;
-const IGFS_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-folder-o'></i>${ICON_SORT}</div>`;
-
-const ACTIONS_TEMPLATE = `
-<div class='text-center ui-grid-cell-actions'>
-    <a class='btn btn-default dropdown-toggle' bs-dropdown='' ng-show='row.entity._id != $root.user._id' data-placement='bottom-right' data-container='.panel'>
-        <i class='fa fa-gear'></i>&nbsp;
-        <span class='caret'></span>
-    </a>
-    <ul class='dropdown-menu' role='menu'>
-        <li>
-            <a ng-click='grid.api.becomeUser(row.entity)'>Become this user</a>
-        </li>
-        <li>
-            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='row.entity.admin && row.entity._id !== $root.user._id'>Revoke admin</a>
-            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='!row.entity.admin && row.entity._id !== $root.user._id'>Grant admin</a>
-        </li>
-        <li>
-            <a ng-click='grid.api.removeUser(row.entity)'>Remove user</a>
-        </li>
-</div>`;
-
-const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
-
-// Controller for Admin screen.
-export default ['adminController', [
-    '$rootScope', '$scope', '$http', '$q', '$state', '$filter', 'uiGridConstants', 'IgniteMessages', 'IgniteConfirm', 'User', 'IgniteNotebookData', 'IgniteCountries',
-    ($rootScope, $scope, $http, $q, $state, $filter, uiGridConstants, Messages, Confirm, User, Notebook, Countries) => {
-        $scope.users = null;
-
-        const companySelectOptions = [];
-        const countrySelectOptions = [];
-
-        const COLUMNS_DEFS = [
-            {displayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'test', minWidth: 80, width: 80, enableFiltering: false, enableSorting: false},
-            {displayName: 'User', field: 'userName', minWidth: 65, enableFiltering: true, filter: { placeholder: 'Filter by name...' }},
-            {displayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
-            {displayName: 'Company', field: 'company', minWidth: 160, filter: {
-                selectOptions: companySelectOptions, type: uiGridConstants.filter.SELECT, condition: uiGridConstants.filter.EXACT }
-            },
-            {displayName: 'Country', field: 'countryCode', minWidth: 80, filter: {
-                selectOptions: countrySelectOptions, type: uiGridConstants.filter.SELECT, condition: uiGridConstants.filter.EXACT }
-            },
-            {displayName: 'Last login', field: 'lastLogin', cellFilter: 'date:"medium"', minWidth: 175, width: 175, enableFiltering: false, sort: { direction: 'desc', priority: 0 }},
-            {displayName: 'Clusters count', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: '_clusters', type: 'number', headerTooltip: 'Clusters count', minWidth: 50, width: 50, enableFiltering: false},
-            {displayName: 'Models count', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: '_models', type: 'number', headerTooltip: 'Models count', minWidth: 50, width: 50, enableFiltering: false},
-            {displayName: 'Caches count', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: '_caches', type: 'number', headerTooltip: 'Caches count', minWidth: 50, width: 50, enableFiltering: false},
-            {displayName: 'IGFS count', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: '_igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 50, width: 50, enableFiltering: false}
-        ];
-
-        const ctrl = $scope.ctrl = {};
-
-        const becomeUser = function(user) {
-            $http.get('/api/v1/admin/become', { params: {viewedUserId: user._id}})
-                .then(() => User.load())
-                .then(() => $state.go('base.configuration.clusters'))
-                .then(() => Notebook.load())
-                .catch(Messages.showError);
-        };
-
-        const removeUser = (user) => {
-            Confirm.confirm(`Are you sure you want to remove user: "${user.userName}"?`)
-                .then(() => {
-                    $http.post('/api/v1/admin/remove', {userId: user._id})
-                        .then(() => {
-                            const i = _.findIndex($scope.users, (u) => u._id === user._id);
-
-                            if (i >= 0)
-                                $scope.users.splice(i, 1);
-
-                            Messages.showInfo(`User has been removed: "${user.userName}"`);
-                        })
-                        .catch(({data, status}) => {
-                            if (status === 503)
-                                Messages.showInfo(data);
-                            else
-                                Messages.showError('Failed to remove user: ', data);
-                        });
-                });
-        };
-
-        const toggleAdmin = (user) => {
-            if (user.adminChanging)
-                return;
-
-            user.adminChanging = true;
-
-            $http.post('/api/v1/admin/save', {userId: user._id, adminFlag: !user.admin})
-                .then(() => {
-                    user.admin = !user.admin;
-
-                    Messages.showInfo(`Admin right was successfully toggled for user: "${user.userName}"`);
-                })
-                .catch((res) => {
-                    Messages.showError('Failed to toggle admin right for user: ', res);
-                })
-                .finally(() => user.adminChanging = false);
-        };
-
-
-        ctrl.gridOptions = {
-            data: [],
-            columnVirtualizationThreshold: 30,
-            columnDefs: COLUMNS_DEFS,
-            categories: [
-                {name: 'Actions', visible: true, selectable: true},
-                {name: 'User', visible: true, selectable: true},
-                {name: 'Email', visible: true, selectable: true},
-                {name: 'Company', visible: true, selectable: true},
-                {name: 'Country', visible: true, selectable: true},
-                {name: 'Last login', visible: true, selectable: true},
-
-                {name: 'Clusters count', visible: true, selectable: true},
-                {name: 'Models count', visible: true, selectable: true},
-                {name: 'Caches count', visible: true, selectable: true},
-                {name: 'IGFS count', visible: true, selectable: true}
-            ],
-            enableFiltering: true,
-            enableRowSelection: false,
-            enableRowHeaderSelection: false,
-            enableColumnMenus: false,
-            multiSelect: false,
-            modifierKeysToMultiSelect: true,
-            noUnselect: true,
-            flatEntityAccess: true,
-            fastWatch: true,
-            onRegisterApi: (api) => {
-                ctrl.gridApi = api;
-
-                api.becomeUser = becomeUser;
-                api.removeUser = removeUser;
-                api.toggleAdmin = toggleAdmin;
-            }
-        };
-
-        /**
-         * Set grid height.
-         *
-         * @param {Number} rows Rows count.
-         * @private
-         */
-        const adjustHeight = (rows) => {
-            const height = Math.min(rows, 20) * 30 + 75;
-
-            // Remove header height.
-            ctrl.gridApi.grid.element.css('height', height + 'px');
-
-            ctrl.gridApi.core.handleWindowResize();
-        };
-
-        const usersToFilterOptions = (column) => {
-            return _.sortBy(
-                _.map(
-                    _.groupBy($scope.users, (usr) => {
-                        const fld = usr[column];
-
-                        return _.isNil(fld) ? fld : fld.toUpperCase();
-                    }),
-                    (arr, value) => ({label: `${_.head(arr)[column] || 'Not set'} (${arr.length})`, value})
-                ),
-                'value');
-        };
-
-        const _reloadUsers = () => {
-            $http.post('/api/v1/admin/list')
-                .then(({ data }) => {
-                    $scope.users = data;
-
-                    companySelectOptions.length = 0;
-                    countrySelectOptions.length = 0;
-
-                    _.forEach($scope.users, (user) => {
-                        user.userName = user.firstName + ' ' + user.lastName;
-                        user.countryCode = Countries.getByName(user.country).code;
-
-                        user._clusters = user.counters.clusters;
-                        user._models = user.counters.models;
-                        user._caches = user.counters.caches;
-                        user._igfs = user.counters.igfs;
-                    });
-
-                    companySelectOptions.push(...usersToFilterOptions('company'));
-                    countrySelectOptions.push(...usersToFilterOptions('countryCode'));
-
-                    $scope.ctrl.gridOptions.data = data;
-
-                    adjustHeight(data.length);
-                })
-                .catch(Messages.showError);
-        };
-
-        _reloadUsers();
-
-        const _enableColumns = (categories, visible) => {
-            _.forEach(categories, (cat) => {
-                cat.visible = visible;
-
-                _.forEach(ctrl.gridOptions.columnDefs, (col) => {
-                    if (col.displayName === cat.name)
-                        col.visible = visible;
-                });
-            });
-
-            ctrl.gridApi.grid.refresh();
-        };
-
-        const _selectableColumns = () => _.filter(ctrl.gridOptions.categories, (cat) => cat.selectable);
-
-        ctrl.toggleColumns = (category, visible) => _enableColumns([category], visible);
-        ctrl.selectAllColumns = () => _enableColumns(_selectableColumns(), true);
-        ctrl.clearAllColumns = () => _enableColumns(_selectableColumns(), false);
-    }
-]];
diff --git a/modules/web-console/frontend/controllers/domains-controller.js b/modules/web-console/frontend/controllers/domains-controller.js
index 303110e086696..bfffe92f62747 100644
--- a/modules/web-console/frontend/controllers/domains-controller.js
+++ b/modules/web-console/frontend/controllers/domains-controller.js
@@ -17,8 +17,8 @@
 
 // Controller for Domain model screen.
 export default ['domainsController', [
-    '$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteAgentMonitor', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes',
-    function($root, $scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Focus, Confirm, ConfirmBatch, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, IgniteAgentMonitor, LegacyTable, Resource, ErrorPopover, FormUtils, JavaTypes, SqlTypes) {
+    '$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteAgentMonitor', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes', 'IgniteActivitiesData',
+    function($root, $scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Focus, Confirm, ConfirmBatch, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, IgniteAgentMonitor, LegacyTable, Resource, ErrorPopover, FormUtils, JavaTypes, SqlTypes, ActivitiesData) {
         UnsavedChangesGuard.install($scope);
 
         const emptyDomain = {empty: true};
@@ -460,6 +460,14 @@ export default ['domainsController', [
                 $scope.importDomain.loadingOptions = LOADING_JDBC_DRIVERS;
 
                 IgniteAgentMonitor.startWatch({text: 'Back to Domain models', goal: 'import domain model from database'})
+                    .then(() => {
+                        ActivitiesData.post({
+                            group: 'configuration',
+                            action: 'configuration/import/model'
+                        });
+
+                        return true;
+                    })
                     .then(importDomainModal.$promise)
                     .then(importDomainModal.show)
                     .then(() => {
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index fd50d5b6e7552..ff52ba44cf9a2 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -44,6 +44,7 @@
     "angular-socket-io": "~0.7.0",
     "angular-strap": "~2.3.8",
     "angular-touch": "~1.5.9",
+    "angular-translate": "~2.13.1",
     "angular-tree-control": "~0.2.26",
     "angular-ui-grid": "~3.2.9",
     "angular-ui-router": "~0.3.1",
diff --git a/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss b/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss
index bfa6c6ce78aad..47555a7c95750 100644
--- a/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss
+++ b/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss
@@ -69,3 +69,31 @@ $fa-font-path: '~font-awesome/fonts';
 
   cursor: default;
 }
+
+.icon-user {
+  @extend .fa;
+  @extend .fa-user-o;
+
+  cursor: default;
+}
+
+.icon-admin {
+  @extend .fa;
+  @extend .fa-user-secret;
+
+  cursor: default;
+}
+
+.icon-datepicker-left {
+  @extend .fa;
+  @extend .fa-chevron-left;
+
+  margin: 0;
+}
+
+.icon-datepicker-right {
+  @extend .fa;
+  @extend .fa-chevron-right;
+  
+  margin: 0;
+}
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index 4318fc2abc5eb..67cfed176d39e 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -2302,12 +2302,13 @@ html,body,.splash-screen {
         cursor: default;
 
         i {
+            margin-top: 2px;
             margin-right: 10px;
         }
 
         label {
             cursor: default;
-            line-height: 24px;
+            line-height: 28px;
         }
 
         sub {
@@ -2326,4 +2327,40 @@ html,body,.splash-screen {
     .ui-grid-filter-select {
         width: calc(100% - 10px);
     }
+
+    .ui-grid-cell-contents > i {
+        line-height: $line-height-base;
+    }
+
+    .ui-grid-row:nth-child(odd):hover .ui-grid-cell {
+        background: $ignite-row-hover;
+    }
+    
+    .ui-grid-row:nth-child(even):hover .ui-grid-cell {
+        background: $ignite-row-hover;
+    }
 }
+
+.datepicker.dropdown-menu {
+    width: 250px;
+    height: 270px;
+
+    button {
+        outline: none;
+        border: 0;
+    }
+
+    tbody {
+        height: 180px;
+    }
+
+    tbody button {
+        padding: 6px;
+    }
+
+    &.datepicker-mode-1, &.datepicker-mode-2 {
+        tbody button {
+            height: 65px;
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/public/stylesheets/variables.scss b/modules/web-console/frontend/public/stylesheets/variables.scss
index 8500eace44e41..e30bbddf9a6e4 100644
--- a/modules/web-console/frontend/public/stylesheets/variables.scss
+++ b/modules/web-console/frontend/public/stylesheets/variables.scss
@@ -26,3 +26,4 @@ $ignite-border-bottom-color: $brand-primary;
 $ignite-background-color: #fff;
 $ignite-header-color: #555;
 $ignite-invalid-color: $brand-primary;
+$ignite-row-hover: #c9dde1;
diff --git a/modules/web-console/frontend/views/settings/admin.jade b/modules/web-console/frontend/views/settings/admin.jade
index c9858269eeaf3..a09fda95a9eb1 100644
--- a/modules/web-console/frontend/views/settings/admin.jade
+++ b/modules/web-console/frontend/views/settings/admin.jade
@@ -14,38 +14,12 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin grid-settings()
-    i.fa.fa-bars(data-animation='am-flip-x' bs-dropdown='' aria-haspopup='true' aria-expanded='expanded' data-auto-close='1' data-trigger='click')
-    ul.select.dropdown-menu(role='menu')
-        li(ng-repeat='item in ctrl.gridOptions.categories|filter:{selectable:true}')
-            a(ng-click='ctrl.toggleColumns(item, !item.visible)')
-                i.fa.fa-check-square-o.pull-left(ng-if='item.visible')
-                i.fa.fa-square-o.pull-left(ng-if='!item.visible')
-                span {{::item.name}}
-        li.divider
-        li
-            a(ng-click='ctrl.selectAllColumns()') Select all
-        li
-            a(ng-click='ctrl.clearAllColumns()') Clear all
-        li.divider
-        li
-            a(ng-click='$hide()') Close
-
-.admin-page.row(ng-controller='adminController')
+.admin-page.row
     .docs-content.greedy
         .docs-header
-            h1 List of registered users
+            h1 Admin panel
             hr
         .docs-body
             .row
                 .col-xs-12
-                    .panel.panel-default
-                        .panel-heading.ui-grid-settings
-                            +grid-settings
-                            label Total users: 
-                                strong {{ users.length }}&nbsp;&nbsp;&nbsp;
-                            label Showing users: 
-                                strong {{ ctrl.gridApi.grid.getVisibleRows().length }}
-                                sub(ng-show='users.length === ctrl.gridApi.grid.getVisibleRows().length') all
-                        .panel-collapse
-                            .grid(ui-grid='ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-pinning)
+                    ignite-list-of-registered-users(data-options='ctrl.data')
diff --git a/modules/web-console/frontend/views/sql/sql.jade b/modules/web-console/frontend/views/sql/sql.jade
index 03015e8aad612..61d5b3037825a 100644
--- a/modules/web-console/frontend/views/sql/sql.jade
+++ b/modules/web-console/frontend/views/sql/sql.jade
@@ -15,7 +15,7 @@
     limitations under the License.
 
 include /app/helpers/jade/mixins.jade
-include /app/directives/ui-grid-settings/ui-grid-settings.jade
+include /app/components/ui-grid-settings/ui-grid-settings.jade
 
 mixin btn-toolbar(btn, click, tip, focusId)
     i.btn.btn-default.fa(class=btn ng-click=click bs-tooltip='' data-title=tip ignite-on-click-focus=focusId data-trigger='hover' data-placement='bottom')
@@ -195,7 +195,7 @@ mixin paragraph-scan
                 +table-result-body
             .footer.clearfix()
                 .pull-left
-                    | Showing results for scan of #[b{{ paragraph.queryArgs.cacheName | defaultName }}]
+                    | Showing results for scan of #[b {{ paragraph.queryArgs.cacheName | defaultName }}]
                     span(ng-if='paragraph.queryArgs.filter') &nbsp; with filter: #[b {{ paragraph.queryArgs.filter }}]
                     span(ng-if='paragraph.queryArgs.localNid') &nbsp; on node: #[b {{ paragraph.queryArgs.localNid | limitTo:8 }}]
 

From b571fd40f009709718bce0c969d47b62be06b8f6 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 8 Feb 2017 17:05:08 +0700
Subject: [PATCH 147/446] IGNITE-4472 Added user activities in Web Console.

(cherry picked from commit 26ee9c2)
---
 .../frontend/app/modules/{Demo => demo}/Demo.module.js            | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename modules/web-console/frontend/app/modules/{Demo => demo}/Demo.module.js (100%)

diff --git a/modules/web-console/frontend/app/modules/Demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js
similarity index 100%
rename from modules/web-console/frontend/app/modules/Demo/Demo.module.js
rename to modules/web-console/frontend/app/modules/demo/Demo.module.js

From 7f270405e3ab94938a9e46e301069825c7bc6fb0 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Mon, 6 Feb 2017 18:07:28 +0300
Subject: [PATCH 148/446] .NET: Extract exceptions tests in CacheStoreTest and
 ignore due to IGNITE-4657

---
 .../Cache/Store/CacheStoreTest.cs             | 30 +++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs
index d39ccde391a96..2a235aaef6dd1 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs
@@ -296,12 +296,38 @@ public void TestPutLoad()
             Assert.AreEqual("val", cache.Get(1));
 
             Assert.AreEqual(1, cache.GetSize());
+        }
+
+        [Test]
+        public void TestExceptions()
+        {
+            var cache = GetCache();
+
+            cache.Put(1, "val");
 
-            // Test errors
             CacheTestStore.ThrowError = true;
             CheckCustomStoreError(Assert.Throws<CacheStoreException>(() => cache.Put(-2, "fail")).InnerException);
 
-            cache.LocalEvict(new[] { 1 });
+            cache.LocalEvict(new[] {1});
+            CheckCustomStoreError(Assert.Throws<CacheStoreException>(() => cache.Get(1)).InnerException);
+
+            CacheTestStore.ThrowError = false;
+
+            cache.Remove(1);
+        }
+
+        [Test]
+        [Ignore("IGNITE-4657")]
+        public void TestExceptionsNoRemove()
+        {
+            var cache = GetCache();
+
+            cache.Put(1, "val");
+
+            CacheTestStore.ThrowError = true;
+            CheckCustomStoreError(Assert.Throws<CacheStoreException>(() => cache.Put(-2, "fail")).InnerException);
+
+            cache.LocalEvict(new[] {1});
             CheckCustomStoreError(Assert.Throws<CacheStoreException>(() => cache.Get(1)).InnerException);
         }
 

From db48f545a83bb72cded77d8269fe4f8820e8380f Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 10 Feb 2017 15:55:05 +0700
Subject: [PATCH 149/446] IGNITE-4678 Web Console: Implemented demo load as
 service.

(cherry picked from commit a600caf)
---
 modules/web-console/backend/app/agent.js      |  34 ++
 modules/web-console/backend/app/browser.js    |  26 +
 modules/web-console/frontend/package.json     |   2 +-
 .../ignite/console/demo/AgentClusterDemo.java | 475 +-----------------
 .../ignite/console/demo/AgentDemoUtils.java   |  79 +++
 .../demo/service/DemoCachesLoadService.java   | 456 +++++++++++++++++
 .../service/DemoRandomCacheLoadService.java   | 120 +++++
 .../service/DemoServiceClusterSingleton.java  |  41 ++
 .../demo/service/DemoServiceKeyAffinity.java  |  41 ++
 .../service/DemoServiceMultipleInstances.java |  41 ++
 .../service/DemoServiceNodeSingleton.java     |  41 ++
 11 files changed, 897 insertions(+), 459 deletions(-)
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentDemoUtils.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceClusterSingleton.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceKeyAffinity.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceMultipleInstances.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceNodeSingleton.java

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index 817028059f50f..4cae8ee124175 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -581,6 +581,40 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo,
 
             return this.executeRest(cmd);
         }
+
+        /**
+         * Collect service information.
+         * @param {Boolean} demo Is need run command on demo node.
+         * @param {String} nid Node ID.
+         * @returns {Promise}
+         */
+        services(demo, nid) {
+            const cmd = new Command(demo, 'exe')
+                .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
+                .addParam('p1', nid)
+                .addParam('p2', 'org.apache.ignite.internal.visor.service.VisorServiceTask')
+                .addParam('p3', 'java.lang.Void');
+
+            return this.executeRest(cmd);
+        }
+
+        /**
+         * Cancel service with specified name.
+         * @param {Boolean} demo Is need run command on demo node.
+         * @param {String} nid Node ID.
+         * @param {String} name Name of service to cancel.
+         * @returns {Promise}
+         */
+        serviceCancel(demo, nid, name) {
+            const cmd = new Command(demo, 'exe')
+                .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
+                .addParam('p1', nid)
+                .addParam('p2', 'org.apache.ignite.internal.visor.service.VisorCancelServiceTask')
+                .addParam('p3', 'java.lang.String')
+                .addParam('p4', name);
+
+            return this.executeRest(cmd);
+        }
     }
 
     /**
diff --git a/modules/web-console/backend/app/browser.js b/modules/web-console/backend/app/browser.js
index 499d84d8e990a..2b1285e4f79c1 100644
--- a/modules/web-console/backend/app/browser.js
+++ b/modules/web-console/backend/app/browser.js
@@ -455,6 +455,32 @@ module.exports.factory = (_, socketio, agentMgr, configure) => {
                         .catch((err) => cb(_errorToJson(err)));
                 });
 
+                // Collect service information from grid.
+                socket.on('service:collect', (nid, cb) => {
+                    agentMgr.findAgent(accountId())
+                        .then((agent) => agent.services(demo, nid))
+                        .then((data) => {
+                            if (data.finished)
+                                return cb(null, data.result);
+
+                            cb(_errorToJson(data.error));
+                        })
+                        .catch((err) => cb(_errorToJson(err)));
+                });
+
+                // Collect service information from grid.
+                socket.on('service:cancel', (nid, name, cb) => {
+                    agentMgr.findAgent(accountId())
+                        .then((agent) => agent.serviceCancel(demo, nid, name))
+                        .then((data) => {
+                            if (data.finished)
+                                return cb(null, data.result);
+
+                            cb(_errorToJson(data.error));
+                        })
+                        .catch((err) => cb(_errorToJson(err)));
+                });
+
                 const count = agentMgr.addAgentListener(user._id, socket);
 
                 socket.emit('agent:count', {count});
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index ff52ba44cf9a2..651f496fa78c7 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -46,7 +46,7 @@
     "angular-touch": "~1.5.9",
     "angular-translate": "~2.13.1",
     "angular-tree-control": "~0.2.26",
-    "angular-ui-grid": "~3.2.9",
+    "angular-ui-grid": "~4.0.0",
     "angular-ui-router": "~0.3.1",
     "bootstrap-sass": "~3.3.6",
     "brace": "~0.8.0",
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
index 489e762eb5362..252692eef57fa 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
@@ -17,37 +17,24 @@
 
 package org.apache.ignite.console.demo;
 
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.Random;
-import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteServices;
 import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.QueryEntity;
-import org.apache.ignite.cache.QueryIndex;
-import org.apache.ignite.cache.QueryIndexType;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.console.agent.AgentConfiguration;
-import org.apache.ignite.console.demo.model.Car;
-import org.apache.ignite.console.demo.model.Country;
-import org.apache.ignite.console.demo.model.Department;
-import org.apache.ignite.console.demo.model.Employee;
-import org.apache.ignite.console.demo.model.Parking;
+import org.apache.ignite.console.demo.service.DemoCachesLoadService;
+import org.apache.ignite.console.demo.service.DemoRandomCacheLoadService;
+import org.apache.ignite.console.demo.service.DemoServiceMultipleInstances;
+import org.apache.ignite.console.demo.service.DemoServiceClusterSingleton;
+import org.apache.ignite.console.demo.service.DemoServiceKeyAffinity;
+import org.apache.ignite.console.demo.service.DemoServiceNodeSingleton;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.logger.log4j.Log4JLogger;
@@ -55,7 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.spi.swapspace.file.FileSwapSpaceSpi;
-import org.apache.ignite.transactions.Transaction;
 import org.apache.log4j.Logger;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE;
@@ -66,8 +52,6 @@
 import static org.apache.ignite.events.EventType.EVTS_DISCOVERY;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_JETTY_ADDRS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_JETTY_PORT;
-import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
-import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
 /**
  * Demo for cluster features like SQL and Monitoring.
@@ -84,247 +68,6 @@ public class AgentClusterDemo {
     /** */
     private static final int NODE_CNT = 3;
 
-    /** */
-    private static final String COUNTRY_CACHE_NAME = "CountryCache";
-
-    /** */
-    private static final String DEPARTMENT_CACHE_NAME = "DepartmentCache";
-
-    /** */
-    private static final String EMPLOYEE_CACHE_NAME = "EmployeeCache";
-
-    /** */
-    private static final String PARKING_CACHE_NAME = "ParkingCache";
-
-    /** */
-    private static final String CAR_CACHE_NAME = "CarCache";
-
-    /** */
-    private static final Set<String> DEMO_CACHES = new HashSet<>(Arrays.asList(COUNTRY_CACHE_NAME,
-        DEPARTMENT_CACHE_NAME, EMPLOYEE_CACHE_NAME, PARKING_CACHE_NAME, CAR_CACHE_NAME));
-
-    /** */
-    private static final Random rnd = new Random();
-
-    /** Countries count. */
-    private static final int CNTR_CNT = 10;
-
-    /** Departments count */
-    private static final int DEP_CNT = 100;
-
-    /** Employees count. */
-    private static final int EMPL_CNT = 1000;
-
-    /** Countries count. */
-    private static final int CAR_CNT = 100;
-
-    /** Departments count */
-    private static final int PARK_CNT = 10;
-
-    /** Counter for threads in pool. */
-    private static final AtomicInteger THREAD_CNT = new AtomicInteger(0);
-
-    /**
-     * Create base cache configuration.
-     *
-     * @param name cache name.
-     * @return Cache configuration with basic properties set.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheConfiguration(String name) {
-        CacheConfiguration<K, V> ccfg = new CacheConfiguration<>(name);
-
-        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-        ccfg.setQueryDetailMetricsSize(10);
-        ccfg.setStartSize(100);
-        ccfg.setStatisticsEnabled(true);
-
-        return ccfg;
-    }
-
-    /**
-     * Configure cacheCountry.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheCountry() {
-        CacheConfiguration<K, V> ccfg = cacheConfiguration(COUNTRY_CACHE_NAME);
-
-        // Configure cacheCountry types.
-        Collection<QueryEntity> qryEntities = new ArrayList<>();
-
-        // COUNTRY.
-        QueryEntity type = new QueryEntity();
-
-        qryEntities.add(type);
-
-        type.setKeyType(Integer.class.getName());
-        type.setValueType(Country.class.getName());
-
-        // Query fields for COUNTRY.
-        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
-
-        qryFlds.put("id", "java.lang.Integer");
-        qryFlds.put("name", "java.lang.String");
-        qryFlds.put("population", "java.lang.Integer");
-
-        type.setFields(qryFlds);
-
-        ccfg.setQueryEntities(qryEntities);
-
-        return ccfg;
-    }
-
-    /**
-     * Configure cacheEmployee.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheDepartment() {
-        CacheConfiguration<K, V> ccfg = cacheConfiguration(DEPARTMENT_CACHE_NAME);
-
-        // Configure cacheDepartment types.
-        Collection<QueryEntity> qryEntities = new ArrayList<>();
-
-        // DEPARTMENT.
-        QueryEntity type = new QueryEntity();
-
-        qryEntities.add(type);
-
-        type.setKeyType(Integer.class.getName());
-        type.setValueType(Department.class.getName());
-
-        // Query fields for DEPARTMENT.
-        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
-
-        qryFlds.put("id", "java.lang.Integer");
-        qryFlds.put("countryId", "java.lang.Integer");
-        qryFlds.put("name", "java.lang.String");
-
-        type.setFields(qryFlds);
-
-        ccfg.setQueryEntities(qryEntities);
-
-        return ccfg;
-    }
-
-    /**
-     * Configure cacheEmployee.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheEmployee() {
-        CacheConfiguration<K, V> ccfg = cacheConfiguration(EMPLOYEE_CACHE_NAME);
-
-        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
-        ccfg.setBackups(1);
-
-        // Configure cacheEmployee types.
-        Collection<QueryEntity> qryEntities = new ArrayList<>();
-
-        // EMPLOYEE.
-        QueryEntity type = new QueryEntity();
-
-        qryEntities.add(type);
-
-        type.setKeyType(Integer.class.getName());
-        type.setValueType(Employee.class.getName());
-
-        // Query fields for EMPLOYEE.
-        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
-
-        qryFlds.put("id", "java.lang.Integer");
-        qryFlds.put("departmentId", "java.lang.Integer");
-        qryFlds.put("managerId", "java.lang.Integer");
-        qryFlds.put("firstName", "java.lang.String");
-        qryFlds.put("lastName", "java.lang.String");
-        qryFlds.put("email", "java.lang.String");
-        qryFlds.put("phoneNumber", "java.lang.String");
-        qryFlds.put("hireDate", "java.sql.Date");
-        qryFlds.put("job", "java.lang.String");
-        qryFlds.put("salary", "java.lang.Double");
-
-        type.setFields(qryFlds);
-
-        // Indexes for EMPLOYEE.
-        Collection<QueryIndex> indexes = new ArrayList<>();
-
-        QueryIndex idx = new QueryIndex();
-
-        idx.setName("EMP_NAMES");
-        idx.setIndexType(QueryIndexType.SORTED);
-        LinkedHashMap<String, Boolean> indFlds = new LinkedHashMap<>();
-
-        indFlds.put("firstName", Boolean.FALSE);
-        indFlds.put("lastName", Boolean.FALSE);
-
-        idx.setFields(indFlds);
-
-        indexes.add(idx);
-        indexes.add(new QueryIndex("salary", QueryIndexType.SORTED, false, "EMP_SALARY"));
-
-        type.setIndexes(indexes);
-
-        ccfg.setQueryEntities(qryEntities);
-
-        return ccfg;
-    }
-
-    /**
-     * Configure cacheEmployee.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheParking() {
-        CacheConfiguration<K, V> ccfg = cacheConfiguration(PARKING_CACHE_NAME);
-
-        // Configure cacheParking types.
-        Collection<QueryEntity> qryEntities = new ArrayList<>();
-
-        // PARKING.
-        QueryEntity type = new QueryEntity();
-
-        qryEntities.add(type);
-
-        type.setKeyType(Integer.class.getName());
-        type.setValueType(Parking.class.getName());
-
-        // Query fields for PARKING.
-        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
-
-        qryFlds.put("id", "java.lang.Integer");
-        qryFlds.put("name", "java.lang.String");
-        qryFlds.put("capacity", "java.lang.Integer");
-
-        type.setFields(qryFlds);
-
-        ccfg.setQueryEntities(qryEntities);
-
-        return ccfg;
-    }
-
-    /**
-     * Configure cacheEmployee.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheCar() {
-        CacheConfiguration<K, V> ccfg = cacheConfiguration(CAR_CACHE_NAME);
-
-        // Configure cacheCar types.
-        Collection<QueryEntity> qryEntities = new ArrayList<>();
-
-        // CAR.
-        QueryEntity type = new QueryEntity();
-
-        qryEntities.add(type);
-
-        type.setKeyType(Integer.class.getName());
-        type.setValueType(Car.class.getName());
-
-        // Query fields for CAR.
-        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
-
-        qryFlds.put("id", "java.lang.Integer");
-        qryFlds.put("parkingId", "java.lang.Integer");
-        qryFlds.put("name", "java.lang.String");
-
-        type.setFields(qryFlds);
-
-        ccfg.setQueryEntities(qryEntities);
-
-        return ccfg;
-    }
-
     /**
      * Configure node.
      * @param gridIdx Grid name index.
@@ -363,120 +106,11 @@ private static  IgniteConfiguration igniteConfiguration(int gridIdx, boolean cli
         if (client)
             cfg.setClientMode(true);
 
-        cfg.setCacheConfiguration(cacheCountry(), cacheDepartment(), cacheEmployee(), cacheParking(), cacheCar());
-
         cfg.setSwapSpaceSpi(new FileSwapSpaceSpi());
 
         return cfg;
     }
 
-    /**
-     * @param val Value to round.
-     * @param places Numbers after point.
-     * @return Rounded value;
-     */
-    private static double round(double val, int places) {
-        if (places < 0)
-            throw new IllegalArgumentException();
-
-        long factor = (long)Math.pow(10, places);
-
-        val *= factor;
-
-        long tmp = Math.round(val);
-
-        return (double)tmp / factor;
-    }
-
-    /**
-     * @param ignite Ignite.
-     * @param range Time range in milliseconds.
-     */
-    private static void populateCacheEmployee(Ignite ignite, long range) {
-        if (log.isDebugEnabled())
-            log.debug("DEMO: Start employees population with data...");
-
-        IgniteCache<Integer, Country> cacheCountry = ignite.cache(COUNTRY_CACHE_NAME);
-
-        for (int i = 0, n = 1; i < CNTR_CNT; i++, n++)
-            cacheCountry.put(i, new Country(i, "Country #" + n, n * 10000000));
-
-        IgniteCache<Integer, Department> cacheDepartment = ignite.cache(DEPARTMENT_CACHE_NAME);
-
-        IgniteCache<Integer, Employee> cacheEmployee = ignite.cache(EMPLOYEE_CACHE_NAME);
-
-        for (int i = 0, n = 1; i < DEP_CNT; i++, n++) {
-            cacheDepartment.put(i, new Department(n, rnd.nextInt(CNTR_CNT), "Department #" + n));
-
-            double r = rnd.nextDouble();
-
-            cacheEmployee.put(i, new Employee(i, rnd.nextInt(DEP_CNT), null, "First name manager #" + n,
-                "Last name manager #" + n, "Email manager #" + n, "Phone number manager #" + n,
-                new java.sql.Date((long)(r * range)), "Job manager #" + n, 1000 + round(r * 4000, 2)));
-        }
-
-        for (int i = 0, n = 1; i < EMPL_CNT; i++, n++) {
-            Integer depId = rnd.nextInt(DEP_CNT);
-
-            double r = rnd.nextDouble();
-
-            cacheEmployee.put(i, new Employee(i, depId, depId, "First name employee #" + n,
-                "Last name employee #" + n, "Email employee #" + n, "Phone number employee #" + n,
-                new java.sql.Date((long)(r * range)), "Job employee #" + n, 500 + round(r * 2000, 2)));
-        }
-
-        if (log.isDebugEnabled())
-            log.debug("DEMO: Finished employees population.");
-    }
-
-    /**
-     * @param ignite Ignite.
-     */
-    private static void populateCacheCar(Ignite ignite) {
-        if (log.isDebugEnabled())
-            log.debug("DEMO: Start cars population...");
-
-        IgniteCache<Integer, Parking> cacheParking = ignite.cache(PARKING_CACHE_NAME);
-
-        for (int i = 0, n = 1; i < PARK_CNT; i++, n++)
-            cacheParking.put(i, new Parking(i, "Parking #" + n, n * 10));
-
-        IgniteCache<Integer, Car> cacheCar = ignite.cache(CAR_CACHE_NAME);
-
-        for (int i = 0, n = 1; i < CAR_CNT; i++, n++)
-            cacheCar.put(i, new Car(i, rnd.nextInt(PARK_CNT), "Car #" + n));
-
-        if (log.isDebugEnabled())
-            log.debug("DEMO: Finished cars population.");
-    }
-
-    /**
-     * Creates a thread pool that can schedule commands to run after a given delay, or to execute periodically.
-     *
-     * @param corePoolSize Number of threads to keep in the pool, even if they are idle.
-     * @param threadName Part of thread name that would be used by thread factory.
-     * @return Newly created scheduled thread pool.
-     */
-    private static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, final String threadName) {
-        ScheduledExecutorService srvc = Executors.newScheduledThreadPool(corePoolSize, new ThreadFactory() {
-            @Override public Thread newThread(Runnable r) {
-                Thread thread = new Thread(r, String.format("%s-%d", threadName, THREAD_CNT.getAndIncrement()));
-
-                thread.setDaemon(true);
-
-                return thread;
-            }
-        });
-
-        ScheduledThreadPoolExecutor executor = (ScheduledThreadPoolExecutor)srvc;
-
-        // Setting up shutdown policy.
-        executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
-        executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
-
-        return srvc;
-    }
-
     /**
      * Starts read and write from cache in background.
      *
@@ -484,91 +118,8 @@ private static ScheduledExecutorService newScheduledThreadPool(int corePoolSize,
      * @param cnt - maximum count read/write key
      */
     private static void startLoad(final Ignite ignite, final int cnt) {
-        final long diff = new java.util.Date().getTime();
-
-        populateCacheEmployee(ignite, diff);
-        populateCacheCar(ignite);
-
-        ScheduledExecutorService cachePool = newScheduledThreadPool(2, "demo-sql-load-cache-tasks");
-
-        cachePool.scheduleWithFixedDelay(new Runnable() {
-            @Override public void run() {
-                try {
-                    for (String cacheName : ignite.cacheNames()) {
-                        if (!DEMO_CACHES.contains(cacheName)) {
-                            IgniteCache<Integer, String> otherCache = ignite.cache(cacheName);
-
-                            if (otherCache != null) {
-                                for (int i = 0, n = 1; i < cnt; i++, n++) {
-                                    Integer key = rnd.nextInt(1000);
-
-                                    String val = otherCache.get(key);
-
-                                    if (val == null)
-                                        otherCache.put(key, "other-" + key);
-                                    else if (rnd.nextInt(100) < 30)
-                                        otherCache.remove(key);
-                                }
-                            }
-                        }
-                    }
-
-                    IgniteCache<Integer, Employee> cacheEmployee = ignite.cache(EMPLOYEE_CACHE_NAME);
-
-                    if (cacheEmployee != null)
-                        try(Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
-                            for (int i = 0, n = 1; i < cnt; i++, n++) {
-                                Integer id = rnd.nextInt(EMPL_CNT);
-
-                                Integer depId = rnd.nextInt(DEP_CNT);
-
-                                double r = rnd.nextDouble();
-
-                                cacheEmployee.put(id, new Employee(id, depId, depId, "First name employee #" + n,
-                                    "Last name employee #" + n, "Email employee #" + n, "Phone number employee #" + n,
-                                    new java.sql.Date((long)(r * diff)), "Job employee #" + n, 500 + round(r * 2000, 2)));
-
-                                if (rnd.nextBoolean())
-                                    cacheEmployee.remove(rnd.nextInt(EMPL_CNT));
-
-                                cacheEmployee.get(rnd.nextInt(EMPL_CNT));
-                            }
-
-                            if (rnd.nextInt(100) > 20)
-                                tx.commit();
-                        }
-                }
-                catch (Throwable e) {
-                    if (!e.getMessage().contains("cache is stopped"))
-                        ignite.log().error("Cache write task execution error", e);
-                }
-            }
-        }, 10, 3, TimeUnit.SECONDS);
-
-        cachePool.scheduleWithFixedDelay(new Runnable() {
-            @Override public void run() {
-                try {
-                    IgniteCache<Integer, Car> cache = ignite.cache(CAR_CACHE_NAME);
-
-                    if (cache != null)
-                        for (int i = 0; i < cnt; i++) {
-                            Integer carId = rnd.nextInt(CAR_CNT);
-
-                            cache.put(carId, new Car(carId, rnd.nextInt(PARK_CNT), "Car #" + (i + 1)));
-
-                            if (rnd.nextBoolean())
-                                cache.remove(rnd.nextInt(CAR_CNT));
-                        }
-                }
-                catch (IllegalStateException ignored) {
-                    // No-op.
-                }
-                catch (Throwable e) {
-                    if (!e.getMessage().contains("cache is stopped"))
-                        ignite.log().error("Cache write task execution error", e);
-                }
-            }
-        }, 10, 3, TimeUnit.SECONDS);
+        ignite.services().deployClusterSingleton("Demo caches load service", new DemoCachesLoadService(cnt));
+        ignite.services().deployNodeSingleton("RandomCache load service", new DemoRandomCacheLoadService(cnt));
     }
 
     /**
@@ -609,6 +160,14 @@ public static boolean testDrive(AgentConfiguration acfg) {
                     }
                 }, 10, 10, TimeUnit.SECONDS);
 
+                IgniteServices services = ignite.services();
+
+                services.deployMultiple("Demo service: Multiple instances", new DemoServiceMultipleInstances(), 7, 3);
+                services.deployNodeSingleton("Demo service: Node singleton", new DemoServiceNodeSingleton());
+                services.deployClusterSingleton("Demo service: Cluster singleton", new DemoServiceClusterSingleton());
+                services.deployKeyAffinitySingleton("Demo service: Key affinity singleton",
+                    new DemoServiceKeyAffinity(), DemoCachesLoadService.CAR_CACHE_NAME, "id");
+
                 if (log.isDebugEnabled())
                     log.debug("DEMO: Started embedded nodes with indexed enabled caches...");
 
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentDemoUtils.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentDemoUtils.java
new file mode 100644
index 0000000000000..fb34de7e8b41f
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentDemoUtils.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Utilites for Agent demo mode.
+ */
+public class AgentDemoUtils {
+    /** Counter for threads in pool. */
+    private static final AtomicInteger THREAD_CNT = new AtomicInteger(0);
+
+    /**
+     * Creates a thread pool that can schedule commands to run after a given delay, or to execute periodically.
+     *
+     * @param corePoolSize Number of threads to keep in the pool, even if they are idle.
+     * @param threadName Part of thread name that would be used by thread factory.
+     * @return Newly created scheduled thread pool.
+     */
+    public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, final String threadName) {
+        ScheduledExecutorService srvc = Executors.newScheduledThreadPool(corePoolSize, new ThreadFactory() {
+            @Override public Thread newThread(Runnable r) {
+                Thread thread = new Thread(r, String.format("%s-%d", threadName, THREAD_CNT.getAndIncrement()));
+
+                thread.setDaemon(true);
+
+                return thread;
+            }
+        });
+
+        ScheduledThreadPoolExecutor executor = (ScheduledThreadPoolExecutor)srvc;
+
+        // Setting up shutdown policy.
+        executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
+        executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
+
+        return srvc;
+    }
+
+    /**
+     * Round value.
+     *
+     * @param val Value to round.
+     * @param places Numbers after point.
+     * @return Rounded value;
+     */
+    public static double round(double val, int places) {
+        if (places < 0)
+            throw new IllegalArgumentException();
+
+        long factor = (long)Math.pow(10, places);
+
+        val *= factor;
+
+        long tmp = Math.round(val);
+
+        return (double)tmp / factor;
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
new file mode 100644
index 0000000000000..911764604502f
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
@@ -0,0 +1,456 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.cache.QueryIndexType;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.console.demo.AgentDemoUtils;
+import org.apache.ignite.console.demo.model.Car;
+import org.apache.ignite.console.demo.model.Country;
+import org.apache.ignite.console.demo.model.Department;
+import org.apache.ignite.console.demo.model.Employee;
+import org.apache.ignite.console.demo.model.Parking;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.transactions.Transaction;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Demo service. Create and populate caches. Run demo load on caches.
+ */
+public class DemoCachesLoadService implements Service {
+    /** Ignite instance. */
+    @IgniteInstanceResource
+    private Ignite ignite;
+
+    /** Thread pool to execute cache load operations. */
+    private ScheduledExecutorService cachePool;
+
+    /** */
+    private static final String COUNTRY_CACHE_NAME = "CountryCache";
+
+    /** */
+    private static final String DEPARTMENT_CACHE_NAME = "DepartmentCache";
+
+    /** */
+    private static final String EMPLOYEE_CACHE_NAME = "EmployeeCache";
+
+    /** */
+    private static final String PARKING_CACHE_NAME = "ParkingCache";
+
+    /** */
+    public static final String CAR_CACHE_NAME = "CarCache";
+
+    /** */
+    static final Set<String> DEMO_CACHES = new HashSet<>(Arrays.asList(COUNTRY_CACHE_NAME,
+        DEPARTMENT_CACHE_NAME, EMPLOYEE_CACHE_NAME, PARKING_CACHE_NAME, CAR_CACHE_NAME));
+
+    /** Countries count. */
+    private static final int CNTR_CNT = 10;
+
+    /** Departments count */
+    private static final int DEP_CNT = 100;
+
+    /** Employees count. */
+    private static final int EMPL_CNT = 1000;
+
+    /** Countries count. */
+    private static final int CAR_CNT = 100;
+
+    /** Departments count */
+    private static final int PARK_CNT = 10;
+
+    /** */
+    private static final Random rnd = new Random();
+
+    /** Maximum count read/write key. */
+    private final int cnt;
+
+    /** Time range in milliseconds. */
+    private final long range;
+
+    /**
+     * @param cnt Maximum count read/write key.
+     */
+    public DemoCachesLoadService(int cnt) {
+        this.cnt = cnt;
+
+        range = new java.util.Date().getTime();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        if (cachePool != null)
+            cachePool.shutdown();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        ignite.createCache(cacheCountry());
+        ignite.createCache(cacheDepartment());
+        ignite.createCache(cacheEmployee());
+        ignite.createCache(cacheCar());
+        ignite.createCache(cacheParking());
+
+        populateCacheEmployee();
+        populateCacheCar();
+
+        cachePool = AgentDemoUtils.newScheduledThreadPool(2, "demo-sql-load-cache-tasks");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        cachePool.scheduleWithFixedDelay(new Runnable() {
+            @Override public void run() {
+                try {
+                    IgniteCache<Integer, Employee> cacheEmployee = ignite.cache(EMPLOYEE_CACHE_NAME);
+
+                    if (cacheEmployee != null)
+                        try(Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                            for (int i = 0, n = 1; i < cnt; i++, n++) {
+                                Integer id = rnd.nextInt(EMPL_CNT);
+
+                                Integer depId = rnd.nextInt(DEP_CNT);
+
+                                double r = rnd.nextDouble();
+
+                                cacheEmployee.put(id, new Employee(id, depId, depId, "First name employee #" + n,
+                                    "Last name employee #" + n, "Email employee #" + n, "Phone number employee #" + n,
+                                    new java.sql.Date((long)(r * range)), "Job employee #" + n,
+                                    500 + AgentDemoUtils.round(r * 2000, 2)));
+
+                                if (rnd.nextBoolean())
+                                    cacheEmployee.remove(rnd.nextInt(EMPL_CNT));
+
+                                cacheEmployee.get(rnd.nextInt(EMPL_CNT));
+                            }
+
+                            if (rnd.nextInt(100) > 20)
+                                tx.commit();
+                        }
+                }
+                catch (Throwable e) {
+                    if (!e.getMessage().contains("cache is stopped"))
+                        ignite.log().error("Cache write task execution error", e);
+                }
+            }
+        }, 10, 3, TimeUnit.SECONDS);
+
+        cachePool.scheduleWithFixedDelay(new Runnable() {
+            @Override public void run() {
+                try {
+                    IgniteCache<Integer, Car> cache = ignite.cache(CAR_CACHE_NAME);
+
+                    if (cache != null)
+                        for (int i = 0; i < cnt; i++) {
+                            Integer carId = rnd.nextInt(CAR_CNT);
+
+                            cache.put(carId, new Car(carId, rnd.nextInt(PARK_CNT), "Car #" + (i + 1)));
+
+                            if (rnd.nextBoolean())
+                                cache.remove(rnd.nextInt(CAR_CNT));
+                        }
+                }
+                catch (IllegalStateException ignored) {
+                    // No-op.
+                }
+                catch (Throwable e) {
+                    if (!e.getMessage().contains("cache is stopped"))
+                        ignite.log().error("Cache write task execution error", e);
+                }
+            }
+        }, 10, 3, TimeUnit.SECONDS);
+    }
+
+
+    /**
+     * Create base cache configuration.
+     *
+     * @param name cache name.
+     * @return Cache configuration with basic properties set.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheConfiguration(String name) {
+        CacheConfiguration<K, V> ccfg = new CacheConfiguration<>(name);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg.setQueryDetailMetricsSize(10);
+        ccfg.setStartSize(100);
+        ccfg.setStatisticsEnabled(true);
+
+        return ccfg;
+    }
+
+    /**
+     * Configure cacheCountry.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheCountry() {
+        CacheConfiguration<K, V> ccfg = cacheConfiguration(COUNTRY_CACHE_NAME);
+
+        // Configure cacheCountry types.
+        Collection<QueryEntity> qryEntities = new ArrayList<>();
+
+        // COUNTRY.
+        QueryEntity type = new QueryEntity();
+
+        qryEntities.add(type);
+
+        type.setKeyType(Integer.class.getName());
+        type.setValueType(Country.class.getName());
+
+        // Query fields for COUNTRY.
+        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
+
+        qryFlds.put("id", "java.lang.Integer");
+        qryFlds.put("name", "java.lang.String");
+        qryFlds.put("population", "java.lang.Integer");
+
+        type.setFields(qryFlds);
+
+        ccfg.setQueryEntities(qryEntities);
+
+        return ccfg;
+    }
+
+    /**
+     * Configure cacheEmployee.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheDepartment() {
+        CacheConfiguration<K, V> ccfg = cacheConfiguration(DEPARTMENT_CACHE_NAME);
+
+        // Configure cacheDepartment types.
+        Collection<QueryEntity> qryEntities = new ArrayList<>();
+
+        // DEPARTMENT.
+        QueryEntity type = new QueryEntity();
+
+        qryEntities.add(type);
+
+        type.setKeyType(Integer.class.getName());
+        type.setValueType(Department.class.getName());
+
+        // Query fields for DEPARTMENT.
+        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
+
+        qryFlds.put("id", "java.lang.Integer");
+        qryFlds.put("countryId", "java.lang.Integer");
+        qryFlds.put("name", "java.lang.String");
+
+        type.setFields(qryFlds);
+
+        ccfg.setQueryEntities(qryEntities);
+
+        return ccfg;
+    }
+
+    /**
+     * Configure cacheEmployee.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheEmployee() {
+        CacheConfiguration<K, V> ccfg = cacheConfiguration(EMPLOYEE_CACHE_NAME);
+
+        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+        ccfg.setBackups(1);
+
+        // Configure cacheEmployee types.
+        Collection<QueryEntity> qryEntities = new ArrayList<>();
+
+        // EMPLOYEE.
+        QueryEntity type = new QueryEntity();
+
+        qryEntities.add(type);
+
+        type.setKeyType(Integer.class.getName());
+        type.setValueType(Employee.class.getName());
+
+        // Query fields for EMPLOYEE.
+        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
+
+        qryFlds.put("id", "java.lang.Integer");
+        qryFlds.put("departmentId", "java.lang.Integer");
+        qryFlds.put("managerId", "java.lang.Integer");
+        qryFlds.put("firstName", "java.lang.String");
+        qryFlds.put("lastName", "java.lang.String");
+        qryFlds.put("email", "java.lang.String");
+        qryFlds.put("phoneNumber", "java.lang.String");
+        qryFlds.put("hireDate", "java.sql.Date");
+        qryFlds.put("job", "java.lang.String");
+        qryFlds.put("salary", "java.lang.Double");
+
+        type.setFields(qryFlds);
+
+        // Indexes for EMPLOYEE.
+        Collection<QueryIndex> indexes = new ArrayList<>();
+
+        QueryIndex idx = new QueryIndex();
+
+        idx.setName("EMP_NAMES");
+        idx.setIndexType(QueryIndexType.SORTED);
+        LinkedHashMap<String, Boolean> indFlds = new LinkedHashMap<>();
+
+        indFlds.put("firstName", Boolean.FALSE);
+        indFlds.put("lastName", Boolean.FALSE);
+
+        idx.setFields(indFlds);
+
+        indexes.add(idx);
+        indexes.add(new QueryIndex("salary", QueryIndexType.SORTED, false, "EMP_SALARY"));
+
+        type.setIndexes(indexes);
+
+        ccfg.setQueryEntities(qryEntities);
+
+        return ccfg;
+    }
+
+    /**
+     * Configure cacheEmployee.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheParking() {
+        CacheConfiguration<K, V> ccfg = cacheConfiguration(PARKING_CACHE_NAME);
+
+        // Configure cacheParking types.
+        Collection<QueryEntity> qryEntities = new ArrayList<>();
+
+        // PARKING.
+        QueryEntity type = new QueryEntity();
+
+        qryEntities.add(type);
+
+        type.setKeyType(Integer.class.getName());
+        type.setValueType(Parking.class.getName());
+
+        // Query fields for PARKING.
+        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
+
+        qryFlds.put("id", "java.lang.Integer");
+        qryFlds.put("name", "java.lang.String");
+        qryFlds.put("capacity", "java.lang.Integer");
+
+        type.setFields(qryFlds);
+
+        ccfg.setQueryEntities(qryEntities);
+
+        return ccfg;
+    }
+
+    /**
+     * Configure cacheEmployee.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheCar() {
+        CacheConfiguration<K, V> ccfg = cacheConfiguration(CAR_CACHE_NAME);
+
+        // Configure cacheCar types.
+        Collection<QueryEntity> qryEntities = new ArrayList<>();
+
+        // CAR.
+        QueryEntity type = new QueryEntity();
+
+        qryEntities.add(type);
+
+        type.setKeyType(Integer.class.getName());
+        type.setValueType(Car.class.getName());
+
+        // Query fields for CAR.
+        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
+
+        qryFlds.put("id", "java.lang.Integer");
+        qryFlds.put("parkingId", "java.lang.Integer");
+        qryFlds.put("name", "java.lang.String");
+
+        type.setFields(qryFlds);
+
+        ccfg.setQueryEntities(qryEntities);
+
+        return ccfg;
+    }
+
+    /** */
+    private void populateCacheEmployee() {
+        if (ignite.log().isDebugEnabled())
+            ignite.log().debug("DEMO: Start employees population with data...");
+
+        IgniteCache<Integer, Country> cacheCountry = ignite.cache(COUNTRY_CACHE_NAME);
+
+        for (int i = 0, n = 1; i < CNTR_CNT; i++, n++)
+            cacheCountry.put(i, new Country(i, "Country #" + n, n * 10000000));
+
+        IgniteCache<Integer, Department> cacheDepartment = ignite.cache(DEPARTMENT_CACHE_NAME);
+
+        IgniteCache<Integer, Employee> cacheEmployee = ignite.cache(EMPLOYEE_CACHE_NAME);
+
+        for (int i = 0, n = 1; i < DEP_CNT; i++, n++) {
+            cacheDepartment.put(i, new Department(n, rnd.nextInt(CNTR_CNT), "Department #" + n));
+
+            double r = rnd.nextDouble();
+
+            cacheEmployee.put(i, new Employee(i, rnd.nextInt(DEP_CNT), null, "First name manager #" + n,
+                "Last name manager #" + n, "Email manager #" + n, "Phone number manager #" + n,
+                new java.sql.Date((long)(r * range)), "Job manager #" + n, 1000 + AgentDemoUtils.round(r * 4000, 2)));
+        }
+
+        for (int i = 0, n = 1; i < EMPL_CNT; i++, n++) {
+            Integer depId = rnd.nextInt(DEP_CNT);
+
+            double r = rnd.nextDouble();
+
+            cacheEmployee.put(i, new Employee(i, depId, depId, "First name employee #" + n,
+                "Last name employee #" + n, "Email employee #" + n, "Phone number employee #" + n,
+                new java.sql.Date((long)(r * range)), "Job employee #" + n, 500 + AgentDemoUtils.round(r * 2000, 2)));
+        }
+
+        if (ignite.log().isDebugEnabled())
+            ignite.log().debug("DEMO: Finished employees population.");
+    }
+
+    /** */
+    private void populateCacheCar() {
+        if (ignite.log().isDebugEnabled())
+            ignite.log().debug("DEMO: Start cars population...");
+
+        IgniteCache<Integer, Parking> cacheParking = ignite.cache(PARKING_CACHE_NAME);
+
+        for (int i = 0, n = 1; i < PARK_CNT; i++, n++)
+            cacheParking.put(i, new Parking(i, "Parking #" + n, n * 10));
+
+        IgniteCache<Integer, Car> cacheCar = ignite.cache(CAR_CACHE_NAME);
+
+        for (int i = 0, n = 1; i < CAR_CNT; i++, n++)
+            cacheCar.put(i, new Car(i, rnd.nextInt(PARK_CNT), "Car #" + n));
+
+        if (ignite.log().isDebugEnabled())
+            ignite.log().debug("DEMO: Finished cars population.");
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
new file mode 100644
index 0000000000000..57b26a2102ded
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import java.util.Random;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.console.demo.AgentDemoUtils;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Demo service. Create cache and populate it by random int pairs.
+ */
+public class DemoRandomCacheLoadService implements Service {
+    /** Ignite instance. */
+    @IgniteInstanceResource
+    private Ignite ignite;
+
+    /** Thread pool to execute cache load operations. */
+    private ScheduledExecutorService cachePool;
+
+    /** */
+    public static final String RANDOM_CACHE_NAME = "RandomCache";
+
+    /** Employees count. */
+    private static final int RND_CNT = 1024;
+
+    /** */
+    private static final Random rnd = new Random();
+
+    /** Maximum count read/write key. */
+    private final int cnt;
+
+    /**
+     * @param cnt Maximum count read/write key.
+     */
+    public DemoRandomCacheLoadService(int cnt) {
+        this.cnt = cnt;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        if (cachePool != null)
+            cachePool.shutdown();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        ignite.getOrCreateCache(cacheRandom());
+
+        cachePool = AgentDemoUtils.newScheduledThreadPool(2, "demo-sql-random-load-cache-tasks");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        cachePool.scheduleWithFixedDelay(new Runnable() {
+            @Override public void run() {
+                try {
+                    for (String cacheName : ignite.cacheNames()) {
+                        if (!DemoCachesLoadService.DEMO_CACHES.contains(cacheName)) {
+                            IgniteCache<Integer, Integer> cache = ignite.cache(cacheName);
+
+                            if (cache != null) {
+                                for (int i = 0, n = 1; i < cnt; i++, n++) {
+                                    Integer key = rnd.nextInt(RND_CNT);
+                                    Integer val = rnd.nextInt(RND_CNT);
+
+                                    cache.put(key, val);
+
+                                    if (rnd.nextInt(100) < 30)
+                                        cache.remove(key);
+                                }
+                            }
+                        }
+                    }
+                }
+                catch (Throwable e) {
+                    if (!e.getMessage().contains("cache is stopped"))
+                        ignite.log().error("Cache write task execution error", e);
+                }
+            }
+        }, 10, 3, TimeUnit.SECONDS);
+    }
+
+    /**
+     * Configure cacheCountry.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheRandom() {
+        CacheConfiguration<K, V> ccfg = new CacheConfiguration<>(RANDOM_CACHE_NAME);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg.setQueryDetailMetricsSize(10);
+        ccfg.setStartSize(100);
+        ccfg.setStatisticsEnabled(true);
+        ccfg.setIndexedTypes(Integer.class, Integer.class);
+
+        return ccfg;
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceClusterSingleton.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceClusterSingleton.java
new file mode 100644
index 0000000000000..8c0623a2901ba
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceClusterSingleton.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Demo service to provide on one node in cluster.
+ */
+public class DemoServiceClusterSingleton implements Service {
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceKeyAffinity.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceKeyAffinity.java
new file mode 100644
index 0000000000000..081ae27de88eb
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceKeyAffinity.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Demo service to provide for cache.
+ */
+public class DemoServiceKeyAffinity implements Service {
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceMultipleInstances.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceMultipleInstances.java
new file mode 100644
index 0000000000000..0d10753beb5ca
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceMultipleInstances.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Demo service to provide on all nodes.
+ */
+public class DemoServiceMultipleInstances implements Service {
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceNodeSingleton.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceNodeSingleton.java
new file mode 100644
index 0000000000000..4d491da90104d
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceNodeSingleton.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Demo service to provide on all nodes by one.
+ */
+public class DemoServiceNodeSingleton implements Service {
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+}

From 2d6735a20ac8009d1ac3f003da4fcd319271bd71 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Thu, 9 Feb 2017 16:44:41 +0700
Subject: [PATCH 150/446] IGNITE-4676 Fixed hang if closure executed nested
 internal task with continuation. Added test.

(cherry picked from commit e7a5307)
---
 .../processors/job/GridJobWorker.java         |  4 +
 .../internal/GridContinuousTaskSelfTest.java  | 79 +++++++++++++++++++
 2 files changed, 83 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index 6a00d96a0abc2..acefde72eefab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -617,6 +617,10 @@ else if (X.hasCause(e, GridServiceNotFoundException.class) ||
                 // Finish here only if not held by this thread.
                 if (!HOLD.get())
                     finishJob(res, ex, sndRes);
+                else
+                    // Make sure flag is not set for current thread.
+                    // This may happen in case of nested internal task call with continuation.
+                    HOLD.set(false);
 
                 ctx.job().currentTaskSession(null);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
index 98e3c5afc8e2d..cec288714c100 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
@@ -21,10 +21,12 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.concurrent.Callable;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
@@ -43,7 +45,9 @@
 import org.apache.ignite.compute.ComputeTaskSessionAttributeListener;
 import org.apache.ignite.compute.ComputeTaskSessionFullSupport;
 import org.apache.ignite.compute.ComputeTaskSplitAdapter;
+import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.resources.TaskContinuousMapperResource;
@@ -51,6 +55,7 @@
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Continuous task test.
@@ -195,6 +200,80 @@ public void testMultipleHoldccCalls() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If test failed.
+     */
+    public void testClosureWithNestedInternalTask() throws Exception {
+        try {
+            IgniteEx ignite = startGrid(0);
+
+            ComputeTaskInternalFuture<String> fut = ignite.context().closure().callAsync(GridClosureCallMode.BALANCE, new Callable<String>() {
+                /** */
+                @IgniteInstanceResource
+                private IgniteEx g;
+
+                @Override public String call() throws Exception {
+                    return g.compute(g.cluster()).execute(NestedHoldccTask.class, null);
+                }
+            }, ignite.cluster().nodes());
+
+            assertEquals("DONE", fut.get(3000));
+        }
+        finally {
+            stopGrid(0, true);
+        }
+    }
+
+    /** Test task with continuation. */
+    @GridInternal
+    public static class NestedHoldccTask extends ComputeTaskAdapter<String, String> {
+        /** {@inheritDoc} */
+        @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
+            @Nullable String arg) throws IgniteException {
+            Map<ComputeJob, ClusterNode> map = new HashMap<>();
+
+            for (ClusterNode node : subgrid)
+                map.put(new NestedHoldccJob(), node);
+
+            return map;
+
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public String reduce(List<ComputeJobResult> results) throws IgniteException {
+            return results.get(0).getData();
+        }
+    }
+
+    /** Test job. */
+    public static class NestedHoldccJob extends ComputeJobAdapter {
+        /** */
+        @JobContextResource
+        private ComputeJobContext jobCtx;
+
+        /** */
+        private int cnt = 0;
+
+        /** {@inheritDoc} */
+        @Override public Object execute() throws IgniteException {
+            if (cnt < 1) {
+                cnt++;
+
+                jobCtx.holdcc();
+
+                new Timer().schedule(new TimerTask() {
+                    @Override public void run() {
+                        jobCtx.callcc();
+                    }
+                }, 500);
+
+                return "NOT DONE";
+            }
+
+            return "DONE";
+        }
+    }
+
     /** */
     @SuppressWarnings({"PublicInnerClass"})
     public static class TestMultipleHoldccCallsClosure implements IgniteClosure<Object, Boolean> {

From 1fe33bc7b6e1dc966aed43533d32d4ce9c18795c Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Mon, 13 Feb 2017 13:04:37 +0300
Subject: [PATCH 151/446] IGNITE-4688: Changed copyrights to 2017.

---
 .../java/org/apache/ignite/internal/IgniteVersionUtils.java     | 2 +-
 .../org/apache/ignite/startup/GridRandomCommandLineLoader.java  | 2 +-
 .../Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs       | 2 +-
 .../dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec     | 2 +-
 .../dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs      | 2 +-
 .../Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs   | 2 +-
 .../dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec  | 2 +-
 .../dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec         | 2 +-
 .../dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs        | 2 +-
 .../dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec         | 2 +-
 .../dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec   | 2 +-
 .../dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs     | 2 +-
 .../dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec         | 2 +-
 .../dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs        | 2 +-
 .../frontend/app/modules/branding/branding.provider.js          | 2 +-
 15 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java
index bd8726f6380a6..7cb15bec5f8ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java
@@ -47,7 +47,7 @@ public class IgniteVersionUtils {
     public static final String ACK_VER_STR;
 
     /** Copyright blurb. */
-    public static final String COPYRIGHT = "2016 Copyright(C) Apache Software Foundation";
+    public static final String COPYRIGHT = "2017 Copyright(C) Apache Software Foundation";
 
     /**
      * Static initializer.
diff --git a/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java b/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java
index 13c38efc8201f..73a9760a91536 100644
--- a/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java
+++ b/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java
@@ -61,7 +61,7 @@ public final class GridRandomCommandLineLoader {
     private static final String IGNITE_PROG_NAME = "IGNITE_PROG_NAME";
 
     /** Copyright text. Ant processed. */
-    private static final String COPYRIGHT = "2016 Copyright(C) Apache Software Foundation.";
+    private static final String COPYRIGHT = "2017 Copyright(C) Apache Software Foundation.";
 
     /** Version. Ant processed. */
     private static final String VER = "x.x.x";
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
index d6cb3df14b072..97ab3ef04e7e0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
@@ -27,7 +27,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2016")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec b/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec
index 2324faa45080f..236584e8ebe55 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec
@@ -48,7 +48,7 @@ More info: https://apacheignite-net.readme.io/
             Apache Ignite ASP.NET Integration
         </summary>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2016</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>OutputCacheProvider Apache Ignite In-Memory Distributed Computing SQL NoSQL Grid Map Reduce Cache</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
index 9bd5c4726d7d4..ad1d5ab0d8ceb 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
@@ -25,7 +25,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2016")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
index ad0e91549fdd0..6349dd9f99bf5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2016")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec
index 367bdd550a631..8c7731b535f44 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec
@@ -43,7 +43,7 @@ XSD file describes the structure of IgniteConfigurationSection and enables Intel
 More info on Apache Ignite.NET: https://apacheignite-net.readme.io/
         </description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2016</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>Apache Ignite XSD Intellisense</tags>
     </metadata>
     <files>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
index bb6227e2c0ef4..2c4dfc173cd78 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
@@ -48,7 +48,7 @@ More info: https://apacheignite-net.readme.io/
             High-performance in-memory platform for computing and transacting on large-scale data sets in real-time.
         </summary>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2016</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL Grid Map Reduce Cache linqpad-samples</tags>
     </metadata>
     <files>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
index 240f2731708f3..f8512f14f21d9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
@@ -25,7 +25,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2016")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec b/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec
index 088ca13bb94d3..93aa8c2cb7e65 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec
@@ -50,7 +50,7 @@ More info: https://apacheignite-net.readme.io/
             LINQ Provider for Apache Ignite
         </summary>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2016</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL LINQ Grid Map Reduce Cache linqpad-samples</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec
index fa5c39a49098c..a3f86c17cc88f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec
@@ -40,7 +40,7 @@ Creating NuGet package:
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <description>log4net Logger for Apache Ignite</description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2016</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL LINQ Grid Map Reduce Cache log4net logger</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
index 824252e90449d..262a7cdd0720a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
@@ -24,7 +24,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2016")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec b/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec
index 765e26f390cfb..e3a6f42db4c4a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec
@@ -40,7 +40,7 @@ Creating NuGet package:
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <description>NLog Logger for Apache Ignite</description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2016</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL LINQ Grid Map Reduce Cache NLog logger</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
index f28278c55723b..f952d7d55f8c9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
@@ -24,7 +24,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2016")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/web-console/frontend/app/modules/branding/branding.provider.js b/modules/web-console/frontend/app/modules/branding/branding.provider.js
index ce14b349bc181..0545ff3a24a06 100644
--- a/modules/web-console/frontend/app/modules/branding/branding.provider.js
+++ b/modules/web-console/frontend/app/modules/branding/branding.provider.js
@@ -26,7 +26,7 @@ export default ['IgniteBranding', [function() {
 
     let footerHtml = [
         '<p>Apache Ignite Web Console</p>',
-        '<p>© 2016 The Apache Software Foundation.</p>',
+        '<p>© 2017 The Apache Software Foundation.</p>',
         '<p>Apache, Apache Ignite, the Apache feather and the Apache Ignite logo are trademarks of The Apache Software Foundation.</p>'
     ];
 

From 0a4366591df30bf9c6841cf487e92f9decac6059 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Mon, 13 Feb 2017 13:19:56 +0300
Subject: [PATCH 152/446] IGNITE-4688: Changed copyrights to 2017.

---
 NOTICE                                                          | 2 +-
 assembly/NOTICE_FABRIC                                          | 2 +-
 assembly/NOTICE_HADOOP                                          | 2 +-
 modules/core/src/main/java/META-INF/NOTICE                      | 2 +-
 .../dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs  | 2 +-
 .../Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs | 2 +-
 .../dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs  | 2 +-
 .../dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs        | 2 +-
 .../platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs   | 2 +-
 .../examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs  | 2 +-
 .../Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs        | 2 +-
 modules/scalar-2.10/pom.xml                                     | 2 +-
 modules/scalar/pom.xml                                          | 2 +-
 parent/pom.xml                                                  | 2 +-
 14 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/NOTICE b/NOTICE
index 5ec3b0e16c50b..33e2479d6e89e 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2015 The Apache Software Foundation
+Copyright 2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/assembly/NOTICE_FABRIC b/assembly/NOTICE_FABRIC
index 2e55768f62040..c5e6f023ad023 100644
--- a/assembly/NOTICE_FABRIC
+++ b/assembly/NOTICE_FABRIC
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2015 The Apache Software Foundation
+Copyright 2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/assembly/NOTICE_HADOOP b/assembly/NOTICE_HADOOP
index 5ec3b0e16c50b..33e2479d6e89e 100644
--- a/assembly/NOTICE_HADOOP
+++ b/assembly/NOTICE_HADOOP
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2015 The Apache Software Foundation
+Copyright 2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/modules/core/src/main/java/META-INF/NOTICE b/modules/core/src/main/java/META-INF/NOTICE
index 5ec3b0e16c50b..33e2479d6e89e 100644
--- a/modules/core/src/main/java/META-INF/NOTICE
+++ b/modules/core/src/main/java/META-INF/NOTICE
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2015 The Apache Software Foundation
+Copyright 2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
index 91f79bde52888..a1d8aefe1a950 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
index a8c4305674e86..3994598a1afa8 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
index 7bf322fa81dae..d8bb169b5dfd6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
index 76596da4a3ffa..446fe1e8d4137 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
@@ -24,7 +24,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
index 85030264cf06f..41b62894ed144 100644
--- a/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
index 5834c57bca37f..396eeb65a5c35 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
index 230e9d8434d53..0855afb383393 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyCopyright("Copyright ©  2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/scalar-2.10/pom.xml b/modules/scalar-2.10/pom.xml
index 3920e0a8ad3ec..e857a3b7e0b17 100644
--- a/modules/scalar-2.10/pom.xml
+++ b/modules/scalar-2.10/pom.xml
@@ -158,7 +158,7 @@
                                             <!--<td>-->
                                                 <!--<nobr>Ignite&#153; - Scalar DSL, ver. <strong>${project.version}</strong></nobr>-->
                                                 <!--<br>-->
-                                                <!--<a target=_blank href="https://ignite.apache.org"><nobr>2015 Copyright &#169; Apache Software Foundation</nobr></a>-->
+                                                <!--<a target=_blank href="https://ignite.apache.org"><nobr>2017 Copyright &#169; Apache Software Foundation</nobr></a>-->
                                             <!--</td>-->
                                         <!--</tr>-->
                                         <!--</table>-->
diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml
index eb8a24cfd0790..807ccb82b1625 100644
--- a/modules/scalar/pom.xml
+++ b/modules/scalar/pom.xml
@@ -143,7 +143,7 @@
                                             <td>
                                                 <nobr>Ignite&#153; - Scalar DSL, ver. <strong>${project.version}</strong></nobr>
                                                 <br>
-                                                <a target=_blank href="https://ignite.apache.org"><nobr>2015 Copyright &#169; Apache Software Foundation</nobr></a>
+                                                <a target=_blank href="https://ignite.apache.org"><nobr>2017 Copyright &#169; Apache Software Foundation</nobr></a>
                                             </td>
                                         </tr>
                                         </table>
diff --git a/parent/pom.xml b/parent/pom.xml
index f766437ef48f2..88532a70dfb5a 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -464,7 +464,7 @@
                                                 <tbody style="padding: 0; margin: 0">
                                                     <tr style="padding: 0; margin: 0">
                                                         <td>
-                                                            <a target=_blank href="https://ignite.apache.org"><nobr>2015 Copyright &#169; Apache Software Foundation</nobr></a>
+                                                            <a target=_blank href="https://ignite.apache.org"><nobr>2017 Copyright &#169; Apache Software Foundation</nobr></a>
                                                         </td>
                                                     </tr>
                                                 </tbody>

From d307e2eced1fd10b007ee08c3dd113e7bb4f22ba Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Mon, 13 Feb 2017 17:35:29 +0700
Subject: [PATCH 153/446] IGNITE-4687 Added pool to process REST request in Web
 Agent.

(cherry picked from commit 262a341)
---
 .../ignite/console/agent/AgentLauncher.java   | 203 +++++++-------
 .../ignite/console/agent/AgentUtils.java      |  80 ++++++
 .../agent/handlers/AbstractHandler.java       | 110 --------
 .../agent/handlers/AbstractListener.java      | 104 +++++++
 ...baseHandler.java => DatabaseListener.java} | 264 ++++++++++--------
 .../{RestHandler.java => RestListener.java}   |  28 +-
 6 files changed, 441 insertions(+), 348 deletions(-)
 delete mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractHandler.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
 rename modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/{DatabaseHandler.java => DatabaseListener.java} (57%)
 rename modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/{RestHandler.java => RestListener.java} (94%)

diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 049791f9107dc..a3d609fb395ae 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -41,8 +41,8 @@
 import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.X509TrustManager;
-import org.apache.ignite.console.agent.handlers.DatabaseHandler;
-import org.apache.ignite.console.agent.handlers.RestHandler;
+import org.apache.ignite.console.agent.handlers.DatabaseListener;
+import org.apache.ignite.console.agent.handlers.RestListener;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.log4j.Logger;
 import org.json.JSONException;
@@ -278,141 +278,138 @@ public static void main(String[] args) throws Exception {
             cfg.tokens(Arrays.asList(tokens.trim().split(",")));
         }
 
-        final RestHandler restHnd = new RestHandler(cfg);
+        URI uri = URI.create(cfg.serverUri());
 
-        try {
-            restHnd.start();
+        // Create proxy authenticator using passed properties.
+        switch (uri.getScheme()) {
+            case "http":
+            case "https":
+                final String username = System.getProperty(uri.getScheme() + ".proxyUsername");
+                final char[] pwd = System.getProperty(uri.getScheme() +  ".proxyPassword", "").toCharArray();
 
-            URI uri = URI.create(cfg.serverUri());
+                Authenticator.setDefault(new Authenticator() {
+                    @Override protected PasswordAuthentication getPasswordAuthentication() {
+                        return new PasswordAuthentication(username, pwd);
+                    }
+                });
 
-            // Create proxy authenticator using passed properties.
-            switch (uri.getScheme()) {
-                case "http":
-                case "https":
-                    final String username = System.getProperty(uri.getScheme() + ".proxyUsername");
-                    final char[] pwd = System.getProperty(uri.getScheme() +  ".proxyPassword", "").toCharArray();
+                break;
 
-                    Authenticator.setDefault(new Authenticator() {
-                        @Override protected PasswordAuthentication getPasswordAuthentication() {
-                            return new PasswordAuthentication(username, pwd);
-                        }
-                    });
-
-                    break;
+            default:
+                // No-op.
+        }
 
-                default:
-                    // No-op.
-            }
+        IO.Options opts = new IO.Options();
 
-            IO.Options opts = new IO.Options();
+        opts.path = "/agents";
 
-            opts.path = "/agents";
+        // Workaround for use self-signed certificate
+        if (Boolean.getBoolean("trust.all")) {
+            SSLContext ctx = SSLContext.getInstance("TLS");
 
-            // Workaround for use self-signed certificate
-            if (Boolean.getBoolean("trust.all")) {
-                SSLContext ctx = SSLContext.getInstance("TLS");
+            // Create an SSLContext that uses our TrustManager
+            ctx.init(null, getTrustManagers(), null);
 
-                // Create an SSLContext that uses our TrustManager
-                ctx.init(null, getTrustManagers(), null);
+            opts.sslContext = ctx;
+        }
 
-                opts.sslContext = ctx;
-            }
+        final Socket client = IO.socket(uri, opts);
 
-            final Socket client = IO.socket(uri, opts);
+        final RestListener restHnd = new RestListener(cfg);
 
-            try {
-                Emitter.Listener onConnecting = new Emitter.Listener() {
-                    @Override public void call(Object... args) {
-                        log.info("Connecting to: " + cfg.serverUri());
-                    }
-                };
+        final DatabaseListener dbHnd = new DatabaseListener(cfg);
 
-                Emitter.Listener onConnect = new Emitter.Listener() {
-                    @Override public void call(Object... args) {
-                        log.info("Connection established.");
+        try {
+            Emitter.Listener onConnecting = new Emitter.Listener() {
+                @Override public void call(Object... args) {
+                    log.info("Connecting to: " + cfg.serverUri());
+                }
+            };
 
-                        JSONObject authMsg = new JSONObject();
+            Emitter.Listener onConnect = new Emitter.Listener() {
+                @Override public void call(Object... args) {
+                    log.info("Connection established.");
 
-                        try {
-                            authMsg.put("tokens", cfg.tokens());
+                    JSONObject authMsg = new JSONObject();
 
-                            String clsName = AgentLauncher.class.getSimpleName() + ".class";
+                    try {
+                        authMsg.put("tokens", cfg.tokens());
 
-                            String clsPath = AgentLauncher.class.getResource(clsName).toString();
+                        String clsName = AgentLauncher.class.getSimpleName() + ".class";
 
-                            if (clsPath.startsWith("jar")) {
-                                String manifestPath = clsPath.substring(0, clsPath.lastIndexOf('!') + 1) +
-                                    "/META-INF/MANIFEST.MF";
+                        String clsPath = AgentLauncher.class.getResource(clsName).toString();
 
-                                Manifest manifest = new Manifest(new URL(manifestPath).openStream());
+                        if (clsPath.startsWith("jar")) {
+                            String manifestPath = clsPath.substring(0, clsPath.lastIndexOf('!') + 1) +
+                                "/META-INF/MANIFEST.MF";
 
-                                Attributes attr = manifest.getMainAttributes();
+                            Manifest manifest = new Manifest(new URL(manifestPath).openStream());
 
-                                authMsg.put("ver", attr.getValue("Implementation-Version"));
-                                authMsg.put("bt", attr.getValue("Build-Time"));
-                            }
+                            Attributes attr = manifest.getMainAttributes();
 
-                            client.emit("agent:auth", authMsg, new Ack() {
-                                @Override public void call(Object... args) {
-                                    // Authentication failed if response contains args.
-                                    if (args != null && args.length > 0) {
-                                        onDisconnect.call(args);
+                            authMsg.put("ver", attr.getValue("Implementation-Version"));
+                            authMsg.put("bt", attr.getValue("Build-Time"));
+                        }
 
-                                        System.exit(1);
-                                    }
+                        client.emit("agent:auth", authMsg, new Ack() {
+                            @Override public void call(Object... args) {
+                                // Authentication failed if response contains args.
+                                if (args != null && args.length > 0) {
+                                    onDisconnect.call(args);
 
-                                    log.info("Authentication success.");
+                                    System.exit(1);
                                 }
-                            });
-                        }
-                        catch (JSONException | IOException e) {
-                            log.error("Failed to construct authentication message", e);
 
-                            client.close();
-                        }
+                                log.info("Authentication success.");
+                            }
+                        });
                     }
-                };
-
-                DatabaseHandler dbHnd = new DatabaseHandler(cfg);
-
-                final CountDownLatch latch = new CountDownLatch(1);
-
-                client
-                    .on(EVENT_CONNECTING, onConnecting)
-                    .on(EVENT_CONNECT, onConnect)
-                    .on(EVENT_CONNECT_ERROR, onError)
-                    .on(EVENT_RECONNECTING, onConnecting)
-                    .on(EVENT_NODE_REST, restHnd)
-                    .on(EVENT_SCHEMA_IMPORT_DRIVERS, dbHnd.availableDriversListener())
-                    .on(EVENT_SCHEMA_IMPORT_SCHEMAS, dbHnd.schemasListener())
-                    .on(EVENT_SCHEMA_IMPORT_METADATA, dbHnd.metadataListener())
-                    .on(EVENT_ERROR, onError)
-                    .on(EVENT_DISCONNECT, onDisconnect)
-                    .on(EVENT_AGENT_WARNING, new Emitter.Listener() {
-                        @Override public void call(Object... args) {
-                            log.warn(args[0]);
-                        }
-                    })
-                    .on(EVENT_AGENT_CLOSE, new Emitter.Listener() {
-                        @Override public void call(Object... args) {
-                            onDisconnect.call(args);
+                    catch (JSONException | IOException e) {
+                        log.error("Failed to construct authentication message", e);
+
+                        client.close();
+                    }
+                }
+            };
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            client
+                .on(EVENT_CONNECTING, onConnecting)
+                .on(EVENT_CONNECT, onConnect)
+                .on(EVENT_CONNECT_ERROR, onError)
+                .on(EVENT_RECONNECTING, onConnecting)
+                .on(EVENT_NODE_REST, restHnd)
+                .on(EVENT_SCHEMA_IMPORT_DRIVERS, dbHnd.availableDriversListener())
+                .on(EVENT_SCHEMA_IMPORT_SCHEMAS, dbHnd.schemasListener())
+                .on(EVENT_SCHEMA_IMPORT_METADATA, dbHnd.metadataListener())
+                .on(EVENT_ERROR, onError)
+                .on(EVENT_DISCONNECT, onDisconnect)
+                .on(EVENT_AGENT_WARNING, new Emitter.Listener() {
+                    @Override public void call(Object... args) {
+                        log.warn(args[0]);
+                    }
+                })
+                .on(EVENT_AGENT_CLOSE, new Emitter.Listener() {
+                    @Override public void call(Object... args) {
+                        onDisconnect.call(args);
 
-                            client.off();
+                        client.off();
 
-                            latch.countDown();
-                        }
-                    });
+                        latch.countDown();
+                    }
+                });
 
-                client.connect();
+            client.connect();
 
-                latch.await();
-            }
-            finally {
-                client.close();
-            }
+            latch.await();
         }
         finally {
+            client.close();
+
             restHnd.stop();
+
+            dbHnd.stop();
         }
     }
 }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java
index 50a849adcc570..cb22651e9f83e 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java
@@ -17,11 +17,19 @@
 
 package org.apache.ignite.console.agent;
 
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
+import io.socket.client.Ack;
 import java.io.File;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.security.ProtectionDomain;
+import java.util.Arrays;
 import org.apache.log4j.Logger;
+import org.json.JSONArray;
+import org.json.JSONObject;
 
 /**
  * Utility methods.
@@ -30,6 +38,28 @@ public class AgentUtils {
     /** */
     private static final Logger log = Logger.getLogger(AgentUtils.class.getName());
 
+    /** JSON object mapper. */
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    /** */
+    private static final Ack NOOP_CB = new Ack() {
+        @Override public void call(Object... args) {
+            if (args != null && args.length > 0 && args[0] instanceof Throwable)
+                log.error("Failed to execute request on agent.", (Throwable) args[0]);
+            else
+                log.info("Request on agent successfully executed " + Arrays.toString(args));
+        }
+    };
+
+    static {
+        JsonOrgModule module = new JsonOrgModule();
+
+        mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
+        mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+
+        mapper.registerModule(module);
+    }
+    
     /**
      * Default constructor.
      */
@@ -108,4 +138,54 @@ public static File resolvePath(String path) {
 
         return null;
     }
+
+    /**
+     * Get callback from handler arguments.
+     *
+     * @param args Arguments.
+     * @return Callback or noop callback.
+     */
+    public static Ack safeCallback(Object[] args) {
+        boolean hasCb = args != null && args.length > 0 && args[args.length - 1] instanceof Ack;
+
+        return hasCb ? (Ack)args[args.length - 1] : NOOP_CB;
+    }
+
+    /**
+     * Remove callback from handler arguments.
+     * 
+     * @param args Arguments.
+     * @return Arguments without callback.
+     */
+    public static Object[] removeCallback(Object[] args) {
+        boolean hasCb = args != null && args.length > 0 && args[args.length - 1] instanceof Ack;
+
+        return hasCb ? Arrays.copyOf(args, args.length - 1) : args;
+    }
+
+    /**
+     * Map java object to JSON object.
+     * 
+     * @param obj Java object.
+     * @return {@link JSONObject} or {@link JSONArray}.
+     * @throws IllegalArgumentException If conversion fails due to incompatible type.
+     */
+    public static Object toJSON(Object obj) {
+        if (obj instanceof Iterable)
+            return mapper.convertValue(obj, JSONArray.class);
+
+        return mapper.convertValue(obj, JSONObject.class);
+    }
+
+    /**
+     * Map JSON object to java object.
+     *
+     * @param obj {@link JSONObject} or {@link JSONArray}.
+     * @param toValType Expected value type.
+     * @return Mapped object type of {@link T}.
+     * @throws IllegalArgumentException If conversion fails due to incompatible type.
+     */
+    public static <T> T fromJSON(Object obj, Class<T> toValType) throws IllegalArgumentException {
+        return mapper.convertValue(obj, toValType);
+    }
 }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractHandler.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractHandler.java
deleted file mode 100644
index 7e4e3204185ca..0000000000000
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractHandler.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.console.agent.handlers;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
-import io.socket.client.Ack;
-import io.socket.emitter.Emitter;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Map;
-import org.json.JSONArray;
-import org.json.JSONObject;
-
-/**
- * Base class for web socket handlers.
- */
-abstract class AbstractHandler implements Emitter.Listener {
-    /** JSON object mapper. */
-    private static final ObjectMapper mapper = new ObjectMapper();
-
-    static {
-        JsonOrgModule module = new JsonOrgModule();
-
-        mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
-        mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
-
-        mapper.registerModule(module);
-    }
-
-    /**
-     * @param obj Object.
-     * @return {@link JSONObject} or {@link JSONArray}.
-     */
-    private Object toJSON(Object obj) {
-        if (obj instanceof Iterable)
-            return mapper.convertValue(obj, JSONArray.class);
-
-        return mapper.convertValue(obj, JSONObject.class);
-    }
-
-    /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public final void call(Object... args) {
-        Ack cb = null;
-
-        try {
-            if (args == null || args.length == 0)
-                throw new IllegalArgumentException("Missing arguments.");
-
-            if (args.length > 2)
-                throw new IllegalArgumentException("Wrong arguments count, must be <= 2: " + Arrays.toString(args));
-
-            JSONObject lsnrArgs = null;
-
-            if (args.length == 1) {
-                if (args[0] instanceof JSONObject)
-                    lsnrArgs = (JSONObject)args[0];
-                else if (args[0] instanceof Ack)
-                    cb = (Ack)args[0];
-                else
-                    throw new IllegalArgumentException("Wrong type of argument, must be JSONObject or Ack: " + args[0]);
-            }
-            else {
-                if (args[0] != null && !(args[0] instanceof JSONObject))
-                    throw new IllegalArgumentException("Wrong type of argument, must be JSONObject: " + args[0]);
-
-                if (!(args[1] instanceof Ack))
-                    throw new IllegalArgumentException("Wrong type of argument, must be Ack: " + args[1]);
-
-                lsnrArgs = (JSONObject)args[0];
-
-                cb = (Ack)args[1];
-            }
-
-            Object res = execute(lsnrArgs == null ? Collections.emptyMap() : mapper.convertValue(lsnrArgs, Map.class));
-
-            if (cb != null)
-                cb.call(null, toJSON(res));
-        }
-        catch (Exception e) {
-            if (cb != null)
-                cb.call(e, null);
-        }
-    }
-
-    /**
-     * Execute command with specified arguments.
-     *
-     * @param args Map with method args.
-     */
-    public abstract Object execute(Map<String, Object> args) throws Exception;
-}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
new file mode 100644
index 0000000000000..987dac90bb77a
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.agent.handlers;
+
+import io.socket.client.Ack;
+import io.socket.emitter.Emitter;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.log4j.Logger;
+
+import static org.apache.ignite.console.agent.AgentUtils.removeCallback;
+import static org.apache.ignite.console.agent.AgentUtils.fromJSON;
+import static org.apache.ignite.console.agent.AgentUtils.safeCallback;
+import static org.apache.ignite.console.agent.AgentUtils.toJSON;
+
+/**
+ * Base class for web socket handlers.
+ */
+abstract class AbstractListener implements Emitter.Listener {
+    /** */
+    private ExecutorService pool;
+
+    /** */
+    final Logger log = Logger.getLogger(this.getClass().getName());
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override public final void call(Object... args) {
+        final Ack cb = safeCallback(args);
+
+        args = removeCallback(args);
+
+        try {
+            final Map<String, Object> params;
+
+            if (args == null || args.length == 0)
+                params = Collections.emptyMap();
+            else if (args.length == 1)
+                params = fromJSON(args[0], Map.class);
+            else
+                throw new IllegalArgumentException("Wrong arguments count, must be <= 1: " + Arrays.toString(args));
+
+            if (pool == null)
+                pool = newThreadPool();
+
+            pool.submit(new Runnable() {
+                @Override public void run() {
+                    try {
+                        Object res = execute(params);
+
+                        cb.call(null, toJSON(res));
+                    } catch (Exception e) {
+                        cb.call(e, null);
+                    }
+                }
+            });
+        }
+        catch (Exception e) {
+            cb.call(e, null);
+        }
+    }
+
+    /**
+     * Stop handler.
+     */
+    public void stop() {
+        if (pool != null)
+            pool.shutdownNow();
+    }
+
+    /**
+     * Creates a thread pool that can schedule commands to run after a given delay, or to execute periodically.
+     *
+     * @return Newly created thread pool.
+     */
+    protected ExecutorService newThreadPool() {
+        return Executors.newSingleThreadExecutor();
+    }
+
+    /**
+     * Execute command with specified arguments.
+     *
+     * @param args Map with method args.
+     */
+    public abstract Object execute(Map<String, Object> args) throws Exception;
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseHandler.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java
similarity index 57%
rename from modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseHandler.java
rename to modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java
index 02146d9a67996..4577228d2034f 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseHandler.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java
@@ -43,17 +43,138 @@
 /**
  * API to extract database metadata.
  */
-public class DatabaseHandler {
+public class DatabaseListener {
     /** */
-    private static final Logger log = Logger.getLogger(DatabaseHandler.class.getName());
+    private static final Logger log = Logger.getLogger(DatabaseListener.class.getName());
 
     /** */
     private final File driversFolder;
 
+    /** */
+    private final AbstractListener schemasLsnr = new AbstractListener() {
+        @Override public Object execute(Map<String, Object> args) throws Exception {
+            String driverPath = null;
+
+            if (args.containsKey("driverPath"))
+                driverPath = args.get("driverPath").toString();
+
+            if (!args.containsKey("driverClass"))
+                throw new IllegalArgumentException("Missing driverClass in arguments: " + args);
+
+            String driverCls = args.get("driverClass").toString();
+
+            if (!args.containsKey("url"))
+                throw new IllegalArgumentException("Missing url in arguments: " + args);
+
+            String url = args.get("url").toString();
+
+            if (!args.containsKey("info"))
+                throw new IllegalArgumentException("Missing info in arguments: " + args);
+
+            Properties info = new Properties();
+
+            info.putAll((Map)args.get("info"));
+
+            return schemas(driverPath, driverCls, url, info);
+        }
+    };
+
+    private final AbstractListener metadataLsnr = new AbstractListener() {
+        @SuppressWarnings("unchecked")
+        @Override public Object execute(Map<String, Object> args) throws Exception {
+            String driverPath = null;
+
+            if (args.containsKey("driverPath"))
+                driverPath = args.get("driverPath").toString();
+
+            if (!args.containsKey("driverClass"))
+                throw new IllegalArgumentException("Missing driverClass in arguments: " + args);
+
+            String driverCls = args.get("driverClass").toString();
+
+            if (!args.containsKey("url"))
+                throw new IllegalArgumentException("Missing url in arguments: " + args);
+
+            String url = args.get("url").toString();
+
+            if (!args.containsKey("info"))
+                throw new IllegalArgumentException("Missing info in arguments: " + args);
+
+            Properties info = new Properties();
+
+            info.putAll((Map)args.get("info"));
+
+            if (!args.containsKey("schemas"))
+                throw new IllegalArgumentException("Missing schemas in arguments: " + args);
+
+            List<String> schemas = (List<String>)args.get("schemas");
+
+            if (!args.containsKey("tablesOnly"))
+                throw new IllegalArgumentException("Missing tablesOnly in arguments: " + args);
+
+            boolean tblsOnly = (boolean)args.get("tablesOnly");
+
+            return metadata(driverPath, driverCls, url, info, schemas, tblsOnly);
+        }
+    };
+
+    private final AbstractListener availableDriversLsnr = new AbstractListener() {
+        @Override public Object execute(Map<String, Object> args) throws Exception {
+            if (driversFolder == null) {
+                log.info("JDBC drivers folder not specified, returning empty list");
+
+                return Collections.emptyList();
+            }
+
+            if (log.isDebugEnabled())
+                log.debug("Collecting JDBC drivers in folder: " + driversFolder.getPath());
+
+            File[] list = driversFolder.listFiles(new FilenameFilter() {
+                @Override public boolean accept(File dir, String name) {
+                    return name.endsWith(".jar");
+                }
+            });
+
+            if (list == null) {
+                log.info("JDBC drivers folder has no files, returning empty list");
+
+                return Collections.emptyList();
+            }
+
+            List<JdbcDriver> res = new ArrayList<>();
+
+            for (File file : list) {
+                try {
+                    boolean win = System.getProperty("os.name").contains("win");
+
+                    URL url = new URL("jar", null,
+                        "file:" + (win ? "/" : "") + file.getPath() + "!/META-INF/services/java.sql.Driver");
+
+                    try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) {
+                        String jdbcDriverCls = reader.readLine();
+
+                        res.add(new JdbcDriver(file.getName(), jdbcDriverCls));
+
+                        if (log.isDebugEnabled())
+                            log.debug("Found: [driver=" + file + ", class=" + jdbcDriverCls + "]");
+                    }
+                }
+                catch (IOException e) {
+                    res.add(new JdbcDriver(file.getName(), null));
+
+                    log.info("Found: [driver=" + file + "]");
+                    log.info("Failed to detect driver class: " + e.getMessage());
+                }
+            }
+
+            return res;
+        }
+    };
+
     /**
      * @param cfg Config.
      */
-    public DatabaseHandler(AgentConfiguration cfg) {
+    public DatabaseListener(AgentConfiguration cfg) {
         driversFolder = resolvePath(cfg.driversFolder() == null ? "jdbc-drivers" : cfg.driversFolder());
     }
 
@@ -105,39 +226,23 @@ protected Collection<String> schemas(String jdbcDriverJarPath, String jdbcDriver
         }
     }
 
+    /**
+     * Listener for drivers.
+     *
+     * @return Drivers in drivers folder
+     * @see AgentConfiguration#driversFolder
+     */
+    public Emitter.Listener availableDriversListener() {
+        return availableDriversLsnr;
+    }
+
     /**
      * Listener for schema names.
      *
      * @return Collection of schema names.
      */
     public Emitter.Listener schemasListener() {
-        return new AbstractHandler() {
-            @Override public Object execute(Map<String, Object> args) throws Exception {
-                String driverPath = null;
-
-                if (args.containsKey("driverPath"))
-                    driverPath = args.get("driverPath").toString();
-
-                if (!args.containsKey("driverClass"))
-                    throw new IllegalArgumentException("Missing driverClass in arguments: " + args);
-
-                String driverCls = args.get("driverClass").toString();
-
-                if (!args.containsKey("url"))
-                    throw new IllegalArgumentException("Missing url in arguments: " + args);
-
-                String url = args.get("url").toString();
-
-                if (!args.containsKey("info"))
-                    throw new IllegalArgumentException("Missing info in arguments: " + args);
-
-                Properties info = new Properties();
-
-                info.putAll((Map)args.get("info"));
-
-                return schemas(driverPath, driverCls, url, info);
-            }
-        };
+        return schemasLsnr;
     }
 
     /**
@@ -176,105 +281,18 @@ protected Collection<DbTable> metadata(String jdbcDriverJarPath, String jdbcDriv
      * @return Collection of tables.
      */
     public Emitter.Listener metadataListener() {
-        return new AbstractHandler() {
-            @SuppressWarnings("unchecked")
-            @Override public Object execute(Map<String, Object> args) throws Exception {
-                String driverPath = null;
-
-                if (args.containsKey("driverPath"))
-                    driverPath = args.get("driverPath").toString();
-
-                if (!args.containsKey("driverClass"))
-                    throw new IllegalArgumentException("Missing driverClass in arguments: " + args);
-
-                String driverCls = args.get("driverClass").toString();
-
-                if (!args.containsKey("url"))
-                    throw new IllegalArgumentException("Missing url in arguments: " + args);
-
-                String url = args.get("url").toString();
-
-                if (!args.containsKey("info"))
-                    throw new IllegalArgumentException("Missing info in arguments: " + args);
-
-                Properties info = new Properties();
-
-                info.putAll((Map)args.get("info"));
-
-                if (!args.containsKey("schemas"))
-                    throw new IllegalArgumentException("Missing schemas in arguments: " + args);
-
-                List<String> schemas = (List<String>)args.get("schemas");
-
-                if (!args.containsKey("tablesOnly"))
-                    throw new IllegalArgumentException("Missing tablesOnly in arguments: " + args);
-
-                boolean tblsOnly = (boolean)args.get("tablesOnly");
-
-                return metadata(driverPath, driverCls, url, info, schemas, tblsOnly);
-            }
-        };
+        return metadataLsnr;
     }
 
     /**
-     * Listener for drivers.
-     *
-     * @return Drivers in drivers folder
-     * @see AgentConfiguration#driversFolder
+     * Stop handler.
      */
-    public Emitter.Listener availableDriversListener() {
-        return new AbstractHandler() {
-            @Override public Object execute(Map<String, Object> args) throws Exception {
-                if (driversFolder == null) {
-                    log.info("JDBC drivers folder not specified, returning empty list");
-
-                    return Collections.emptyList();
-                }
-
-                if (log.isDebugEnabled())
-                    log.debug("Collecting JDBC drivers in folder: " + driversFolder.getPath());
-
-                File[] list = driversFolder.listFiles(new FilenameFilter() {
-                    @Override public boolean accept(File dir, String name) {
-                        return name.endsWith(".jar");
-                    }
-                });
+    public void stop() {
+        availableDriversLsnr.stop();
 
-                if (list == null) {
-                    log.info("JDBC drivers folder has no files, returning empty list");
+        schemasLsnr.stop();
 
-                    return Collections.emptyList();
-                }
-
-                List<JdbcDriver> res = new ArrayList<>();
-
-                for (File file : list) {
-                    try {
-                        boolean win = System.getProperty("os.name").contains("win");
-
-                        URL url = new URL("jar", null,
-                            "file:" + (win ? "/" : "") + file.getPath() + "!/META-INF/services/java.sql.Driver");
-
-                        try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) {
-                            String jdbcDriverCls = reader.readLine();
-
-                            res.add(new JdbcDriver(file.getName(), jdbcDriverCls));
-
-                            if (log.isDebugEnabled())
-                                log.debug("Found: [driver=" + file + ", class=" + jdbcDriverCls + "]");
-                        }
-                    }
-                    catch (IOException e) {
-                        res.add(new JdbcDriver(file.getName(), null));
-
-                        log.info("Found: [driver=" + file + "]");
-                        log.info("Failed to detect driver class: " + e.getMessage());
-                    }
-                }
-
-                return res;
-            }
-        };
+        metadataLsnr.stop();
     }
 
     /**
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestHandler.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
similarity index 94%
rename from modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestHandler.java
rename to modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
index 1b4b565ee68f0..1e86549e25927 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestHandler.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
@@ -24,6 +24,8 @@
 import java.nio.charset.Charset;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import org.apache.commons.codec.Charsets;
 import org.apache.http.Header;
 import org.apache.http.NameValuePair;
@@ -45,9 +47,9 @@
 /**
  * API to translate REST requests to Ignite cluster.
  */
-public class RestHandler extends AbstractHandler {
+public class RestListener extends AbstractListener {
     /** */
-    private static final Logger log = Logger.getLogger(RestHandler.class.getName());
+    private static final Logger log = Logger.getLogger(RestListener.class.getName());
 
     /** */
     private final AgentConfiguration cfg;
@@ -58,21 +60,18 @@ public class RestHandler extends AbstractHandler {
     /**
      * @param cfg Config.
      */
-    public RestHandler(AgentConfiguration cfg) {
+    public RestListener(AgentConfiguration cfg) {
+        super();
+
         this.cfg = cfg;
-    }
 
-    /**
-     * Start HTTP client for communication with node via REST.
-     */
-    public void start() {
         httpClient = HttpClientBuilder.create().build();
     }
 
-    /**
-     * Stop HTTP client.
-     */
-    public void stop() {
+    /** {@inheritDoc} */
+    @Override public void stop() {
+        super.stop();
+
         if (httpClient != null) {
             try {
                 httpClient.close();
@@ -83,6 +82,11 @@ public void stop() {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override protected ExecutorService newThreadPool() {
+        return Executors.newCachedThreadPool();
+    }
+
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public Object execute(Map<String, Object> args) throws Exception {

From 8874f99f44dc2edf08a525619edb49d5db70b938 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 14 Feb 2017 18:44:57 +0300
Subject: [PATCH 154/446] IGNITE-4641 - Refresh client attributes during
 reconnect

---
 .../internal/managers/GridManagerAdapter.java |   4 +
 .../apache/ignite/spi/IgniteSpiAdapter.java   |   5 +
 .../apache/ignite/spi/IgniteSpiContext.java   |   6 +
 .../ignite/spi/discovery/tcp/ClientImpl.java  |   2 +-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   5 +
 .../tcp/internal/TcpDiscoveryNode.java        |   6 +-
 ...ryNodeAttributesUpdateOnReconnectTest.java | 110 +++++++++++++++++
 .../tcp/TestReconnectPluginProvider.java      | 111 ++++++++++++++++++
 .../discovery/tcp/TestReconnectProcessor.java |  93 +++++++++++++++
 .../testframework/GridSpiTestContext.java     |   5 +
 .../IgniteSpiDiscoverySelfTestSuite.java      |   3 +
 .../org.apache.ignite.plugin.PluginProvider   |   1 +
 parent/pom.xml                                |   1 +
 13 files changed, 349 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryNodeAttributesUpdateOnReconnectTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectPluginProvider.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
 create mode 100644 modules/core/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
index 584cc56e23b3f..25cc715ee12b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
@@ -569,6 +569,10 @@ protected final String stopInfo() {
                         ctx.timeout().removeTimeoutObject(new GridSpiTimeoutObject(obj));
                     }
 
+                    @Override public Map<String, Object> nodeAttributes() {
+                        return ctx.nodeAttributes();
+                    }
+
                     /**
                      * @param e Exception to handle.
                      * @return GridSpiException Converted exception.
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index 219d07be5f725..8879364dca5dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -928,5 +928,10 @@ private class GridDummySpiContext implements IgniteSpiContext {
 
             ((IgniteKernal)ignite0).context().timeout().removeTimeoutObject(new GridSpiTimeoutObject(obj));
         }
+
+        /** {@inheritDoc} */
+        @Override public Map<String, Object> nodeAttributes() {
+            return Collections.emptyMap();
+        }
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
index 5eb5227a0a46c..96b3e61a94131 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
@@ -19,6 +19,7 @@
 
 import java.io.Serializable;
 import java.util.Collection;
+import java.util.Map;
 import java.util.UUID;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteException;
@@ -352,4 +353,9 @@ public interface IgniteSpiContext {
      * @param c Timeout object.
      */
     public void removeTimeoutObject(IgniteSpiTimeoutObject c);
+
+    /**
+     * @return Current node attributes.
+     */
+    public Map<String, Object> nodeAttributes();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 39c539c30fb89..932e7d1e754a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -628,7 +628,7 @@ else if (addrs.isEmpty()) {
                     TcpDiscoveryNode node = locNode;
 
                     if (locNode.order() > 0)
-                        node = locNode.clientReconnectNode();
+                        node = locNode.clientReconnectNode(spi.spiCtx.nodeAttributes());
 
                     msg = new TcpDiscoveryJoinRequestMessage(node, spi.collectExchangeData(getLocalNodeId()));
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 45933e112f78d..00ae97d1e302c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -409,6 +409,9 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, T
     /** */
     private boolean clientReconnectDisabled;
 
+    /** */
+    protected IgniteSpiContext spiCtx;
+
     /** {@inheritDoc} */
     @Override public String getSpiState() {
         return impl.getSpiState();
@@ -1161,6 +1164,8 @@ LinkedHashSet<InetSocketAddress> getNodeAddresses(TcpDiscoveryNode node, boolean
     @Override protected void onContextInitialized0(IgniteSpiContext spiCtx) throws IgniteSpiException {
         super.onContextInitialized0(spiCtx);
 
+        this.spiCtx = spiCtx;
+
         ctxInitLatch.countDown();
 
         ipFinder.onSpiContextInitialized(spiCtx);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
index 307aefe641908..d8b1fc12243e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
@@ -26,6 +26,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -513,13 +514,14 @@ public void onClientDisconnected(UUID newId) {
     }
 
     /**
+     * @param nodeAttrs Current node attributes.
      * @return Copy of local node for client reconnect request.
      */
-    public TcpDiscoveryNode clientReconnectNode() {
+    public TcpDiscoveryNode clientReconnectNode(Map<String, Object> nodeAttrs) {
         TcpDiscoveryNode node = new TcpDiscoveryNode(id, addrs, hostNames, discPort, metricsProvider, ver,
             null);
 
-        node.attrs = attrs;
+        node.attrs = Collections.unmodifiableMap(new HashMap<>(nodeAttrs));
         node.clientRouterNodeId = clientRouterNodeId;
 
         return node;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryNodeAttributesUpdateOnReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryNodeAttributesUpdateOnReconnectTest.java
new file mode 100644
index 0000000000000..56dc4ece5f4fc
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryNodeAttributesUpdateOnReconnectTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteEvents;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.internal.IgniteClientReconnectAbstractTest.reconnectClientNode;
+
+/**
+ * Checks whether on client reconnect node attributes from kernal context are sent.
+ */
+public class TcpDiscoveryNodeAttributesUpdateOnReconnectTest extends GridCommonAbstractTest {
+    /** */
+    private volatile String rejoinAttr;
+
+    /** */
+    @LoggerResource
+    private IgniteLogger log;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("client")) {
+            Map<String, String> attrs = new HashMap<>();
+
+            attrs.put("test", "1");
+
+            cfg.setUserAttributes(attrs);
+            cfg.setClientMode(true);
+        }
+
+        IgniteClientReconnectAbstractTest.TestTcpDiscoverySpi spi = new IgniteClientReconnectAbstractTest.TestTcpDiscoverySpi();
+
+        TcpDiscoveryIpFinder finder = ((TcpDiscoverySpi)cfg.getDiscoverySpi()).getIpFinder();
+
+        spi.setIpFinder(finder);
+
+        cfg.setDiscoverySpi(spi);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        TestReconnectPluginProvider.enabled = false;
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        TestReconnectPluginProvider.enabled = true;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnect() throws Exception {
+        Ignite srv = startGrid("server");
+
+        IgniteEvents evts = srv.events();
+
+        evts.enableLocal(EventType.EVTS_DISCOVERY_ALL);
+        evts.localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                ClusterNode node = ((DiscoveryEvent)evt).eventNode();
+
+                rejoinAttr = node.attribute("test");
+
+                return true;
+            }
+        }, EventType.EVT_NODE_JOINED);
+
+        Ignite client = startGrid("client");
+
+        reconnectClientNode(log, client, srv, null);
+
+        assertEquals("2", rejoinAttr);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectPluginProvider.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectPluginProvider.java
new file mode 100644
index 0000000000000..692774ca79dc8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectPluginProvider.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.Serializable;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
+import org.apache.ignite.plugin.ExtensionRegistry;
+import org.apache.ignite.plugin.IgnitePlugin;
+import org.apache.ignite.plugin.PluginContext;
+import org.apache.ignite.plugin.PluginProvider;
+import org.apache.ignite.plugin.PluginValidationException;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Creates TestReconnectProcessor.
+ */
+public class TestReconnectPluginProvider implements PluginProvider {
+    /** */
+    private GridKernalContext igniteCtx;
+
+    /** */
+    public static volatile boolean enabled;
+
+    /** {@inheritDoc} */
+    @Override public String name() {
+        return "TestReconnectPlugin";
+    }
+
+    /** {@inheritDoc} */
+    @Override public String version() {
+        return "1.0";
+    }
+
+    /** {@inheritDoc} */
+    @Override public String copyright() {
+        return "";
+    }
+
+    /** {@inheritDoc} */
+    @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) {
+        igniteCtx = ((IgniteKernal)ctx.grid()).context();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void start(PluginContext ctx) throws IgniteCheckedException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void stop(boolean cancel) throws IgniteCheckedException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onIgniteStart() throws IgniteCheckedException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onIgniteStop(boolean cancel) {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Serializable provideDiscoveryData(UUID nodeId) {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void receiveDiscoveryData(UUID nodeId, Serializable data) {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void validateNewNode(ClusterNode node) throws PluginValidationException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object createComponent(PluginContext ctx, Class cls) {
+        if (enabled && GridSecurityProcessor.class.equals(cls))
+            return new TestReconnectProcessor(igniteCtx);
+
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgnitePlugin plugin() {
+        return new IgnitePlugin() {};
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
new file mode 100644
index 0000000000000..f0ed35c4ccfab
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.Collection;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.plugin.security.AuthenticationContext;
+import org.apache.ignite.plugin.security.SecurityCredentials;
+import org.apache.ignite.plugin.security.SecurityException;
+import org.apache.ignite.plugin.security.SecurityPermission;
+import org.apache.ignite.plugin.security.SecuritySubject;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Updates node attributes on disconnect.
+ */
+public class TestReconnectProcessor extends GridProcessorAdapter implements GridSecurityProcessor {
+    /**
+     * @param ctx Kernal context.
+     */
+    protected TestReconnectProcessor(GridKernalContext ctx) {
+        super(ctx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public SecurityContext authenticateNode(ClusterNode node,
+        SecurityCredentials cred) throws IgniteCheckedException {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isGlobalNodeAuthentication() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public SecurityContext authenticate(AuthenticationContext ctx) throws IgniteCheckedException {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<SecuritySubject> authenticatedSubjects() throws IgniteCheckedException {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public SecuritySubject authenticatedSubject(UUID subjId) throws IgniteCheckedException {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void authorize(String name, SecurityPermission perm,
+        @Nullable SecurityContext securityCtx) throws SecurityException {
+
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onSessionExpired(UUID subjId) {
+
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean enabled() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onDisconnected(IgniteFuture<?> reconnectFut) throws IgniteCheckedException {
+        ctx.addNodeAttribute("test", "2");
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index ac50ef90e29e1..1c8acbc3960ce 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -601,6 +601,11 @@ public void triggerEvent(Event evt) {
             timeoutProcessor.removeTimeoutObject(new GridSpiTimeoutObject(obj));
     }
 
+    /** {@inheritDoc} */
+    @Override public Map<String, Object> nodeAttributes() {
+        return Collections.emptyMap();
+    }
+
     /**
      * @param cacheName Cache name.
      * @return Map representing cache.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 5f870a474578b..548e1a5a51f81 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMultiThreadedTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeAttributesUpdateOnReconnectTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeConfigConsistentIdSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeConsistentIdSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryRestartTest;
@@ -84,6 +85,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(TcpDiscoveryRestartTest.class));
         suite.addTest(new TestSuite(TcpDiscoveryMultiThreadedTest.class));
 
+        suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
+
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
 
diff --git a/modules/core/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider b/modules/core/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider
new file mode 100644
index 0000000000000..a7fdf4384e626
--- /dev/null
+++ b/modules/core/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider
@@ -0,0 +1 @@
+org.apache.ignite.spi.discovery.tcp.TestReconnectPluginProvider
\ No newline at end of file
diff --git a/parent/pom.xml b/parent/pom.xml
index 88532a70dfb5a..8bf5dde9e12a8 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -821,6 +821,7 @@
                                         <exclude>src/main/java/META-INF/services/javax.cache.spi.CachingProvider</exclude><!--cannot be changed-->
                                         <exclude>src/main/java/org/jetbrains/annotations/*.java</exclude><!--copyright-->
                                         <exclude>src/main/resources/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider</exclude><!--cannot be changed-->
+                                        <exclude>/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider</exclude>
                                         <exclude>dev-tools/IGNITE-*.patch</exclude>
                                         <exclude>dev-tools/.gradle/**/*</exclude>
                                         <exclude>dev-tools/gradle/wrapper/**/*</exclude>

From ca5956bedfc0c3bd18290c64b6a6c2e3f114a440 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Tue, 14 Feb 2017 23:28:06 +0700
Subject: [PATCH 155/446] IGNITE-4472 UI fix, minor fixes

(cherry picked from commit 79e1e53)
---
 .../web-console/backend/routes/activities.js  |   7 -
 .../backend/services/activities.js            |  27 +--
 .../test/unit/ActivitiesService.test.js       | 131 +++++++++++++
 .../web-console/frontend/app/app.config.js    |   4 +
 .../activities-user-dialog.controller.js      |  39 +---
 .../activities-user-dialog.jade               |   2 +-
 .../activities-user-dialog/index.js           |   5 +-
 .../form-field-datepicker.jade                |   4 +-
 .../list-of-registered-users.column-defs.js   |  26 +--
 .../list-of-registered-users.controller.js    |  32 ++-
 .../app/core/activities/Activities.data.js    |   5 -
 modules/web-console/frontend/app/data/i18n.js |   1 +
 .../ui-ace-pom/ui-ace-pom.controller.js       |   4 +-
 .../app/modules/agent/agent.module.js         |  15 --
 .../modules/configuration/Version.service.js  |  13 +-
 .../configuration/generator/Maven.service.js  |  10 +-
 .../configuration/summary/summary.worker.js   |   6 +-
 modules/web-console/frontend/package.json     | 182 +++++++++---------
 .../frontend/public/stylesheets/style.scss    |   8 -
 .../public/stylesheets/variables.scss         |   1 -
 .../views/templates/agent-download.jade       |   6 +-
 21 files changed, 290 insertions(+), 238 deletions(-)
 create mode 100644 modules/web-console/backend/test/unit/ActivitiesService.test.js

diff --git a/modules/web-console/backend/routes/activities.js b/modules/web-console/backend/routes/activities.js
index 08c27cf1cdf8c..ad0e46962821d 100644
--- a/modules/web-console/backend/routes/activities.js
+++ b/modules/web-console/backend/routes/activities.js
@@ -33,13 +33,6 @@ module.exports.factory = function(express, activitiesService) {
     return new Promise((factoryResolve) => {
         const router = new express.Router();
 
-        // Get user activities.
-        router.get('/user/:userId', (req, res) => {
-            activitiesService.listByUser(req.params.userId, req.query)
-                .then(res.api.ok)
-                .catch(res.api.error);
-        });
-
         // Post user activities to page.
         router.post('/page', (req, res) => {
             activitiesService.merge(req.user._id, req.body)
diff --git a/modules/web-console/backend/services/activities.js b/modules/web-console/backend/services/activities.js
index 7f3a7775f83d2..124c775dc7a86 100644
--- a/modules/web-console/backend/services/activities.js
+++ b/modules/web-console/backend/services/activities.js
@@ -35,10 +35,12 @@ module.exports.factory = (_, mongo) => {
          * Update page activities.
          *
          * @param {String} owner - User ID
-         * @param {Object} page - The page
+         * @param {String} action - Action string presentation.
+         * @param {String} group - Action group string presentation.
+         * @param {Date} [date] - Optional date to save in activity.
          * @returns {Promise.<mongo.ObjectId>} that resolve activity
          */
-        static merge(owner, {action, group}) {
+        static merge(owner, {action, group}, date = new Date()) {
             mongo.Account.findById(owner)
                 .then((user) => {
                     user.lastActivity = new Date();
@@ -46,8 +48,6 @@ module.exports.factory = (_, mongo) => {
                     return user.save();
                 });
 
-            const date = new Date();
-
             date.setDate(1);
             date.setHours(0, 0, 0, 0);
 
@@ -63,25 +63,6 @@ module.exports.factory = (_, mongo) => {
                 });
         }
 
-        /**
-         * Get user activities
-         * @param {String} owner - User ID
-         * @returns {Promise.<mongo.ObjectId>} that resolve activities
-         */
-        static listByUser(owner, {startDate, endDate}) {
-            const $match = {owner};
-
-            if (startDate)
-                $match.date = {$gte: new Date(startDate)};
-
-            if (endDate) {
-                $match.date = $match.date || {};
-                $match.date.$lt = new Date(endDate);
-            }
-
-            return mongo.Activities.find($match);
-        }
-
         static total({startDate, endDate}) {
             const $match = {};
 
diff --git a/modules/web-console/backend/test/unit/ActivitiesService.test.js b/modules/web-console/backend/test/unit/ActivitiesService.test.js
new file mode 100644
index 0000000000000..40088bf75c333
--- /dev/null
+++ b/modules/web-console/backend/test/unit/ActivitiesService.test.js
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const assert = require('chai').assert;
+const injector = require('../injector');
+const testAccounts = require('../data/accounts.json');
+
+let activitiesService;
+let mongo;
+let db;
+
+const owner = testAccounts[0]._id;
+const group = 'test';
+const action1 = '/test/activity1';
+const action2 = '/test/activity2';
+
+suite('ActivitiesServiceTestsSuite', () => {
+    suiteSetup(() => {
+        return Promise.all([
+            injector('services/activities'),
+            injector('mongo'),
+            injector('dbHelper')
+        ])
+            .then(([_activitiesService, _mongo, _db]) => {
+                mongo = _mongo;
+                activitiesService = _activitiesService;
+                db = _db;
+            });
+    });
+
+    setup(() => db.init());
+
+    test('Activities creation and update', (done) => {
+        activitiesService.merge(owner, { group, action: action1 })
+            .then((activity) => {
+                assert.isNotNull(activity);
+                assert.equal(activity.amount, 1);
+
+                return mongo.Activities.findById(activity._id);
+            })
+            .then((activityDoc) => {
+                assert.isNotNull(activityDoc);
+                assert.equal(activityDoc.amount, 1);
+            })
+            .then(() => activitiesService.merge(owner, { group, action: action1 }))
+            .then((activity) => {
+                assert.isNotNull(activity);
+                assert.equal(activity.amount, 2);
+
+                return mongo.Activities.findById(activity._id);
+            })
+            .then((activityDoc) => {
+                assert.isNotNull(activityDoc);
+                assert.equal(activityDoc.amount, 2);
+            })
+            .then(done)
+            .catch(done);
+    });
+
+    test('Activities total and detail information', (done) => {
+        const startDate = new Date();
+
+        startDate.setDate(1);
+        startDate.setHours(0, 0, 0, 0);
+
+        const endDate = new Date(startDate);
+        endDate.setMonth(endDate.getMonth() + 1);
+
+        Promise.all([
+            activitiesService.merge(owner, {group, action: action1}),
+            activitiesService.merge(owner, {group, action: action2})
+        ])
+            .then(() => activitiesService.total(owner, {startDate, endDate}))
+            .then((activities) =>
+                assert.equal(activities[owner].test, 2)
+            )
+            .then(() => activitiesService.detail(owner, {startDate, endDate}))
+            .then((activities) =>
+                assert.deepEqual(activities[owner], {
+                    '/test/activity2': 1, '/test/activity1': 1
+                })
+            )
+            .then(done)
+            .catch(done);
+    });
+
+    test('Activities periods', (done) => {
+        const startDate = new Date();
+
+        startDate.setDate(1);
+        startDate.setHours(0, 0, 0, 0);
+
+        const nextMonth = (baseDate) => {
+            const date = new Date(baseDate);
+
+            date.setMonth(date.getMonth() + 1);
+
+            return date;
+        };
+
+        const borderDate = nextMonth(startDate);
+        const endDate = nextMonth(borderDate);
+
+        activitiesService.merge(owner, { group, action: action1 })
+            .then(() => activitiesService.merge(owner, { group, action: action1 }, borderDate))
+            .then(() => activitiesService.total({ startDate, endDate: borderDate }))
+            .then((activities) =>
+                assert.equal(activities[owner].test, 1)
+            )
+            .then(() => activitiesService.total({ startDate: borderDate, endDate }))
+            .then((activities) =>
+                assert.equal(activities[owner].test, 1)
+            )
+            .then(done)
+            .catch(done);
+    });
+});
diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js
index 0e85711141799..39d761f418eef 100644
--- a/modules/web-console/frontend/app/app.config.js
+++ b/modules/web-console/frontend/app/app.config.js
@@ -103,3 +103,7 @@ igniteConsoleCfg.config(['$datepickerProvider', ($datepickerProvider) => {
         iconRight: 'icon-datepicker-right'
     });
 }]);
+
+igniteConsoleCfg.config(['$translateProvider', ($translateProvider) => {
+    $translateProvider.useSanitizeValueStrategy('sanitize');
+}]);
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
index 46853b262eaad..078f725fd8e2c 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
@@ -15,46 +15,13 @@
  * limitations under the License.
  */
 
-const COLUMNS_DEFS = [
-    {displayName: 'Action', field: 'action', minWidth: 65 },
-    {displayName: 'Description', field: 'title', minWidth: 65 },
-    {displayName: 'Visited', field: 'amount', minWidth: 65 }
-];
-
 export default class ActivitiesCtrl {
-    static $inject = ['$state', 'user', 'params', 'IgniteActivitiesData'];
+    static $inject = ['user'];
 
-    constructor($state, user, params, ActivitiesData) {
+    constructor(user) {
         const $ctrl = this;
-        const userId = user._id;
 
         $ctrl.user = user;
-
-        $ctrl.gridOptions = {
-            data: [],
-            columnVirtualizationThreshold: 30,
-            columnDefs: COLUMNS_DEFS,
-            categories: [
-                {name: 'Action', visible: true, selectable: true},
-                {name: 'Description', visible: true, selectable: true},
-                {name: 'Visited', visible: true, selectable: true}
-            ],
-            enableRowSelection: false,
-            enableRowHeaderSelection: false,
-            enableColumnMenus: false,
-            multiSelect: false,
-            modifierKeysToMultiSelect: true,
-            noUnselect: true,
-            flatEntityAccess: true,
-            fastWatch: true,
-            onRegisterApi: (api) => {
-                $ctrl.gridApi = api;
-            }
-        };
-
-        ActivitiesData.listByUser(userId, params)
-            .then((data) => {
-                $ctrl.data = data;
-            });
+        $ctrl.data = _.map(user.activitiesDetail, (amount, action) => ({ action, amount }));
     }
 }
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
index 2c55ebd6da29a..074851ca55d66 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
@@ -24,7 +24,7 @@
             .modal-body.modal-body-with-scroll(id='activities-user-dialog')
                 table.table.table-striped.table-bordered.table-hover(scrollable-container='#activities-user-dialog' st-table='displayedRows' st-safe-src='ctrl.data')
                     thead
-                        th.text-center(st-sort='title') Description
+                        th.text-center(st-sort='action | translate') Description
                         th.text-center(st-sort='action') Action
                         th.text-center(st-sort='amount') Visited
                     tbody
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/index.js b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
index 03d35852b147a..dca6ba92ede6d 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/index.js
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
@@ -18,13 +18,12 @@
  import controller from './activities-user-dialog.controller';
  import templateUrl from './activities-user-dialog.jade';
 
- export default ['$modal', ($modal) => ({ show = true, user, params }) => {
+ export default ['$modal', ($modal) => ({ show = true, user }) => {
      const ActivitiesUserDialog = $modal({
          templateUrl,
          show,
          resolve: {
-             user: () => user,
-             params: () => params
+             user: () => user
          },
          placement: 'center',
          controller,
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
index 6792977b65831..2578cf4b17189 100644
--- a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
+++ b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
@@ -28,8 +28,8 @@ mixin ignite-form-field-datepicker(label, model, name, disabled, required, place
             data-ng-disabled=disabled && '#{disabled}'
 
             bs-datepicker
-            data-date-format='MMM yyyy' 
-            data-start-view='1' 
+            data-date-format='MMM yyyy'
+            data-start-view='1'
             data-min-view='1' 
             data-max-date='today'
 
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
index 61e1bd8575b22..4dc465505b6ab 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
@@ -49,7 +49,7 @@ const ACTIONS_TEMPLATE = `
 const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
 
 export default [
-    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'test', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, pinnedLeft: true},
+    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, pinnedLeft: true},
     {displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by name...' }, pinnedLeft: true},
     {displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
     {displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 160, enableFiltering: true},
@@ -62,19 +62,19 @@ export default [
     {displayName: 'Caches count', categoryDisplayName: 'Configurations', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: 'counters.caches', type: 'number', headerTooltip: 'Caches count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
     {displayName: 'IGFS count', categoryDisplayName: 'Configurations', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: 'counters.igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
     // Activities Total
-    {displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Configuration', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Queries', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Demo', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'AD', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Agent Download', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'AS', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Agent Start', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Total count of configuration usages', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Total count of queries usages', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Total count of demo startup', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Total count of agent downloads', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Str', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Total count of agent startup', minWidth: 50, width: 50, enableFiltering: false},
     // Activities Configuration
-    {displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration Clusters', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration Model', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', headerTooltip: 'Configuration Caches', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration clusters', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration model', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', headerTooltip: 'Configuration caches', minWidth: 50, width: 80, enableFiltering: false, visible: false},
     {displayName: 'IGFS', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/igfs"] || 0', type: 'number', headerTooltip: 'Configuration IGFS', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', headerTooltip: 'Configuration Summary', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', headerTooltip: 'Configuration summary', minWidth: 50, width: 80, enableFiltering: false, visible: false},
     // Activities Queries
-    {displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', headerTooltip: 'Query execute', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', headerTooltip: 'Query explain', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', headerTooltip: 'Scan', minWidth: 50, width: 80, enableFiltering: false, visible: false}
+    {displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', headerTooltip: 'Query executions', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', headerTooltip: 'Query explain executions', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', headerTooltip: 'Scan query executions', minWidth: 50, width: 80, enableFiltering: false, visible: false}
 ];
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 19f7921ac7f17..1f2a348daad7a 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -20,15 +20,26 @@ import headerTemplate from 'app/components/ui-grid-header/ui-grid-header.jade';
 import columnDefs from './list-of-registered-users.column-defs';
 import categories from './list-of-registered-users.categories';
 
+const rowTemplate = `<div
+  ng-repeat="(colRenderIndex, col) in colContainer.renderedColumns track by col.uid"
+  ng-mouseover="grid.api.selection.selectRow(row.entity);"
+  ui-grid-one-bind-id-grid="rowRenderIndex + '-' + col.uid + '-cell'"
+  class="ui-grid-cell"
+  ng-class="{ 'ui-grid-row-header-cell': col.isRowHeader }"
+  role="{{col.isRowHeader ? 'rowheader' : 'gridcell'}}"
+  ui-grid-cell/>`;
+
 export default class IgniteListOfRegisteredUsersCtrl {
-    static $inject = ['$scope', '$state', '$templateCache', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
+    static $inject = ['$scope', '$state', '$filter', '$templateCache', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
 
-    constructor($scope, $state, $templateCache, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
+    constructor($scope, $state, $filter, $templateCache, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
         const $ctrl = this;
 
         const companySelectOptions = [];
         const countrySelectOptions = [];
 
+        const dtFilter = $filter('date');
+
         $ctrl.params = {
             startDate: new Date()
         };
@@ -82,7 +93,7 @@ export default class IgniteListOfRegisteredUsersCtrl {
         };
 
         const showActivities = (user) => {
-            return new ActivitiesUserDialog({ user, params: $ctrl.params });
+            return new ActivitiesUserDialog({ user });
         };
 
         $ctrl.gridOptions = {
@@ -91,14 +102,17 @@ export default class IgniteListOfRegisteredUsersCtrl {
             columnDefs,
             categories,
             headerTemplate: $templateCache.get(headerTemplate),
+            rowTemplate,
             enableFiltering: true,
-            enableRowSelection: false,
+            enableRowSelection: true,
             enableRowHeaderSelection: false,
             enableColumnMenus: false,
             multiSelect: false,
             modifierKeysToMultiSelect: true,
             noUnselect: true,
             fastWatch: true,
+            exporterSuppressColumns: ['actions'],
+            exporterCsvColumnSeparator: ';',
             onRegisterApi: (api) => {
                 $ctrl.gridApi = api;
 
@@ -139,14 +153,14 @@ export default class IgniteListOfRegisteredUsersCtrl {
                 .then((data) => $ctrl.adjustHeight(data.length));
         };
 
-        $scope.$watch(() => $ctrl.params.startDate, () => {
-            const endDate = new Date($ctrl.params.startDate);
+        $scope.$watch(() => $ctrl.params.startDate, (dt) => {
+            $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(dt, 'yyyy_MM')}.csv`;
 
-            endDate.setMonth(endDate.getMonth() + 1);
+            const endDate = new Date(dt);
 
-            $ctrl.params.endDate = endDate;
+            endDate.setMonth(endDate.getMonth() + 1);
 
-            reloadUsers($ctrl.params);
+            reloadUsers({startDate: dtFilter(dt, 'yyyy-MM-dd'), endDate: dtFilter(endDate, 'yyyy-MM-dd')});
         });
     }
 
diff --git a/modules/web-console/frontend/app/core/activities/Activities.data.js b/modules/web-console/frontend/app/core/activities/Activities.data.js
index 8a67a97a5dd43..8d9447cd2572d 100644
--- a/modules/web-console/frontend/app/core/activities/Activities.data.js
+++ b/modules/web-console/frontend/app/core/activities/Activities.data.js
@@ -31,9 +31,4 @@ export default class ActivitiesData {
 
         return this.$http.post('/api/v1/activities/page', { group, action });
     }
-
-    listByUser(userId, params) {
-        return this.$http.get(`/api/v1/activities/user/${userId}`, { params })
-            .then(({ data }) => data);
-    }
 }
diff --git a/modules/web-console/frontend/app/data/i18n.js b/modules/web-console/frontend/app/data/i18n.js
index bc8c700d8bfa6..3385f6061c911 100644
--- a/modules/web-console/frontend/app/data/i18n.js
+++ b/modules/web-console/frontend/app/data/i18n.js
@@ -23,6 +23,7 @@ export default {
     '/configuration/domains': 'Configure domain model',
     '/configuration/igfs': 'Configure IGFS',
     '/configuration/summary': 'Configurations summary',
+    '/configuration/download': 'Download project',
     '/demo/resume': 'Demo resume',
     '/demo/reset': 'Demo reset',
     '/queries/execute': 'Query execute',
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
index 477cf20999271..0ae12695dd5f6 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-export default ['$scope', 'IgniteMavenGenerator', 'IgniteVersion', function($scope, maven, Version) {
+export default ['$scope', 'IgniteMavenGenerator', function($scope, maven) {
     const ctrl = this;
 
     // Watchers definition.
@@ -25,7 +25,7 @@ export default ['$scope', 'IgniteMavenGenerator', 'IgniteVersion', function($sco
         if (!value)
             return;
 
-        ctrl.data = maven.generate($scope.cluster, Version.productVersion().ignite).asString();
+        ctrl.data = maven.generate($scope.cluster);
     };
 
     // Setup watchers.
diff --git a/modules/web-console/frontend/app/modules/agent/agent.module.js b/modules/web-console/frontend/app/modules/agent/agent.module.js
index d6fc86311cf24..7ac39d10f3594 100644
--- a/modules/web-console/frontend/app/modules/agent/agent.module.js
+++ b/modules/web-console/frontend/app/modules/agent/agent.module.js
@@ -62,21 +62,6 @@ class IgniteAgentMonitor {
                 this._scope.$$postDigest(() => $state.go(this._scope.backState));
         };
 
-        this._scope.downloadAgent = () => {
-            const lnk = document.createElement('a');
-
-            lnk.setAttribute('href', '/api/v1/agent/download/zip');
-            lnk.setAttribute('target', '_self');
-            lnk.setAttribute('download', null);
-            lnk.style.display = 'none';
-
-            document.body.appendChild(lnk);
-
-            lnk.click();
-
-            document.body.removeChild(lnk);
-        };
-
         this._scope.hasAgents = null;
         this._scope.showModal = false;
 
diff --git a/modules/web-console/frontend/app/modules/configuration/Version.service.js b/modules/web-console/frontend/app/modules/configuration/Version.service.js
index f0e9c4c2a2a89..746b1edd9b090 100644
--- a/modules/web-console/frontend/app/modules/configuration/Version.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/Version.service.js
@@ -23,6 +23,9 @@ const VERSION_MATCHER = /(\d+)\.(\d+)\.(\d+)([-.]([^0123456789][^-]+)(-SNAPSHOT)
 const numberComparator = (a, b) => a > b ? 1 : a < b ? -1 : 0;
 
 export default class IgniteVersion {
+    /** Current product version. */
+    static ignite = '1.8.0';
+
     /**
      * Tries to parse product version from it's string representation.
      *
@@ -73,16 +76,6 @@ export default class IgniteVersion {
         return numberComparator(pa.revTs, pb.revTs);
     }
 
-    /**
-     * Return current product version.
-     * @returns {{ignite: string}}
-     */
-    productVersion() {
-        return {
-            ignite: '1.8.0'
-        };
-    }
-
     /**
      * Check if node version is newer or same
      * @param {String} nodeVer
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js
index 2e017610e0e81..23a9c4e811656 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js
@@ -16,6 +16,7 @@
  */
 
 import StringBuilder from './StringBuilder';
+import IgniteVersion from 'app/modules/configuration/Version.service';
 
 // Java built-in class names.
 import POM_DEPENDENCIES from 'app/data/pom-dependencies.json';
@@ -142,11 +143,10 @@ export default class IgniteMavenGenerator {
      * Generate pom.xml.
      *
      * @param cluster Cluster  to take info about dependencies.
-     * @param version Ignite version for Ignite dependencies.
-     * @param sb Resulting output with generated pom.
+     * @param version Version for Ignite dependencies.
      * @returns {string} Generated content.
      */
-    generate(cluster, version, sb = new StringBuilder()) {
+    generate(cluster, version = IgniteVersion.ignite) {
         const caches = cluster.caches;
         const deps = [];
         const storeDeps = [];
@@ -162,6 +162,8 @@ export default class IgniteMavenGenerator {
                 this.addDependency(deps, 'org.apache.ignite', 'ignite-extdata-p2p', version);
         });
 
+        const sb = new StringBuilder();
+
         sb.append('<?xml version="1.0" encoding="UTF-8"?>');
 
         sb.emptyLine();
@@ -229,6 +231,6 @@ export default class IgniteMavenGenerator {
 
         this.build(sb, cluster, excludeGroupIds);
 
-        return sb;
+        return sb.asString();
     }
 }
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
index 6b240017ab017..070b6ce87aa69 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
@@ -17,8 +17,6 @@
 
 import JSZip from 'jszip';
 
-import IgniteVersion from 'app/modules/configuration/Version.service';
-
 import MavenGenerator from 'app/modules/configuration/generator/Maven.service';
 import DockerGenerator from 'app/modules/configuration/generator/Docker.service';
 import ReadmeGenerator from 'app/modules/configuration/generator/Readme.service';
@@ -28,8 +26,6 @@ import ConfigurationGenerator from 'app/modules/configuration/generator/Configur
 import JavaTransformer from 'app/modules/configuration/generator/JavaTransformer.service';
 import SpringTransformer from 'app/modules/configuration/generator/SpringTransformer.service';
 
-const Version = new IgniteVersion();
-
 const maven = new MavenGenerator();
 const docker = new DockerGenerator();
 const readme = new ReadmeGenerator();
@@ -100,7 +96,7 @@ onmessage = function(e) {
     zip.file(`${startupPath}/ClientNodeCodeStartup.java`, java.nodeStartup(cluster, 'startup.ClientNodeCodeStartup',
         'ClientConfigurationFactory.createConfiguration()', 'config.ClientConfigurationFactory', clientNearCaches));
 
-    zip.file('pom.xml', maven.generate(cluster, Version.productVersion().ignite).asString());
+    zip.file('pom.xml', maven.generate(cluster));
 
     zip.file('README.txt', readme.generate());
     zip.file('jdbc-drivers/README.txt', readme.generateJDBC());
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 651f496fa78c7..2d126550f16b4 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -29,100 +29,100 @@
     "win32"
   ],
   "dependencies": {
-    "angular": "~1.5.9",
-    "angular-acl": "~0.1.7",
-    "angular-animate": "~1.5.9",
-    "angular-aria": "~1.5.9",
-    "angular-cookies": "~1.5.9",
-    "angular-drag-and-drop-lists": "~1.4.0",
-    "angular-gridster": "~0.13.3",
-    "angular-motion": "~0.4.4",
-    "angular-nvd3": "~1.0.9",
-    "angular-retina": "~0.3.13",
-    "angular-sanitize": "~1.5.9",
-    "angular-smart-table": "~2.1.8",
-    "angular-socket-io": "~0.7.0",
-    "angular-strap": "~2.3.8",
-    "angular-touch": "~1.5.9",
-    "angular-translate": "~2.13.1",
-    "angular-tree-control": "~0.2.26",
-    "angular-ui-grid": "~4.0.0",
-    "angular-ui-router": "~0.3.1",
-    "bootstrap-sass": "~3.3.6",
-    "brace": "~0.8.0",
-    "es6-promise": "~3.3.1",
-    "file-saver": "~1.3.2",
-    "font-awesome": "~4.7.0",
-    "glob": "~7.1.1",
-    "jquery": "~3.1.1",
-    "jszip": "~3.1.3",
-    "lodash": "~4.17.2",
+    "angular": "1.5.11",
+    "angular-acl": "0.1.7",
+    "angular-animate": "1.5.11",
+    "angular-aria": "1.5.11",
+    "angular-cookies": "1.5.11",
+    "angular-drag-and-drop-lists": "1.4.0",
+    "angular-gridster": "0.13.14",
+    "angular-motion": "0.4.4",
+    "angular-nvd3": "1.0.9",
+    "angular-retina": "0.4.0",
+    "angular-sanitize": "1.5.11",
+    "angular-smart-table": "2.1.8",
+    "angular-socket-io": "0.7.0",
+    "angular-strap": "2.3.12",
+    "angular-touch": "1.5.11",
+    "angular-translate": "2.14.0",
+    "angular-tree-control": "0.2.28",
+    "angular-ui-grid": "4.0.2",
+    "angular-ui-router": "0.4.2",
+    "bootstrap-sass": "3.3.7",
+    "brace": "0.8.0",
+    "es6-promise": "3.3.1",
+    "file-saver": "1.3.3",
+    "font-awesome": "4.7.0",
+    "glob": "7.1.1",
+    "jquery": "3.1.1",
+    "jszip": "3.1.3",
+    "lodash": "4.17.4",
     "nvd3": "1.8.4",
-    "raleway-webfont": "~3.0.1",
-    "roboto-font": "~0.1.0",
-    "socket.io-client": "~1.7.2",
-    "ui-router-metatags": "~1.0.3"
+    "raleway-webfont": "3.0.1",
+    "roboto-font": "0.1.0",
+    "socket.io-client": "1.7.2",
+    "ui-router-metatags": "1.0.3"
   },
   "devDependencies": {
-    "assets-webpack-plugin": "~3.5.0",
-    "autoprefixer-core": "~6.0.1",
-    "babel-core": "~6.20.0",
-    "babel-eslint": "~7.0.0",
-    "babel-loader": "~6.2.4",
-    "babel-plugin-add-module-exports": "~0.2.1",
-    "babel-plugin-transform-builtin-extend": "~1.1.0",
-    "babel-plugin-transform-runtime": "~6.15.0",
-    "babel-polyfill": "~6.20.0",
-    "babel-preset-angular": "~6.0.15",
-    "babel-preset-es2015": "~6.18.0",
-    "babel-runtime": "~6.20.0",
-    "chai": "~3.5.0",
-    "cross-env": "~1.0.7",
-    "css-loader": "~0.23.0",
-    "eslint": "~3.12.2",
-    "eslint-friendly-formatter": "~2.0.5",
-    "eslint-loader": "~1.6.1",
-    "expose-loader": "~0.7.1",
-    "extract-text-webpack-plugin": "~1.0.1",
-    "file-loader": "~0.9.0",
-    "gulp": "~3.9.1",
-    "gulp-eslint": "~3.0.0",
-    "gulp-inject": "~4.1.0",
-    "gulp-jade": "~1.1.0",
-    "gulp-ll": "~1.0.4",
-    "gulp-rimraf": "~0.2.0",
-    "gulp-sequence": "~0.4.1",
-    "gulp-util": "~3.0.7",
-    "html-loader": "~0.4.3",
-    "html-webpack-plugin": "~2.24.1",
-    "jade": "~1.11.0",
+    "assets-webpack-plugin": "3.5.1",
+    "autoprefixer-core": "6.0.1",
+    "babel-core": "6.20.0",
+    "babel-eslint": "7.0.0",
+    "babel-loader": "6.2.10",
+    "babel-plugin-add-module-exports": "0.2.1",
+    "babel-plugin-transform-builtin-extend": "1.1.2",
+    "babel-plugin-transform-runtime": "6.15.0",
+    "babel-polyfill": "6.20.0",
+    "babel-preset-angular": "6.0.15",
+    "babel-preset-es2015": "6.18.0",
+    "babel-runtime": "6.20.0",
+    "chai": "3.5.0",
+    "cross-env": "1.0.8",
+    "css-loader": "0.26.1",
+    "eslint": "3.12.2",
+    "eslint-friendly-formatter": "2.0.7",
+    "eslint-loader": "1.6.1",
+    "expose-loader": "0.7.1",
+    "extract-text-webpack-plugin": "1.0.1",
+    "file-loader": "0.9.0",
+    "gulp": "3.9.1",
+    "gulp-eslint": "3.0.1",
+    "gulp-inject": "4.1.0",
+    "gulp-jade": "1.1.0",
+    "gulp-ll": "1.0.4",
+    "gulp-rimraf": "0.2.1",
+    "gulp-sequence": "0.4.6",
+    "gulp-util": "3.0.8",
+    "html-loader": "0.4.4",
+    "html-webpack-plugin": "2.24.1",
+    "jade": "1.11.0",
     "jade-html-loader": "git://github.com/courcelan/jade-html-loader",
-    "jasmine-core": "~2.5.2",
-    "json-loader": "~0.5.4",
-    "karma": "~0.13.22",
-    "karma-babel-preprocessor": "~6.0.1",
-    "karma-jasmine": "~1.1.0",
-    "karma-mocha": "~1.3.0",
-    "karma-mocha-reporter": "~2.2.0",
-    "karma-phantomjs-launcher": "~1.0.0",
-    "karma-teamcity-reporter": "~1.0.0",
-    "karma-webpack": "~1.8.0",
-    "mocha": "~2.5.3",
-    "mocha-teamcity-reporter": "~1.1.1",
-    "morgan": "~1.7.0",
-    "ngtemplate-loader": "~1.3.1",
-    "node-sass": "~3.13.1",
-    "phantomjs-prebuilt": "~2.1.7",
-    "postcss-loader": "~0.9.1",
-    "progress-bar-webpack-plugin": "~1.9.0",
-    "require-dir": "~0.3.0",
-    "resolve-url-loader": "~1.6.1",
-    "sass-loader": "~3.1.1",
-    "style-loader": "~0.13.1",
-    "url": "~0.11.0",
-    "url-loader": "~0.5.6",
-    "webpack": "~1.14.0",
-    "webpack-dev-server": "~1.16.2",
-    "worker-loader": "~0.7.1"
+    "jasmine-core": "2.5.2",
+    "json-loader": "0.5.4",
+    "karma": "0.13.22",
+    "karma-babel-preprocessor": "6.0.1",
+    "karma-jasmine": "1.1.0",
+    "karma-mocha": "1.3.0",
+    "karma-mocha-reporter": "2.2.2",
+    "karma-phantomjs-launcher": "1.0.2",
+    "karma-teamcity-reporter": "1.0.0",
+    "karma-webpack": "1.8.1",
+    "mocha": "2.5.3",
+    "mocha-teamcity-reporter": "1.1.1",
+    "morgan": "1.7.0",
+    "ngtemplate-loader": "1.3.1",
+    "node-sass": "3.13.1",
+    "phantomjs-prebuilt": "2.1.14",
+    "postcss-loader": "0.9.1",
+    "progress-bar-webpack-plugin": "1.9.3",
+    "require-dir": "0.3.1",
+    "resolve-url-loader": "1.6.1",
+    "sass-loader": "3.2.2",
+    "style-loader": "0.13.1",
+    "url": "0.11.0",
+    "url-loader": "0.5.7",
+    "webpack": "1.14.0",
+    "webpack-dev-server": "1.16.3",
+    "worker-loader": "0.7.1"
   }
 }
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index 67cfed176d39e..ab7e3dd94286f 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -2331,14 +2331,6 @@ html,body,.splash-screen {
     .ui-grid-cell-contents > i {
         line-height: $line-height-base;
     }
-
-    .ui-grid-row:nth-child(odd):hover .ui-grid-cell {
-        background: $ignite-row-hover;
-    }
-    
-    .ui-grid-row:nth-child(even):hover .ui-grid-cell {
-        background: $ignite-row-hover;
-    }
 }
 
 .datepicker.dropdown-menu {
diff --git a/modules/web-console/frontend/public/stylesheets/variables.scss b/modules/web-console/frontend/public/stylesheets/variables.scss
index e30bbddf9a6e4..8500eace44e41 100644
--- a/modules/web-console/frontend/public/stylesheets/variables.scss
+++ b/modules/web-console/frontend/public/stylesheets/variables.scss
@@ -26,4 +26,3 @@ $ignite-border-bottom-color: $brand-primary;
 $ignite-background-color: #fff;
 $ignite-header-color: #555;
 $ignite-invalid-color: $brand-primary;
-$ignite-row-hover: #c9dde1;
diff --git a/modules/web-console/frontend/views/templates/agent-download.jade b/modules/web-console/frontend/views/templates/agent-download.jade
index f57bf1d87e21c..f5a6ba06d35e7 100644
--- a/modules/web-console/frontend/views/templates/agent-download.jade
+++ b/modules/web-console/frontend/views/templates/agent-download.jade
@@ -23,10 +23,10 @@
                     span(ng-if='!hasAgents') Connection to Ignite Web Agent is not established
                     span(ng-if='hasAgents') Connection to Ignite Node is not established
             .agent-download(ng-if='!hasAgents')
-                p Please download and run #[a(href='javascript:void(0)' ng-click='downloadAgent()') ignite-web-agent] in order to {{::agentGoal}}
+                p Please download and run #[a(href='/api/v1/agent/download/zip' target='_self') ignite-web-agent] in order to {{::agentGoal}}
                 p For run:
                 ul
-                    li Download and unzip #[a(href='javascript:void(0)' ng-click='downloadAgent()') ignite-web-agent] archive
+                    li Download and unzip #[a(href='/api/v1/agent/download/zip' target='_self') ignite-web-agent] archive
                     li Run shell file #[b ignite-web-agent.{sh|bat}]
                 p Refer to #[b README.txt] in agent folder for more information
                 .modal-advanced-options
@@ -47,4 +47,4 @@
                     li Refer to #[b README.txt] in agent folder for more information
             .modal-footer
                 button.btn.btn-default(ng-click='back()') {{::backText}}
-                button.btn.btn-primary(ng-if='!hasAgents' ng-click='downloadAgent()') Download agent
+                a.btn.btn-primary(ng-if='!hasAgents' href='/api/v1/agent/download/zip' target='_self') Download agent

From db9252c9588c671279664484bb8c397312d265e6 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 15 Feb 2017 16:08:57 +0700
Subject: [PATCH 156/446] IGNITE-4678 Node version in range.

(cherry picked from commit 2cfd55d)
---
 .../modules/configuration/Version.service.js  | 30 ++++++++++++++++---
 .../frontend/test/unit/Version.test.js        | 26 +++++++++++++++-
 .../demo/service/DemoCachesLoadService.java   |  2 +-
 .../service/DemoRandomCacheLoadService.java   |  2 +-
 4 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/configuration/Version.service.js b/modules/web-console/frontend/app/modules/configuration/Version.service.js
index 746b1edd9b090..3fc719946729f 100644
--- a/modules/web-console/frontend/app/modules/configuration/Version.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/Version.service.js
@@ -76,13 +76,35 @@ export default class IgniteVersion {
         return numberComparator(pa.revTs, pb.revTs);
     }
 
+    /**
+     * Check if node version in range
+     * @param {String} nodeVer Node version.
+     * @param {Array.<String>} ranges Version ranges to compare with.
+     * @returns {Boolean} `True` if node version is equal or greater than specified range.
+     */
+    includes(nodeVer, ...ranges) {
+        return !!_.find(ranges, ([after, before]) =>
+            this.compare(nodeVer, after) >= 0 && (_.isNil(before) || this.compare(nodeVer, before) < 0)
+        );
+    }
+
     /**
      * Check if node version is newer or same
-     * @param {String} nodeVer
-     * @param {String} sinceVer
-     * @returns {Boolean}
+     * @param {String} nodeVer Node version.
+     * @param {String} sinceVer Version to compare with.
+     * @returns {Boolean} `True` if node version is equal or greater than specified version.
      */
     since(nodeVer, sinceVer) {
-        return this.compare(nodeVer, sinceVer) >= 0;
+        return this.includes(nodeVer, [sinceVer]);
+    }
+
+    /**
+     * Check whether node version before than specified version.
+     * @param {String} nodeVer Node version.
+     * @param {String} sinceVer Version to compare with.
+     * @return {Boolean} `True` if node version before than specified version.
+     */
+    before(nodeVer, sinceVer) {
+        return !this.since(nodeVer, sinceVer);
     }
 }
diff --git a/modules/web-console/frontend/test/unit/Version.test.js b/modules/web-console/frontend/test/unit/Version.test.js
index 2d75ab583a2c6..72685eae03239 100644
--- a/modules/web-console/frontend/test/unit/Version.test.js
+++ b/modules/web-console/frontend/test/unit/Version.test.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import VersionService from '../../app/modules/configuration/Version.service.js';
+import VersionService from '../../app/modules/configuration/Version.service';
 
 const INSTANCE = new VersionService();
 
@@ -53,6 +53,7 @@ suite('VersionServiceTestsSuite', () => {
     });
 
     test('Check since call', () => {
+        assert.equal(INSTANCE.since('1.5.0', '1.5.0'), true);
         assert.equal(INSTANCE.since('1.6.0', '1.5.0'), true);
     });
 
@@ -60,6 +61,29 @@ suite('VersionServiceTestsSuite', () => {
         assert.equal(INSTANCE.since('1.3.0', '1.5.0'), false);
     });
 
+    test('Check before call', () => {
+        assert.equal(INSTANCE.before('1.5.0', '1.5.0'), false);
+        assert.equal(INSTANCE.before('1.5.0', '1.6.0'), true);
+    });
+
+    test('Check wrong before call', () => {
+        assert.equal(INSTANCE.before('1.5.0', '1.3.0'), false);
+    });
+
+    test('Check includes call', () => {
+        assert.equal(INSTANCE.includes('1.5.4', ['1.5.5', '1.6.0'], ['1.6.2']), false);
+        assert.equal(INSTANCE.includes('1.5.5', ['1.5.5', '1.6.0'], ['1.6.2']), true);
+        assert.equal(INSTANCE.includes('1.5.11', ['1.5.5', '1.6.0'], ['1.6.2']), true);
+        assert.equal(INSTANCE.includes('1.6.0', ['1.5.5', '1.6.0'], ['1.6.2']), false);
+        assert.equal(INSTANCE.includes('1.6.1', ['1.5.5', '1.6.0'], ['1.6.2']), false);
+        assert.equal(INSTANCE.includes('1.6.2', ['1.5.5', '1.6.0'], ['1.6.2']), true);
+        assert.equal(INSTANCE.includes('1.6.3', ['1.5.5', '1.6.0'], ['1.6.2']), true);
+    });
+
+    test('Check wrong before call', () => {
+        assert.equal(INSTANCE.before('1.5.0', '1.3.0'), false);
+    });
+
     test('Parse 1.7.0-SNAPSHOT', () => {
         const version = INSTANCE.parse('1.7.0-SNAPSHOT');
         assert.equal(version.major, 1);
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
index 911764604502f..fbfa2ae6e09e2 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
@@ -114,7 +114,7 @@ public DemoCachesLoadService(int cnt) {
     /** {@inheritDoc} */
     @Override public void cancel(ServiceContext ctx) {
         if (cachePool != null)
-            cachePool.shutdown();
+            cachePool.shutdownNow();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
index 57b26a2102ded..c704dbe09cf06 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
@@ -62,7 +62,7 @@ public DemoRandomCacheLoadService(int cnt) {
     /** {@inheritDoc} */
     @Override public void cancel(ServiceContext ctx) {
         if (cachePool != null)
-            cachePool.shutdown();
+            cachePool.shutdownNow();
     }
 
     /** {@inheritDoc} */

From 58c0c49d31605bf4608e7ee97099b75b324a782f Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 16 Feb 2017 10:41:30 +0700
Subject: [PATCH 157/446] IGNITE-4472 Fixed became this user.

(cherry picked from commit ee832e4)
---
 modules/web-console/backend/routes/profile.js              | 4 ++--
 modules/web-console/backend/services/sessions.js           | 6 +-----
 modules/web-console/backend/services/users.js              | 7 ++-----
 .../frontend/views/templates/agent-download.jade           | 4 ++--
 4 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/modules/web-console/backend/routes/profile.js b/modules/web-console/backend/routes/profile.js
index 1d6fccb8213f3..76edf721fde56 100644
--- a/modules/web-console/backend/routes/profile.js
+++ b/modules/web-console/backend/routes/profile.js
@@ -50,7 +50,7 @@ module.exports.factory = function(_, express, mongo, usersService) {
                     if (becomeUsed) {
                         req.session.viewedUser = user;
 
-                        return user;
+                        return req.user;
                     }
 
                     return new Promise((resolve, reject) => {
@@ -64,7 +64,7 @@ module.exports.factory = function(_, express, mongo, usersService) {
                         });
                     });
                 })
-                .then(() => usersService.get(req.user, req.session.viewedUser))
+                .then((user) => usersService.get(user, req.session.viewedUser))
                 .then(res.api.ok)
                 .catch(res.api.error);
         });
diff --git a/modules/web-console/backend/services/sessions.js b/modules/web-console/backend/services/sessions.js
index 7f62a606c9369..7f3fc7351b19b 100644
--- a/modules/web-console/backend/services/sessions.js
+++ b/modules/web-console/backend/services/sessions.js
@@ -42,11 +42,7 @@ module.exports.factory = (_, mongo, errors) => {
                 return Promise.reject(new errors.IllegalAccessError('Became this user is not permitted. Only administrators can perform this actions.'));
 
             return mongo.Account.findById(viewedUserId).lean().exec()
-                .then((viewedUser) => {
-                    viewedUser.token = session.req.user.token;
-
-                    session.viewedUser = viewedUser;
-                });
+                .then((viewedUser) => session.viewedUser = viewedUser);
         }
 
         /**
diff --git a/modules/web-console/backend/services/users.js b/modules/web-console/backend/services/users.js
index 2dd603f193382..0aff45f15356b 100644
--- a/modules/web-console/backend/services/users.js
+++ b/modules/web-console/backend/services/users.js
@@ -212,11 +212,8 @@ module.exports.factory = (_, mongo, settings, spacesService, mailsService, activ
 
             const becomeUsed = viewedUser && user.admin;
 
-            if (becomeUsed) {
-                user = viewedUser;
-
-                user.becomeUsed = true;
-            }
+            if (becomeUsed)
+                user = _.extend({}, viewedUser, {becomeUsed: true, becameToken: user.token});
             else
                 user = user.toJSON();
 
diff --git a/modules/web-console/frontend/views/templates/agent-download.jade b/modules/web-console/frontend/views/templates/agent-download.jade
index f5a6ba06d35e7..b913636cb0fcd 100644
--- a/modules/web-console/frontend/views/templates/agent-download.jade
+++ b/modules/web-console/frontend/views/templates/agent-download.jade
@@ -34,8 +34,8 @@
                     i.fa.fa-chevron-circle-right(ng-show='!agentLoad.showToken' ng-click='agentLoad.showToken = ! agentLoad.showToken')
                     a(ng-click='agentLoad.showToken = ! agentLoad.showToken') {{agentLoad.showToken ? 'Hide security token...' : 'Show security token...'}}
                 .details-row(ng-show='agentLoad.showToken')
-                    label.labelField Security token: {{user.token}}
-                    i.tipLabel.fa.fa-clipboard(ignite-copy-to-clipboard='{{user.token}}' bs-tooltip='' data-title='Copy security token to clipboard')
+                    label.labelField Security token: {{user.becameToken || user.token}}
+                    i.tipLabel.fa.fa-clipboard(ignite-copy-to-clipboard='{{user.becameToken || user.token}}' bs-tooltip='' data-title='Copy security token to clipboard')
                     i.tipLabel.icon-help(ng-if=lines bs-tooltip='' data-title='The security token is used for authorization of web agent')
             .agent-download(ng-if='hasAgents')
                 p Connection to Ignite Web Agent is established, but agent failed to connect to Ignite Node

From 2ccbf32ea3ecaff1068832accf37235a32734b4b Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 16 Feb 2017 14:22:22 +0700
Subject: [PATCH 158/446] IGNITE-4472 Minor UI fix.

(cherry picked from commit 97c7ed7)
---
 .../list-of-registered-users.column-defs.js   |  8 ++--
 .../list-of-registered-users.controller.js    | 18 +++++++++
 .../list-of-registered-users.jade             | 16 +++++---
 .../ui-grid-settings/ui-grid-settings.scss    | 39 ++++++++++++++++---
 4 files changed, 66 insertions(+), 15 deletions(-)

diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
index 4dc465505b6ab..e6ba842aa055b 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
@@ -49,7 +49,7 @@ const ACTIONS_TEMPLATE = `
 const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
 
 export default [
-    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, pinnedLeft: true},
+    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 65, width: 65, enableFiltering: false, enableSorting: false, pinnedLeft: true},
     {displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by name...' }, pinnedLeft: true},
     {displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
     {displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 160, enableFiltering: true},
@@ -64,9 +64,9 @@ export default [
     // Activities Total
     {displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Total count of configuration usages', minWidth: 50, width: 50, enableFiltering: false},
     {displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Total count of queries usages', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Total count of demo startup', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Total count of agent downloads', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Str', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Total count of agent startup', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Total count of demo startup', minWidth: 60, width: 60, enableFiltering: false},
+    {displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Total count of agent downloads', minWidth: 55, width: 55, enableFiltering: false},
+    {displayName: 'Starts', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Total count of agent startup', minWidth: 60, width: 60, enableFiltering: false},
     // Activities Configuration
     {displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration clusters', minWidth: 50, width: 80, enableFiltering: false, visible: false},
     {displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration model', minWidth: 50, width: 80, enableFiltering: false, visible: false},
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 1f2a348daad7a..5761073509d7e 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -96,6 +96,18 @@ export default class IgniteListOfRegisteredUsersCtrl {
             return new ActivitiesUserDialog({ user });
         };
 
+        const companiesExcludeFilter = (renderableRows) => {
+            if (_.isNil($ctrl.params.companiesExclude))
+                return renderableRows;
+
+            _.forEach(renderableRows, (row) => {
+                row.visible = _.isEmpty($ctrl.params.companiesExclude) ||
+                    row.entity.company.toLowerCase().indexOf($ctrl.params.companiesExclude.toLowerCase()) === -1;
+            });
+
+            return renderableRows;
+        };
+
         $ctrl.gridOptions = {
             data: [],
             columnVirtualizationThreshold: 30,
@@ -120,6 +132,8 @@ export default class IgniteListOfRegisteredUsersCtrl {
                 api.removeUser = removeUser;
                 api.toggleAdmin = toggleAdmin;
                 api.showActivities = showActivities;
+
+                api.grid.registerRowsProcessor(companiesExcludeFilter, 300);
             }
         };
 
@@ -153,6 +167,10 @@ export default class IgniteListOfRegisteredUsersCtrl {
                 .then((data) => $ctrl.adjustHeight(data.length));
         };
 
+        $scope.$watch(() => $ctrl.params.companiesExclude, () => {
+            $ctrl.gridApi.grid.refreshRows();
+        });
+
         $scope.$watch(() => $ctrl.params.startDate, (dt) => {
             $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(dt, 'yyyy_MM')}.csv`;
 
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
index efed9c0edee16..119591036d9da 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
@@ -39,16 +39,20 @@ mixin grid-settings()
         +grid-settings
         label Total users: 
             strong {{ $ctrl.gridOptions.data.length }}&nbsp;&nbsp;&nbsp;
-        label Showing users: 
+        label Showing users:
             strong {{ $ctrl.gridApi.grid.getVisibleRows().length }}
             sub(ng-show='users.length === $ctrl.gridApi.grid.getVisibleRows().length') all
-        div.ui-grid-settings-dateperiod
-            form(ng-form=form novalidate)
-                -var form = 'admin'
 
+        form.pull-right(ng-form=form novalidate)
+            -var form = 'admin'
+
+            button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
+
+            .ui-grid-settings-dateperiod
                 +ignite-form-field-datepicker('Period:', '$ctrl.params.startDate', '"period"')
-                
-                button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
+
+            .ui-grid-settings-filter
+                +ignite-form-field-text('Exclude:', '$ctrl.params.companiesExclude', '"exclude"', false, false, 'Exclude by company name...')
 
     .panel-collapse
         .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-exporter ui-grid-pinning)
diff --git a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
index 3016488f6bb6e..bc16271a57bc0 100644
--- a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
@@ -36,6 +36,40 @@
         cursor: pointer;
     }
 
+    .btn {
+        float: right;
+
+        line-height: 20px;
+        margin-right: 0;
+    }
+
+    &-filter {
+        float: right;
+
+        .ignite-form-field {
+            width: 260px;
+            margin-right: 10px;
+
+            &__label {
+            }
+
+            &__control {
+            }
+
+            &:nth-child(1) {
+                float: left;
+
+                .ignite-form-field__label {
+                    width: 30%;
+                }
+
+                .ignite-form-field__control {
+                    width: 70%;
+                }
+            }
+        }
+    }
+
     &-dateperiod {
         float: right;
 
@@ -61,10 +95,5 @@
                 }
             }
         }
-
-        .btn {
-            line-height: 20px;
-            margin-right: 0;
-        }
     }
 }

From ef4886d5be7c708b917e97b1cd5fd766b2ad8450 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 16 Feb 2017 18:38:40 +0700
Subject: [PATCH 159/446] IGNITE-4472 Minor UI fix.

(cherry picked from commit d4efbf3)
---
 .../list-of-registered-users.controller.js                      | 2 +-
 .../app/components/ui-grid-settings/ui-grid-settings.scss       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 5761073509d7e..272681a5f576b 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -133,7 +133,7 @@ export default class IgniteListOfRegisteredUsersCtrl {
                 api.toggleAdmin = toggleAdmin;
                 api.showActivities = showActivities;
 
-                api.grid.registerRowsProcessor(companiesExcludeFilter, 300);
+                api.grid.registerRowsProcessor(companiesExcludeFilter, 50);
             }
         };
 
diff --git a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
index bc16271a57bc0..4beb2a1baa40a 100644
--- a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
@@ -47,7 +47,7 @@
         float: right;
 
         .ignite-form-field {
-            width: 260px;
+            width: 280px;
             margin-right: 10px;
 
             &__label {

From 05788b3188b30b5a3b39a75fe66301e03658408f Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 17 Feb 2017 12:14:53 +0300
Subject: [PATCH 160/446] IGNITE-3429: Added BinaryResolver configuration
 samples for  org.hibernate.cache.spi.CacheKey. This closes #1516.

---
 .../Hibernate5CacheKeyTypeConfiguration.java  | 52 +++++++++++++++++++
 .../HibernateCacheKeyTypeConfiguration.java   | 51 ++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java

diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
new file mode 100644
index 0000000000000..886f69b2500ba
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate.config;
+
+import java.util.Objects;
+import org.apache.ignite.binary.BinaryAbstractIdentityResolver;
+import org.apache.ignite.binary.BinaryIdentityResolver;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryTypeConfiguration;
+
+/**
+ * This configuration provides correct {@link BinaryIdentityResolver} implementation
+ * for Hibernate CacheKey class can be used as a key object.
+ *
+ * Note: for Hibernate version < 5.0 {@link HibernateCacheKeyTypeConfiguration} should be used.
+
+ */
+public class Hibernate5CacheKeyTypeConfiguration extends BinaryTypeConfiguration {
+
+    /** {@inheritDoc} */
+    public Hibernate5CacheKeyTypeConfiguration() {
+        super("org.hibernate.cache.internal.CacheKeyImplementation");
+
+        setIdentityResolver(new BinaryAbstractIdentityResolver() {
+            @Override protected int hashCode0(BinaryObject obj) {
+                return obj.field("id").hashCode();
+            }
+
+            @Override protected boolean equals0(BinaryObject o1, BinaryObject o2) {
+                Object obj0 = o1.field("id");
+                Object obj1 = o2.field("id");
+
+                return Objects.equals(obj0, obj1);
+            }
+        });
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java
new file mode 100644
index 0000000000000..c54292e6f40e6
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate.config;
+
+import java.util.Objects;
+import org.apache.ignite.binary.BinaryAbstractIdentityResolver;
+import org.apache.ignite.binary.BinaryIdentityResolver;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryTypeConfiguration;
+
+/**
+ * This configuration provides correct {@link BinaryIdentityResolver} implementation
+ * for Hibernate CacheKey class can be used as a key object.
+ *
+ * Note: for Hibernate version >= 5.0 {@link Hibernate5CacheKeyTypeConfiguration} should be used.
+ */
+public class HibernateCacheKeyTypeConfiguration extends BinaryTypeConfiguration {
+
+    /** {@inheritDoc} */
+    public HibernateCacheKeyTypeConfiguration() {
+        super("org.hibernate.cache.spi.CacheKey");
+
+        setIdentityResolver(new BinaryAbstractIdentityResolver() {
+            @Override protected int hashCode0(BinaryObject obj) {
+                return obj.field("key").hashCode();
+            }
+
+            @Override protected boolean equals0(BinaryObject o1, BinaryObject o2) {
+                Object obj0 = o1.field("key");
+                Object obj1 = o2.field("key");
+
+                return Objects.equals(obj0, obj1);
+            }
+        });
+    }
+}

From 1f881aa70a3894af01135f4cc5e341a8130462c2 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 17 Feb 2017 12:34:41 +0300
Subject: [PATCH 161/446] IGNITE-4147 - Throw exception on connecting node to
 cluster with different SSL configuration

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  30 ++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  19 ++
 .../TcpDiscoverySslSecuredUnsecuredTest.java  | 185 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   2 +
 4 files changed, 235 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 932e7d1e754a1..95e2cda4f6907 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -20,6 +20,7 @@
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StreamCorruptedException;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
@@ -44,6 +45,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.atomic.AtomicReference;
+import javax.net.ssl.SSLException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
@@ -587,6 +589,8 @@ else if (addrs.isEmpty()) {
 
         int connectAttempts = 1;
 
+        int sslConnectAttempts = 3;
+
         UUID locNodeId = getLocalNodeId();
 
         IgniteSpiOperationTimeoutHelper timeoutHelper = new IgniteSpiOperationTimeoutHelper(spi);
@@ -662,6 +666,22 @@ else if (addrs.isEmpty()) {
 
                 errs.add(e);
 
+                if (X.hasCause(e, SSLException.class)) {
+                    if (--sslConnectAttempts == 0)
+                        throw new IgniteSpiException("Unable to establish secure connection. " +
+                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
+                    continue;
+                }
+
+                if (X.hasCause(e, StreamCorruptedException.class)) {
+                    if (--sslConnectAttempts == 0)
+                        throw new IgniteSpiException("Unable to establish plain connection. " +
+                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
+                    continue;
+                }
+
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
@@ -1593,7 +1613,15 @@ private void tryJoin() throws InterruptedException {
 
             joinCnt++;
 
-            T2<SocketStream, Boolean> joinRes = joinTopology(false, spi.joinTimeout);
+            T2<SocketStream, Boolean> joinRes;
+            try {
+                joinRes = joinTopology(false, spi.joinTimeout);
+            }
+            catch (IgniteSpiException e) {
+                joinError(e);
+
+                return;
+            }
 
             if (joinRes == null) {
                 if (join)
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index d462ac2638fd2..4600be094c840 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -24,6 +24,7 @@
 import java.io.ObjectStreamException;
 import java.io.OutputStream;
 import java.io.Serializable;
+import java.io.StreamCorruptedException;
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -1109,6 +1110,8 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
         int connectAttempts = 1;
 
+        int sslConnectAttempts = 3;
+
         boolean joinReqSent;
 
         UUID locNodeId = getLocalNodeId();
@@ -1220,6 +1223,22 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
                 errs.add(e);
 
+                if (X.hasCause(e, SSLException.class)) {
+                    if (--sslConnectAttempts == 0)
+                        throw new IgniteException("Unable to establish secure connection. " +
+                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
+                    continue;
+                }
+
+                if (X.hasCause(e, StreamCorruptedException.class)) {
+                    if (--sslConnectAttempts == 0)
+                        throw new IgniteException("Unable to establish plain connection. " +
+                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
+                    continue;
+                }
+
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java
new file mode 100644
index 0000000000000..ca34f779760cf
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StreamCorruptedException;
+import java.net.Socket;
+import java.util.concurrent.Callable;
+import javax.net.ssl.SSLException;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Tests cases when node connects to cluster with different SSL configuration.
+ * Exception with meaningful message should be thrown.
+ */
+public class TcpDiscoverySslSecuredUnsecuredTest extends GridCommonAbstractTest {
+    /** */
+    private volatile TcpDiscoverySpi spi;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(gridName.contains("client"));
+
+        if (gridName.contains("ssl"))
+            cfg.setSslContextFactory(GridTestUtils.sslFactory());
+
+        if (spi != null) {
+            final TcpDiscoveryIpFinder finder = ((TcpDiscoverySpi)cfg.getDiscoverySpi()).getIpFinder();
+
+            spi.setIpFinder(finder);
+
+            cfg.setDiscoverySpi(spi);
+        }
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSecuredUnsecuredServerConnection() throws Exception {
+        checkConnection("plain-server", "ssl-server");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testUnsecuredSecuredServerConnection() throws Exception {
+        checkConnection("ssl-server", "plain-server");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSecuredClientUnsecuredServerConnection() throws Exception {
+        checkConnection("plain-server", "ssl-client");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testUnsecuredClientSecuredServerConnection() throws Exception {
+        checkConnection("ssl-server", "plain-client");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPlainServerNodesRestart() throws Exception {
+        checkNodesRestart("plain-server-1", "plain-server-2");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSslServerNodesRestart() throws Exception {
+        checkNodesRestart("ssl-server-1", "ssl-server-2");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPlainClientNodesRestart() throws Exception {
+        checkNodesRestart("plain-server", "plain-client");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSslClientNodesRestart() throws Exception {
+        checkNodesRestart("ssl-server", "ssl-client");
+    }
+
+    /**
+     * @param name1 First grid name.
+     * @param name2 Second grid name.
+     * @throws Exception If failed.
+     */
+    private void checkNodesRestart(String name1, String name2) throws Exception {
+        startGrid(name1);
+
+        spi = new FailDiscoverySpi(!name1.contains("ssl"));
+
+        startGrid(name2);
+    }
+
+    /**
+     * @param name1 First grid name.
+     * @param name2 Second grid name.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    private void checkConnection(final String name1, final String name2) throws Exception {
+        startGrid(name1);
+
+        GridTestUtils.assertThrows(null, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                startGrid(name2);
+
+                return null;
+            }
+        }, IgniteCheckedException.class, null);
+    }
+
+    /**
+     *
+     */
+    private class FailDiscoverySpi extends TcpDiscoverySpi {
+        /** */
+        private int cnt = 1;
+
+        /** */
+        private final boolean plain;
+
+        /**
+         * @param plain Plain conection flag.
+         */
+        private FailDiscoverySpi(final boolean plain) {
+            this.plain = plain;
+        }
+
+        /** {@inheritDoc} */
+        @Override protected <T> T readMessage(final Socket sock, @Nullable final InputStream in,
+            final long timeout) throws IOException, IgniteCheckedException {
+            if (cnt-- > 0) {
+                if (plain)
+                    throw new StreamCorruptedException("Test exception");
+                else
+                    throw new SSLException("Test SSL exception");
+            }
+
+            return super.readMessage(sock, in, timeout);
+        }
+    }
+
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 548e1a5a51f81..e6b39f740a0dd 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiStartStopSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSecuredUnsecuredTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSelfTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.jdbc.TcpDiscoveryJdbcIpFinderSelfTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinderSelfTest;
@@ -89,6 +90,7 @@ public static TestSuite suite() throws Exception {
 
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
+        suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));
 
         return suite;
     }

From 11bbec487dc174fac1acd6b50c940840305bc75a Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 17 Feb 2017 17:57:50 +0700
Subject: [PATCH 162/446] IGNITE-4436 API for collecting list of running
 queries and cancel them. (cherry picked from commit
 49237343d53ee33d44e5599cd7fe7da868ee30a1)

---
 .../cache/query/GridCacheTwoStepQuery.java    |  18 +-
 .../processors/query/GridQueryIndexing.java   |  17 +-
 .../processors/query/GridQueryProcessor.java  |  26 ++-
 .../query/GridRunningQueryInfo.java           | 132 +++++++++++
 .../internal/visor/VisorMultiNodeTask.java    |   2 +-
 .../visor/query/VisorCancelQueriesTask.java   |  72 ++++++
 .../query/VisorCollectRunningQueriesTask.java |  96 ++++++++
 .../visor/query/VisorRunningQuery.java        | 132 +++++++++++
 .../processors/query/h2/IgniteH2Indexing.java |  83 ++++++-
 .../query/h2/sql/GridSqlQuerySplitter.java    |   4 +-
 .../h2/twostep/GridReduceQueryExecutor.java   |  60 ++++-
 .../cache/CacheSqlQueryValueCopySelfTest.java | 208 +++++++++++++++++-
 .../GridCacheCrossCacheQuerySelfTest.java     |   2 +-
 .../h2/GridIndexingSpiAbstractSelfTest.java   |   7 +
 14 files changed, 821 insertions(+), 38 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
index 8dcba2f2f34dc..f53936fc374a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
@@ -45,6 +45,9 @@ public class GridCacheTwoStepQuery {
     /** */
     private boolean explain;
 
+    /** */
+    private String originalSql;
+
     /** */
     private Collection<String> spaces;
 
@@ -67,10 +70,12 @@ public class GridCacheTwoStepQuery {
     private List<Integer> extraCaches;
 
     /**
+     * @param originalSql Original query SQL.
      * @param schemas Schema names in query.
      * @param tbls Tables in query.
      */
-    public GridCacheTwoStepQuery(Set<String> schemas, Set<String> tbls) {
+    public GridCacheTwoStepQuery(String originalSql, Set<String> schemas, Set<String> tbls) {
+        this.originalSql = originalSql;
         this.schemas = schemas;
         this.tbls = tbls;
     }
@@ -195,6 +200,13 @@ public void extraCaches(List<Integer> extraCaches) {
         this.extraCaches = extraCaches;
     }
 
+    /**
+     * @return Original query SQL.
+     */
+    public String originalSql() {
+        return originalSql;
+    }
+
     /**
      * @return Spaces.
      */
@@ -223,7 +235,7 @@ public Set<String> schemas() {
     public GridCacheTwoStepQuery copy(Object[] args) {
         assert !explain;
 
-        GridCacheTwoStepQuery cp = new GridCacheTwoStepQuery(schemas, tbls);
+        GridCacheTwoStepQuery cp = new GridCacheTwoStepQuery(originalSql, schemas, tbls);
 
         cp.caches = caches;
         cp.extraCaches = extraCaches;
@@ -250,4 +262,4 @@ public Set<String> tables() {
     @Override public String toString() {
         return S.toString(GridCacheTwoStepQuery.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index 539ebc07fec79..1cebbb0a54ff6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -223,8 +223,23 @@ public void store(@Nullable String spaceName, GridQueryTypeDescriptor type, Cach
      */
     public void onDisconnected(IgniteFuture<?> reconnectFut);
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> runningQueries(long duration);
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries ID's to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries);
+
     /**
      * Cancels all executing queries.
      */
     public void cancelAllQueries();
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index c2e5717fbddf8..0a0d16614a92b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -44,7 +44,6 @@
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryField;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryType;
@@ -118,7 +117,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
     private static final int QRY_DETAIL_METRICS_EVICTION_FREQ = 3_000;
 
     /** */
-    private static Set<Class<?>> SQL_TYPES = new HashSet<>(F.<Class<?>>asList(
+    private static final Set<Class<?>> SQL_TYPES = new HashSet<>(F.<Class<?>>asList(
         Integer.class,
         Boolean.class,
         Byte.class,
@@ -912,6 +911,29 @@ public <K, V> Iterator<Cache.Entry<K, V>> queryLocal(
         }
     }
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+        if (moduleEnabled())
+            return idx.runningQueries(duration);
+
+        return Collections.emptyList();
+    }
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries ID's to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries) {
+        if (moduleEnabled())
+            idx.cancelQueries(queries);
+    }
+
     /**
      * @param sqlQry Sql query.
      * @param params Params.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
new file mode 100644
index 0000000000000..d77c8c01a8ede
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
+
+/**
+ * Query descriptor.
+ */
+public class GridRunningQueryInfo {
+    /** */
+    private final long id;
+
+    /** */
+    private final String qry;
+
+    /** Query type. */
+    private final GridCacheQueryType qryType;
+
+    /** */
+    private final String cache;
+
+    /** */
+    private final long startTime;
+
+    /** */
+    private final GridQueryCancel cancel;
+
+    /** */
+    private final boolean loc;
+
+    /**
+     * @param id Query ID.
+     * @param qry Query text.
+     * @param qryType Query type.
+     * @param cache Cache where query was executed.
+     * @param startTime Query start time.
+     * @param cancel Query cancel.
+     * @param loc Local query flag.
+     */
+    public GridRunningQueryInfo(Long id, String qry, GridCacheQueryType qryType, String cache, long startTime,
+        GridQueryCancel cancel, boolean loc) {
+        this.id = id;
+        this.qry = qry;
+        this.qryType = qryType;
+        this.cache = cache;
+        this.startTime = startTime;
+        this.cancel = cancel;
+        this.loc = loc;
+    }
+
+    /**
+     * @return Query ID.
+     */
+    public Long id() {
+        return id;
+    }
+
+    /**
+     * @return Query text.
+     */
+    public String query() {
+        return qry;
+    }
+
+    /**
+     * @return Query type.
+     */
+    public GridCacheQueryType queryType() {
+        return qryType;
+    }
+
+    /**
+     * @return Cache where query was executed.
+     */
+    public String cache() {
+        return cache;
+    }
+
+    /**
+     * @return Query start time.
+     */
+    public long startTime() {
+        return startTime;
+    }
+
+    /**
+     * @param curTime Current time.
+     * @param duration Duration of long query.
+     * @return {@code true} if this query should be considered as long running query.
+     */
+    public boolean longQuery(long curTime, long duration) {
+        return curTime - startTime > duration;
+    }
+
+    /**
+     * Cancel query.
+     */
+    public void cancel() {
+        if (cancel != null)
+            cancel.cancel();
+    }
+
+    /**
+     * @return {@code true} if query can be cancelled.
+     */
+    public boolean cancelable() {
+        return cancel != null;
+    }
+
+    /**
+     * @return {@code true} if query is local.
+     */
+    public boolean local() {
+        return loc;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
index 57f134698ea56..ece1a17953f39 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
@@ -130,4 +130,4 @@ protected Map<? extends ComputeJob, ClusterNode> map0(List<ClusterNode> subgrid,
                 logFinish(ignite.log(), getClass(), start);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
new file mode 100644
index 0000000000000..a6f2d821f15a0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task to cancel queries.
+ */
+@GridInternal
+public class VisorCancelQueriesTask extends VisorOneNodeTask<Collection<Long>, Void> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCancelQueriesJob job(Collection<Long> arg) {
+        return new VisorCancelQueriesJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected Void reduce0(List<ComputeJobResult> results) throws IgniteException {
+        return null;
+    }
+
+    /**
+     * Job to cancel queries on node.
+     */
+    private static class VisorCancelQueriesJob extends VisorJob<Collection<Long>, Void> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with specified argument.
+         *
+         * @param arg Job argument.
+         * @param debug Flag indicating whether debug information should be printed into node log.
+         */
+        protected VisorCancelQueriesJob(@Nullable Collection<Long> arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Void run(@Nullable Collection<Long> queries) throws IgniteException {
+            ignite.context().query().cancelQueries(queries);
+
+            return null;
+        }
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
new file mode 100644
index 0000000000000..2b40e61e74f50
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task to collect currently running queries.
+ */
+@GridInternal
+public class VisorCollectRunningQueriesTask extends VisorMultiNodeTask<Long, Map<UUID, Collection<VisorRunningQuery>>, Collection<VisorRunningQuery>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCollectRunningQueriesJob job(Long arg) {
+        return new VisorCollectRunningQueriesJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected Map<UUID, Collection<VisorRunningQuery>> reduce0(List<ComputeJobResult> results) throws IgniteException {
+        Map<UUID, Collection<VisorRunningQuery>> map = new HashMap<>();
+
+        for (ComputeJobResult res : results)
+            if (res.getException() == null) {
+                Collection<VisorRunningQuery> queries = res.getData();
+
+                map.put(res.getNode().id(), queries);
+            }
+
+        return map;
+    }
+
+    /**
+     * Job to collect currently running queries from node.
+     */
+    private static class VisorCollectRunningQueriesJob extends VisorJob<Long, Collection<VisorRunningQuery>> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with specified argument.
+         *
+         * @param arg Job argument.
+         * @param debug Flag indicating whether debug information should be printed into node log.
+         */
+        protected VisorCollectRunningQueriesJob(@Nullable Long arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Collection<VisorRunningQuery> run(@Nullable Long duration) throws IgniteException {
+            Collection<GridRunningQueryInfo> queries = ignite.context().query()
+                .runningQueries(duration != null ? duration : 0);
+
+            Collection<VisorRunningQuery> res = new ArrayList<>(queries.size());
+
+            long curTime = U.currentTimeMillis();
+
+            for (GridRunningQueryInfo qry : queries)
+                res.add(new VisorRunningQuery(qry.id(), qry.query(), qry.queryType(), qry.cache(),
+                    qry.startTime(), curTime - qry.startTime(),
+                    qry.cancelable(), qry.local()));
+
+            return res;
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java
new file mode 100644
index 0000000000000..fc6bc7a9222d4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
+
+/**
+ * Descriptor of running query.
+ */
+public class VisorRunningQuery implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private long id;
+
+    /** Query text. */
+    private String qry;
+
+    /** Query type. */
+    private GridCacheQueryType qryType;
+
+    /** Cache name for query. */
+    private String cache;
+
+    /** */
+    private long startTime;
+
+    /** */
+    private long duration;
+
+    /** */
+    private boolean cancellable;
+
+    /** */
+    private boolean loc;
+
+    /**
+     * @param id Query ID.
+     * @param qry Query text.
+     * @param qryType Query type.
+     * @param cache Cache where query was executed.
+     * @param startTime Query start time.
+     * @param duration Query current duration.
+     * @param cancellable {@code true} if query can be canceled.
+     * @param loc {@code true} if query is local.
+     */
+    public VisorRunningQuery(long id, String qry, GridCacheQueryType qryType, String cache,
+        long startTime, long duration,
+        boolean cancellable, boolean loc) {
+        this.id = id;
+        this.qry = qry;
+        this.qryType = qryType;
+        this.cache = cache;
+        this.startTime = startTime;
+        this.duration = duration;
+        this.cancellable = cancellable;
+        this.loc = loc;
+    }
+
+    /**
+     * @return Query ID.
+     */
+    public long getId() {
+        return id;
+    }
+
+    /**
+     * @return Query txt.
+     */
+    public String getQuery() {
+        return qry;
+    }
+
+    /**
+     * @return Query type.
+     */
+    public GridCacheQueryType getQueryType() {
+        return qryType;
+    }
+
+    /**
+     * @return Cache name.
+     */
+    public String getCache() {
+        return cache;
+    }
+
+    /**
+     * @return Query start time.
+     */
+    public long getStartTime() {
+        return startTime;
+    }
+
+    /**
+     * @return Query duration.
+     */
+    public long getDuration() {
+        return duration;
+    }
+
+    /**
+     * @return {@code true} if query can be cancelled.
+     */
+    public boolean isCancelable() {
+        return cancellable;
+    }
+
+    /**
+     * @return {@code true} if query is local.
+     */
+    public boolean isLocal() {
+        return loc;
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index cbf2ebde27056..62b47b85c6507 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -52,6 +52,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import javax.cache.Cache;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
@@ -79,6 +80,7 @@
 import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
 import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
@@ -172,6 +174,9 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_CLEANUP_PERIOD;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_THREAD_USAGE_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.getString;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.TEXT;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.FULLTEXT;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.GEO_SPATIAL;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.SORTED;
@@ -278,9 +283,15 @@ public class IgniteH2Indexing implements GridQueryIndexing {
     /** space name -> schema name */
     private final Map<String, String> space2schema = new ConcurrentHashMap8<>();
 
+    /** */
+    private AtomicLong qryIdGen;
+
     /** */
     private GridSpinBusyLock busyLock;
 
+    /** */
+    private final ConcurrentMap<Long, GridRunningQueryInfo> runs = new ConcurrentHashMap8<>();
+
     /** */
     private final ThreadLocal<ConnectionWrapper> connCache = new ThreadLocal<ConnectionWrapper>() {
         @Nullable @Override public ConnectionWrapper get() {
@@ -751,8 +762,19 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
         IndexingQueryFilter filters) throws IgniteCheckedException {
         TableDescriptor tbl = tableDescriptor(spaceName, type);
 
-        if (tbl != null && tbl.luceneIdx != null)
-            return tbl.luceneIdx.query(qry, filters);
+        if (tbl != null && tbl.luceneIdx != null) {
+            GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, TEXT, spaceName,
+                U.currentTimeMillis(), null, true);
+
+            try {
+                runs.put(run.id(), run);
+
+                return tbl.luceneIdx.query(qry, filters);
+            }
+            finally {
+                runs.remove(run.id());
+            }
+        }
 
         return new GridEmptyCloseableIterator<>();
     }
@@ -796,6 +818,11 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
 
                 GridH2QueryContext.set(ctx);
 
+                GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL_FIELDS,
+                    spaceName, U.currentTimeMillis(), cancel, true);
+
+                runs.putIfAbsent(run.id(), run);
+
                 try {
                     ResultSet rs = executeSqlQueryWithTimer(spaceName, stmt, conn, qry, params, timeout, cancel);
 
@@ -803,6 +830,8 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
                 }
                 finally {
                     GridH2QueryContext.clearThreadLocal();
+
+                    runs.remove(run.id());
                 }
             }
         };
@@ -1061,6 +1090,11 @@ public void setupConnection(Connection conn, boolean distributedJoins, boolean e
 
         GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter).distributedJoins(false));
 
+        GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL, spaceName,
+            U.currentTimeMillis(), null, true);
+
+        runs.put(run.id(), run);
+
         try {
             ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, null);
 
@@ -1068,6 +1102,8 @@ public void setupConnection(Connection conn, boolean distributedJoins, boolean e
         }
         finally {
             GridH2QueryContext.clearThreadLocal();
+
+            runs.remove(run.id());
         }
     }
 
@@ -1692,6 +1728,8 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
 
         this.busyLock = busyLock;
 
+        qryIdGen = new AtomicLong();
+
         if (SysProperties.serializeJavaObject) {
             U.warn(log, "Serialization of Java objects in H2 was enabled.");
 
@@ -1742,7 +1780,7 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
             marshaller = ctx.config().getMarshaller();
 
             mapQryExec = new GridMapQueryExecutor(busyLock);
-            rdcQryExec = new GridReduceQueryExecutor(busyLock);
+            rdcQryExec = new GridReduceQueryExecutor(qryIdGen, busyLock);
 
             mapQryExec.start(ctx, this);
             rdcQryExec.start(ctx, this);
@@ -2196,6 +2234,37 @@ private static List<IndexColumn> treeIndexColumns(List<IndexColumn> cols, IndexC
         return cols;
     }
 
+
+    /** {@inheritDoc} */
+    @Override public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+        Collection<GridRunningQueryInfo> res = new ArrayList<>();
+
+        res.addAll(runs.values());
+        res.addAll(rdcQryExec.longRunningQueries(duration));
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancelQueries(Collection<Long> queries) {
+        if (!F.isEmpty(queries)) {
+            for (Long qryId : queries) {
+                GridRunningQueryInfo run = runs.get(qryId);
+
+                if (run != null)
+                    run.cancel();
+            }
+
+            rdcQryExec.cancelQueries(queries);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancelAllQueries() {
+        for (Connection conn : conns)
+            U.close(conn, log);
+    }
+
     /**
      * Wrapper to store connection and flag is schema set or not.
      */
@@ -3086,10 +3155,4 @@ private void updateLastUsage() {
             lastUsage = U.currentTimeMillis();
         }
     }
-
-    /** {@inheritDoc} */
-    @Override public void cancelAllQueries() {
-        for (Connection conn : conns)
-            U.close(conn, log);
-    }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 7d43bf647ec1a..8284c45e0183d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -170,7 +170,7 @@ public static GridCacheTwoStepQuery split(
         qry = collectAllTables(qry, schemas, tbls);
 
         // Build resulting two step query.
-        GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(schemas, tbls);
+        GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(qry.getSQL(), schemas, tbls);
 
         // Map query will be direct reference to the original query AST.
         // Thus all the modifications will be performed on the original AST, so we should be careful when
@@ -954,4 +954,4 @@ private static GridSqlOperation op(GridSqlOperationType type, GridSqlElement lef
     private static GridSqlFunction function(GridSqlFunctionType type) {
         return new GridSqlFunction(type);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 40b1197317acd..3f886ee686d66 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -62,6 +62,7 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
 import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.h2.GridH2ResultSetIterator;
@@ -99,6 +100,7 @@
 import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
 import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REDUCE;
 
 /**
@@ -121,7 +123,7 @@ public class GridReduceQueryExecutor {
     private IgniteLogger log;
 
     /** */
-    private final AtomicLong reqIdGen = new AtomicLong();
+    private final AtomicLong qryIdGen;
 
     /** */
     private final ConcurrentMap<Long, QueryRun> runs = new ConcurrentHashMap8<>();
@@ -168,9 +170,11 @@ public class GridReduceQueryExecutor {
     };
 
     /**
+     * @param qryIdGen Query ID generator.
      * @param busyLock Busy lock.
      */
-    public GridReduceQueryExecutor(GridSpinBusyLock busyLock) {
+    public GridReduceQueryExecutor(AtomicLong qryIdGen, GridSpinBusyLock busyLock) {
+        this.qryIdGen = qryIdGen;
         this.busyLock = busyLock;
     }
 
@@ -494,11 +498,13 @@ public Iterator<List<?>> query(
                 }
             }
 
-            final long qryReqId = reqIdGen.incrementAndGet();
+            final long qryReqId = qryIdGen.incrementAndGet();
 
             final String space = cctx.name();
 
-            final QueryRun r = new QueryRun(h2.connectionForSpace(space), qry.mapQueries().size(), qry.pageSize());
+            final QueryRun r = new QueryRun(qryReqId, qry.originalSql(), space,
+                h2.connectionForSpace(space), qry.mapQueries().size(), qry.pageSize(),
+                U.currentTimeMillis(), cancel);
 
             AffinityTopologyVersion topVer = h2.readyTopologyVersion();
 
@@ -1303,10 +1309,46 @@ public void onDisconnected(IgniteFuture<?> reconnectFut) {
             e.getValue().disconnected(err);
     }
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of IDs and statements of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> longRunningQueries(long duration) {
+        Collection<GridRunningQueryInfo> res = new ArrayList<>();
+
+        long curTime = U.currentTimeMillis();
+
+        for (QueryRun run : runs.values()) {
+            if (run.qry.longQuery(curTime, duration))
+                res.add(run.qry);
+        }
+
+        return res;
+    }
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries IDs to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries) {
+        for (Long qryId : queries) {
+            QueryRun run = runs.get(qryId);
+
+            if (run != null)
+                run.qry.cancel();
+        }
+    }
+
     /**
      * Query run.
      */
     private static class QueryRun {
+        /** */
+        private final GridRunningQueryInfo qry;
+
         /** */
         private final List<GridMergeIndex> idxs;
 
@@ -1323,11 +1365,17 @@ private static class QueryRun {
         private final AtomicReference<Object> state = new AtomicReference<>();
 
         /**
+         * @param id Query ID.
+         * @param qry Query text.
+         * @param cache Cache where query was executed.
          * @param conn Connection.
          * @param idxsCnt Number of indexes.
          * @param pageSize Page size.
+         * @param startTime Start time.
+         * @param cancel Query cancel handler.
          */
-        private QueryRun(Connection conn, int idxsCnt, int pageSize) {
+        private QueryRun(Long id, String qry, String cache, Connection conn, int idxsCnt, int pageSize, long startTime, GridQueryCancel cancel) {
+            this.qry = new GridRunningQueryInfo(id, qry, SQL_FIELDS, cache, startTime, cancel, false);
             this.conn = (JdbcConnection)conn;
             this.idxs = new ArrayList<>(idxsCnt);
             this.pageSize = pageSize > 0 ? pageSize : GridCacheTwoStepQuery.DFLT_PAGE_SIZE;
@@ -1410,4 +1458,4 @@ private ExplicitPartitionsSpecializer(Map<ClusterNode,IntArray> partsMap) {
             return copy(msg, n, partsMap);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
index e47e893530384..66e7e4abf9cdb 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
@@ -17,15 +17,23 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.Query;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cache.query.annotations.QuerySqlFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -54,6 +62,7 @@ public class CacheSqlQueryValueCopySelfTest extends GridCommonAbstractTest {
 
         cc.setCopyOnRead(true);
         cc.setIndexedTypes(Integer.class, Value.class);
+        cc.setSqlFunctionClasses(TestSQLFunctions.class);
 
         cfg.setCacheConfiguration(cc);
 
@@ -72,7 +81,7 @@ public class CacheSqlQueryValueCopySelfTest extends GridCommonAbstractTest {
         IgniteCache<Integer, Value> cache = grid(0).cache(null);
 
         for (int i = 0; i < KEYS; i++)
-            cache.put(i, new Value("before"));
+            cache.put(i, new Value(i, "before-" + i));
     }
 
     /** {@inheritDoc} */
@@ -195,17 +204,148 @@ public void testLocalSqlFieldsQuery() {
         check(cache);
     }
 
-    /** */
-    private static class Value {
-        /** */
-        private String str;
+    /**
+     * Run specified query in separate thread.
+     *
+     * @param qry Query to execute.
+     */
+    private IgniteInternalFuture<?> runQueryAsync(final Query<?> qry) throws Exception {
+        return multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    log.info(">>> Query started");
+
+                    grid(0).cache(null).query(qry).getAll();
+
+                    log.info(">>> Query finished");
+                }
+                catch (Throwable e) {
+                    e.printStackTrace();
+                }
+            }
+        }, 1, "run-query");
+    }
 
-        /**
-         * @param str String.
-         */
-        public Value(String str) {
-            this.str = str;
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRunningSqlFieldsQuery() throws Exception {
+        IgniteInternalFuture<?> fut = runQueryAsync(new SqlFieldsQuery("select _val, sleep(1000) from Value limit 3"));
+
+        Thread.sleep(500);
+
+        GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+
+        SqlFieldsQuery qry = new SqlFieldsQuery("select _val, sleep(1000) from Value limit 3");
+        qry.setLocal(true);
+
+        fut = runQueryAsync(qry);
+
+        Thread.sleep(500);
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+    }
+
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRunningSqlQuery() throws Exception {
+        IgniteInternalFuture<?> fut = runQueryAsync(new SqlQuery<Integer, Value>(Value.class, "id > sleep(100)"));
+
+        Thread.sleep(500);
+
+        GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+
+        SqlQuery<Integer, Value> qry = new SqlQuery<>(Value.class, "id > sleep(100)");
+        qry.setLocal(true);
+
+        fut = runQueryAsync(qry);
+
+        Thread.sleep(500);
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+    }
+
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCancelingSqlFieldsQuery() throws Exception {
+        runQueryAsync(new SqlFieldsQuery("select * from (select _val, sleep(100) from Value limit 50)"));
+
+        Thread.sleep(500);
+
+        final GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        final Collection<GridRunningQueryInfo> finalQueries = queries;
+
+        for (GridRunningQueryInfo query : finalQueries)
+            qryProc.cancelQueries(Collections.singleton(query.id()));
+
+        int n = 100;
+
+        // Give cluster some time to cancel query and cleanup resources.
+        while (n > 0) {
+            Thread.sleep(100);
+
+            queries = qryProc.runningQueries(0);
+
+            if (queries.isEmpty())
+                break;
+
+            log.info(">>>> Wait for cancel: " + n);
+
+            n--;
         }
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
     }
 
     /**
@@ -218,9 +358,53 @@ private void check(IgniteCache<Integer, Value> cache) {
         for (Cache.Entry<Integer, Value> entry : cache) {
             cnt++;
 
-            assertEquals("before", entry.getValue().str);
+            assertEquals("before-" + entry.getKey(), entry.getValue().str);
         }
 
         assertEquals(KEYS, cnt);
     }
-}
\ No newline at end of file
+
+    /** */
+    private static class Value {
+        /** */
+        @QuerySqlField
+        private int id;
+
+        /** */
+        @QuerySqlField
+        private String str;
+
+        /**
+         * @param id ID.
+         * @param str String.
+         */
+        public Value(int id, String str) {
+            this.id = id;
+            this.str = str;
+        }
+    }
+
+    /**
+     * Utility class with custom SQL functions.
+     */
+    public static class TestSQLFunctions {
+        /**
+         * Sleep function to simulate long running queries.
+         *
+         * @param x Time to sleep.
+         * @return Return specified argument.
+         */
+        @QuerySqlFunction
+        public static long sleep(long x) {
+            if (x >= 0)
+                try {
+                    Thread.sleep(x);
+                }
+                catch (InterruptedException ignored) {
+                    // No-op.
+                }
+
+            return x;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
index 1f10593e61e0c..01fefa3ac95e3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
@@ -477,4 +477,4 @@ public int getStoreId() {
             return storeId;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
index ad8a7e34e82a7..814d0e0eef516 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
@@ -106,6 +106,9 @@ protected void startIndexing(IgniteH2Indexing spi) throws Exception {
         spi.registerCache(null, cacheCfg("B"));
     }
 
+    /**
+     * @param name Name.
+     */
     private CacheConfiguration cacheCfg(String name) {
         CacheConfiguration<?,?> cfg = new CacheConfiguration<>();
 
@@ -114,6 +117,7 @@ private CacheConfiguration cacheCfg(String name) {
         return cfg;
     }
 
+    /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         idx.stop();
 
@@ -182,6 +186,9 @@ private IgniteH2Indexing getIndexing() {
         return idx;
     }
 
+    /**
+     * @return {@code true} if OFF-HEAP mode should be tested.
+     */
     protected boolean offheap() {
         return false;
     }

From 2f57760dbb4fba948cd035498d2c7f71869c0665 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 17 Feb 2017 16:15:31 +0300
Subject: [PATCH 163/446] IGNITE-4624: Scan query optimization. This closes
 #1509.

---
 .../distributed/dht/GridDhtCacheAdapter.java  | 19 +++-
 .../cache/query/GridCacheQueryManager.java    | 97 ++++++++++---------
 2 files changed, 64 insertions(+), 52 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index dcd379a040bd0..be7fa5525deba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -1247,14 +1247,27 @@ public Iterator<Cache.Entry<K, V>> localEntriesIterator(final boolean primary,
         final boolean backup,
         final boolean keepBinary,
         final AffinityTopologyVersion topVer) {
+
+        return iterator(localEntriesIteratorEx(primary, backup, topVer), !keepBinary);
+    }
+
+    /**
+     * @param primary If {@code true} includes primary entries.
+     * @param backup If {@code true} includes backup entries.
+     * @param topVer Specified affinity topology version.
+     * @return Local entries iterator.
+     */
+    public Iterator<? extends GridCacheEntryEx> localEntriesIteratorEx(final boolean primary,
+        final boolean backup,
+        final AffinityTopologyVersion topVer) {
         assert primary || backup;
 
         if (primary && backup)
-            return iterator(entries().iterator(), !keepBinary);
+            return entries().iterator();
         else {
             final Iterator<GridDhtLocalPartition> partIt = topology().currentLocalPartitions().iterator();
 
-            Iterator<GridCacheMapEntry> it = new Iterator<GridCacheMapEntry>() {
+            return new Iterator<GridCacheMapEntry>() {
                 private GridCacheMapEntry next;
 
                 private Iterator<GridCacheMapEntry> curIt;
@@ -1311,8 +1324,6 @@ private void advance() {
                     while (partIt.hasNext());
                 }
             };
-
-            return iterator(it, !keepBinary);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index d64dff4b1a02e..14b1106eb5e1c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -1033,23 +1033,25 @@ private GridIterator<IgniteBiTuple<K, V>> swapIterator(
      * @throws GridDhtUnreservedPartitionException If failed to reserve partition.
      */
     private GridIterator<IgniteBiTuple<K, V>> onheapIterator(
-        GridCacheQueryAdapter<?> qry,
+        final GridCacheQueryAdapter<?> qry,
         AffinityTopologyVersion topVer,
         final IgniteBiPredicate<K, V> keyValFilter,
-        boolean backups,
+        final boolean backups,
         final ExpiryPolicy plc,
         final boolean locNode) throws GridDhtUnreservedPartitionException {
-        Iterator<K> keyIter;
+        Iterator<? extends GridCacheEntryEx> entryIter;
 
         GridDhtLocalPartition locPart = null;
 
         Integer part = qry.partition();
 
-        if (part == null || cctx.isLocal()) {
-            // Performance optimization.
-            if (locNode && plc == null && !cctx.isLocal()) {
-                GridDhtCacheAdapter<K, V> cache = cctx.isNear() ? cctx.near().dht() : cctx.dht();
+        if (cctx.isLocal())
+            entryIter = cctx.local().allEntries().iterator();
+        else if (part == null) {
+            GridDhtCacheAdapter<K, V> cache = cctx.isNear() ? cctx.near().dht() : cctx.dht();
 
+            // Performance optimization.
+            if (locNode && plc == null) {
                 final Iterator<Cache.Entry<K, V>> iter = cache.localEntriesIterator(true,
                     backups, cache.context().keepBinary(), topVer);
 
@@ -1099,12 +1101,10 @@ private void advance() {
                 };
             }
 
-            IgniteInternalCache<K, V> keepBinaryCache = cctx.cache().keepBinary();
-
-            keyIter = backups ? keepBinaryCache.keySetx().iterator() : keepBinaryCache.primaryKeySet().iterator();
+            entryIter = cache.localEntriesIteratorEx(true, backups, topVer);
         }
         else if (part < 0 || part >= cctx.affinity().partitions())
-            keyIter = new GridEmptyIterator<>();
+            return new GridEmptyIterator<>();
         else {
             final GridDhtCacheAdapter dht = cctx.isNear() ? cctx.near().dht() : cctx.dht();
 
@@ -1115,28 +1115,12 @@ else if (part < 0 || part >= cctx.affinity().partitions())
                 throw new GridDhtUnreservedPartitionException(part, cctx.affinity().affinityTopologyVersion(),
                     "Partition can not be reserved.");
 
-            final GridDhtLocalPartition locPart0 = locPart;
-
-            keyIter = new Iterator<K>() {
-                private Iterator<KeyCacheObject> iter0 = locPart0.keySet().iterator();
-
-                @Override public boolean hasNext() {
-                    return iter0.hasNext();
-                }
-
-                @Override public K next() {
-                    return (K)iter0.next();
-                }
-
-                @Override public void remove() {
-                    iter0.remove();
-                }
-            };
+            entryIter = locPart.allEntries().iterator();
         }
 
         final GridDhtLocalPartition locPart0 = locPart;
 
-        return new PeekValueExpiryAwareIterator(keyIter, plc, topVer, keyValFilter, qry.keepBinary(), locNode, true) {
+        return new PeekValueExpiryAwareIterator(entryIter, plc, topVer, keyValFilter, qry.keepBinary(), locNode, true) {
             @Override protected void onClose() {
                 super.onClose();
 
@@ -1263,18 +1247,20 @@ private GridIterator<IgniteBiTuple<K, V>> scanExpiryIterator(
         ExpiryPolicy expPlc,
         final boolean keepBinary,
         boolean locNode) {
-        Iterator<K> keyIter = new Iterator<K>() {
+        Iterator<? extends GridCacheEntryEx> keyIter = new Iterator<GridCacheEntryEx>() {
             /** {@inheritDoc} */
             @Override public boolean hasNext() {
                 return it.hasNext();
             }
 
             /** {@inheritDoc} */
-            @Override public K next() {
+            @Override public GridCacheEntryEx next() {
                 try {
                     KeyCacheObject key = cctx.toCacheKeyObject(it.next().getKey());
 
-                    return (K)cctx.unwrapBinaryIfNeeded(key, keepBinary);
+                    final GridCacheEntryEx entryEx = cctx.cache().entryEx(key);
+
+                    return entryEx;
                 }
                 catch (IgniteCheckedException e) {
                     throw new IgniteException(e);
@@ -2189,8 +2175,8 @@ public QueryMetrics metrics() {
     }
 
     /**
-     * Gets cache queries detailed metrics.
-     * Detail metrics could be enabled by setting non-zero value via {@link CacheConfiguration#setQueryDetailMetricsSize(int)}
+     * Gets cache queries detailed metrics. Detail metrics could be enabled by setting non-zero value via {@link
+     * CacheConfiguration#setQueryDetailMetricsSize(int)}
      *
      * @return Cache queries metrics aggregated by query type and query text.
      */
@@ -3091,8 +3077,10 @@ private CompoundIterator(List<GridIterator<T>> iters) {
     private abstract static class CachedResult<R> extends GridFutureAdapter<IgniteSpiCloseableIterator<R>> {
         /** Absolute position of each recipient. */
         private final Map<Object, QueueIterator> recipients = new GridLeanMap<>(1);
+
         /** */
         private CircularQueue<R> queue;
+
         /** */
         private int pruned;
 
@@ -3529,10 +3517,10 @@ private class PeekValueExpiryAwareIterator extends GridCloseableIteratorAdapter<
         private IgniteCacheExpiryPolicy expiryPlc;
 
         /** */
-        private Iterator<K> keyIt;
+        private Iterator<? extends GridCacheEntryEx> entryIt;
 
         /**
-         * @param keyIt Key iterator.
+         * @param entryIter Key iterator.
          * @param plc Expiry policy.
          * @param topVer Topology version.
          * @param keyValFilter Key-value filter.
@@ -3540,8 +3528,8 @@ private class PeekValueExpiryAwareIterator extends GridCloseableIteratorAdapter<
          * @param locNode Local node.
          * @param heapOnly Heap only.
          */
-        private PeekValueExpiryAwareIterator(
-            Iterator<K> keyIt,
+        PeekValueExpiryAwareIterator(
+            Iterator<? extends GridCacheEntryEx> entryIter,
             ExpiryPolicy plc,
             AffinityTopologyVersion topVer,
             IgniteBiPredicate<K, V> keyValFilter,
@@ -3549,7 +3537,7 @@ private PeekValueExpiryAwareIterator(
             boolean locNode,
             boolean heapOnly
         ) {
-            this.keyIt = keyIt;
+            this.entryIt = entryIter;
             this.plc = plc;
             this.topVer = topVer;
             this.keyValFilter = keyValFilter;
@@ -3593,15 +3581,27 @@ private PeekValueExpiryAwareIterator(
         private void advance() {
             IgniteBiTuple<K, V> next0 = null;
 
-            while (keyIt.hasNext()) {
+            while (entryIt.hasNext()) {
                 next0 = null;
 
-                K key = keyIt.next();
+                GridCacheEntryEx entry = entryIt.next();
+
+                if (entry.deleted())
+                    continue;
 
+                KeyCacheObject key = entry.key();
                 CacheObject val;
 
                 try {
-                    val = value(key);
+                    if (heapOnly)
+                        val = entry.peek(true, false, false, expiryPlc);
+                    else
+                        val = value(entry, entry.key());
+                }
+                catch (GridCacheEntryRemovedException ignore) {
+                    assert heapOnly;
+
+                    continue;
                 }
                 catch (IgniteCheckedException e) {
                     if (log.isDebugEnabled())
@@ -3664,23 +3664,24 @@ private void sendTtlUpdate() {
         }
 
         /**
+         * @param entry Entry.
          * @param key Key.
          * @return Value.
          * @throws IgniteCheckedException If failed to peek value.
          */
-        private CacheObject value(K key) throws IgniteCheckedException {
+        private CacheObject value(GridCacheEntryEx entry, KeyCacheObject key) throws IgniteCheckedException {
             while (true) {
                 try {
-                    GridCacheEntryEx entry = heapOnly ? cache.peekEx(key) : cache.entryEx(key);
+                    if (entry == null)
+                        entry = cache.entryEx(key);
 
-                    if (expiryPlc != null && !heapOnly)
+                    if (expiryPlc != null)
                         entry.unswap();
 
-                    return entry != null ? entry.peek(true, !heapOnly, !heapOnly, topVer, expiryPlc) : null;
+                    return entry.peek(true, true, true, topVer, expiryPlc);
                 }
                 catch (GridCacheEntryRemovedException ignore) {
-                    if (heapOnly)
-                        return null;
+                    entry = null;
                 }
             }
         }

From c0e2df26f056cd11690d821146f05e3fd938906e Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 20 Feb 2017 11:17:35 +0300
Subject: [PATCH 164/446] IGNITE-3429 - Rollback due to broken compilation

---
 .../Hibernate5CacheKeyTypeConfiguration.java  | 52 -------------------
 .../HibernateCacheKeyTypeConfiguration.java   | 51 ------------------
 2 files changed, 103 deletions(-)
 delete mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
 delete mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java

diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
deleted file mode 100644
index 886f69b2500ba..0000000000000
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.hibernate.config;
-
-import java.util.Objects;
-import org.apache.ignite.binary.BinaryAbstractIdentityResolver;
-import org.apache.ignite.binary.BinaryIdentityResolver;
-import org.apache.ignite.binary.BinaryObject;
-import org.apache.ignite.binary.BinaryTypeConfiguration;
-
-/**
- * This configuration provides correct {@link BinaryIdentityResolver} implementation
- * for Hibernate CacheKey class can be used as a key object.
- *
- * Note: for Hibernate version < 5.0 {@link HibernateCacheKeyTypeConfiguration} should be used.
-
- */
-public class Hibernate5CacheKeyTypeConfiguration extends BinaryTypeConfiguration {
-
-    /** {@inheritDoc} */
-    public Hibernate5CacheKeyTypeConfiguration() {
-        super("org.hibernate.cache.internal.CacheKeyImplementation");
-
-        setIdentityResolver(new BinaryAbstractIdentityResolver() {
-            @Override protected int hashCode0(BinaryObject obj) {
-                return obj.field("id").hashCode();
-            }
-
-            @Override protected boolean equals0(BinaryObject o1, BinaryObject o2) {
-                Object obj0 = o1.field("id");
-                Object obj1 = o2.field("id");
-
-                return Objects.equals(obj0, obj1);
-            }
-        });
-    }
-}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java
deleted file mode 100644
index c54292e6f40e6..0000000000000
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.hibernate.config;
-
-import java.util.Objects;
-import org.apache.ignite.binary.BinaryAbstractIdentityResolver;
-import org.apache.ignite.binary.BinaryIdentityResolver;
-import org.apache.ignite.binary.BinaryObject;
-import org.apache.ignite.binary.BinaryTypeConfiguration;
-
-/**
- * This configuration provides correct {@link BinaryIdentityResolver} implementation
- * for Hibernate CacheKey class can be used as a key object.
- *
- * Note: for Hibernate version >= 5.0 {@link Hibernate5CacheKeyTypeConfiguration} should be used.
- */
-public class HibernateCacheKeyTypeConfiguration extends BinaryTypeConfiguration {
-
-    /** {@inheritDoc} */
-    public HibernateCacheKeyTypeConfiguration() {
-        super("org.hibernate.cache.spi.CacheKey");
-
-        setIdentityResolver(new BinaryAbstractIdentityResolver() {
-            @Override protected int hashCode0(BinaryObject obj) {
-                return obj.field("key").hashCode();
-            }
-
-            @Override protected boolean equals0(BinaryObject o1, BinaryObject o2) {
-                Object obj0 = o1.field("key");
-                Object obj1 = o2.field("key");
-
-                return Objects.equals(obj0, obj1);
-            }
-        });
-    }
-}

From c849534df6583043d6a1d4f454cb981a20896d1a Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Tue, 21 Feb 2017 10:08:55 +0700
Subject: [PATCH 165/446] IGNITE-4472 Added user activities in Web Console.

(cherry picked from commit 26ee9c2865648118da97ee8ef84df990359edb96)
---
 .../frontend/app/modules/demo/Demo.module.js       | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js
index a3700ca3ed141..bd759df552c23 100644
--- a/modules/web-console/frontend/app/modules/demo/Demo.module.js
+++ b/modules/web-console/frontend/app/modules/demo/Demo.module.js
@@ -23,22 +23,26 @@ angular
 .module('ignite-console.demo', [
     'ignite-console.socket'
 ])
-.config(['$stateProvider', ($stateProvider) => {
+.config(['$stateProvider', 'AclRouteProvider', ($stateProvider, AclRoute) => {
     $stateProvider
         .state('demo', {
             abstract: true,
+            url: '/demo',
             template: '<ui-view></ui-view>'
         })
         .state('demo.resume', {
-            url: '/demo',
+            url: '/resume',
+            onEnter: AclRoute.checkAccess('demo'),
             controller: ['$state', ($state) => {
                 $state.go('base.configuration.clusters');
             }],
             metaTags: {
+                title: 'Demo resume'
             }
         })
         .state('demo.reset', {
-            url: '/demo/reset',
+            url: '/reset',
+            onEnter: AclRoute.checkAccess('demo'),
             controller: ['$state', '$http', 'IgniteMessages', ($state, $http, Messages) => {
                 $http.post('/api/v1/demo/reset')
                     .then(() => $state.go('base.configuration.clusters'))
@@ -48,7 +52,9 @@ angular
                         Messages.showError(res);
                     });
             }],
-            metaTags: {}
+            metaTags: {
+                title: 'Demo reset'
+            }
         });
 }])
 .provider('Demo', ['$stateProvider', '$httpProvider', 'igniteSocketFactoryProvider', function($state, $http, socketFactory) {

From 9df5e94d5cf14ddd55e29b81989177a7798f7e1a Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 21 Feb 2017 15:34:59 +0300
Subject: [PATCH 166/446] IGNITE-4671 - FairAffinityFunction fails on node
 restart with backupFilter set and no backups

---
 .../affinity/fair/FairAffinityFunction.java   |  33 ++---
 .../fair/FairAffinityNodesRestart.java        | 130 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite.java      |   2 +
 3 files changed, 149 insertions(+), 16 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityNodesRestart.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
index cf1cb02117c75..7acb5b429e693 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
@@ -911,37 +911,37 @@ private boolean isAssignable(int part, int tier, final ClusterNode node, boolean
             if (exclNeighbors)
                 return allowNeighbors || !neighborsContainPartition(node, part);
             else if (affinityBackupFilter != null) {
-                List<ClusterNode> assigment = assignments.get(part);
+                List<ClusterNode> assignment = assignments.get(part);
 
-                assert assigment.size() > 0;
+                if (assignment.isEmpty())
+                    return true;
 
                 List<ClusterNode> newAssignment;
 
                 if (tier == 0) {
-                    for (int t = 1; t < assigment.size(); t++) {
-                        newAssignment = new ArrayList<>(assigment.size() - 1);
+                    for (int t = 1; t < assignment.size(); t++) {
+                        newAssignment = new ArrayList<>(assignment.size() - 1);
 
                         newAssignment.add(node);
 
                         if (t != 1)
-                            newAssignment.addAll(assigment.subList(1, t));
+                            newAssignment.addAll(assignment.subList(1, t));
 
-                        if (t + 1 < assigment.size())
-                            newAssignment.addAll(assigment.subList(t + 1, assigment.size()));
+                        if (t + 1 < assignment.size())
+                            newAssignment.addAll(assignment.subList(t + 1, assignment.size()));
 
-                        if (!affinityBackupFilter.apply(assigment.get(t), newAssignment))
+                        if (!affinityBackupFilter.apply(assignment.get(t), newAssignment))
                             return false;
-
                     }
 
                     return true;
                 }
-                else if (tier < assigment.size()) {
-                    newAssignment = new ArrayList<>(assigment.size() - 1);
+                else if (tier < assignment.size()) {
+                    newAssignment = new ArrayList<>(assignment.size() - 1);
 
                     int i = 0;
 
-                    for (ClusterNode assignmentNode: assigment) {
+                    for (ClusterNode assignmentNode: assignment) {
                         if (i != tier)
                             newAssignment.add(assignmentNode);
 
@@ -949,17 +949,18 @@ else if (tier < assigment.size()) {
                     }
                 }
                 else
-                    newAssignment = assigment;
+                    newAssignment = assignment;
 
                 return affinityBackupFilter.apply(node, newAssignment);
             }
             else if (backupFilter != null) {
                 if (tier == 0) {
-                    List<ClusterNode> assigment = assignments.get(part);
+                    List<ClusterNode> assignment = assignments.get(part);
 
-                    assert assigment.size() > 0;
+                    if (assignment.isEmpty())
+                        return true;
 
-                    List<ClusterNode> backups = assigment.subList(1, assigment.size());
+                    List<ClusterNode> backups = assignment.subList(1, assignment.size());
 
                     return !F.exist(backups, new IgnitePredicate<ClusterNode>() {
                         @Override public boolean apply(ClusterNode n) {
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityNodesRestart.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityNodesRestart.java
new file mode 100644
index 0000000000000..37f1bfb211297
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityNodesRestart.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.affinity.fair;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.P2;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests that FairAffinityFunction doesn't throw exception on nodes restart,
+ * with backup filter set and 0 cache backups.
+ */
+public class FairAffinityNodesRestart extends GridCommonAbstractTest {
+    /** */
+    private final static P2<ClusterNode, ClusterNode> BACKUP_FILTER = new P2<ClusterNode, ClusterNode>() {
+        @Override public boolean apply(ClusterNode node, ClusterNode node2) {
+            return true;
+        }
+    };
+
+    /** */
+    private final static P2<ClusterNode, List<ClusterNode>> AFF_BACKUP_FILTER = new P2<ClusterNode, List<ClusterNode>>() {
+        @Override public boolean apply(ClusterNode node, List<ClusterNode> nodes) {
+            return true;
+        }
+    };
+
+    /** */
+    private boolean affBackup;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = new CacheConfiguration("fair-cache");
+
+        FairAffinityFunction aff = new FairAffinityFunction(32);
+
+        if (!affBackup)
+            aff.setBackupFilter(BACKUP_FILTER);
+        else
+            aff.setAffinityBackupFilter(AFF_BACKUP_FILTER);
+
+        ccfg.setAffinity(aff);
+        ccfg.setBackups(0);
+        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /**
+     * @param idx Node index.
+     * @return Future.
+     */
+    private IgniteInternalFuture<IgniteEx> startAsyncGrid(final int idx) {
+        return GridTestUtils.runAsync(new Callable<IgniteEx>() {
+            @Override public IgniteEx call() throws Exception {
+                return startGrid(idx);
+            }
+        });
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBackupFilter() throws Exception {
+        affBackup = false;
+
+        check();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAffinityBackupFilter() throws Exception {
+        affBackup = true;
+
+        check();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void check() throws Exception {
+        for (int i = 0; i < 2; i++) {
+            IgniteInternalFuture<IgniteEx> fut0 = startAsyncGrid(0);
+            IgniteInternalFuture<IgniteEx> fut1 = startAsyncGrid(1);
+            IgniteInternalFuture<IgniteEx> fut2 = startAsyncGrid(2);
+
+            IgniteEx ignite = fut0.get();
+            fut1.get();
+            fut2.get();
+
+            IgniteCache<Integer, String> cache = ignite.cache("fair-cache");
+
+            for (int j = 0; j < 100; j++)
+                cache.put(i, String.valueOf(i));
+
+            stopGrid(0);
+            stopGrid(1);
+            stopGrid(2);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 092d95eff12bd..a24f020103e54 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.cache.affinity.fair.FairAffinityDynamicCacheSelfTest;
 import org.apache.ignite.cache.affinity.fair.FairAffinityFunctionNodesSelfTest;
 import org.apache.ignite.cache.affinity.fair.FairAffinityFunctionSelfTest;
+import org.apache.ignite.cache.affinity.fair.FairAffinityNodesRestart;
 import org.apache.ignite.cache.affinity.local.LocalAffinityFunctionTest;
 import org.apache.ignite.cache.store.GridCacheBalancingStoreSelfTest;
 import org.apache.ignite.cache.store.GridCacheLoadOnlyStoreAdapterSelfTest;
@@ -231,6 +232,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(FairAffinityFunctionNodesSelfTest.class);
         suite.addTestSuite(FairAffinityFunctionSelfTest.class);
         suite.addTestSuite(FairAffinityDynamicCacheSelfTest.class);
+        suite.addTestSuite(FairAffinityNodesRestart.class);
         suite.addTestSuite(GridCacheAffinityBackupsSelfTest.class);
         suite.addTestSuite(IgniteCacheAffinitySelfTest.class);
         suite.addTestSuite(AffinityClientNodeSelfTest.class);

From 9fcb3e74f91c8497b7b1358cdff40950cdf5c568 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 28 Feb 2017 16:05:06 +0300
Subject: [PATCH 167/446] IGNITE-4740 - Fix. Service could be
 deployed/undeployed twice on concurrent cancel and discovery event.

---
 .../cache/DynamicCacheChangeBatch.java        |  14 ++
 .../service/GridServiceProcessor.java         |  49 ++---
 .../GridServiceContinuousQueryRedeploy.java   | 167 ++++++++++++++++++
 .../testsuites/IgniteKernalSelfTestSuite.java |   2 +
 4 files changed, 208 insertions(+), 24 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceContinuousQueryRedeploy.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
index 4dcff9b8bc2f2..a2500633a1ccf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
@@ -113,6 +113,20 @@ public boolean clientReconnect() {
         return clientReconnect;
     }
 
+    /**
+     * @return {@code True} if request should trigger partition exchange.
+     */
+    public boolean exchangeNeeded() {
+        if (reqs != null) {
+            for (DynamicCacheChangeRequest req : reqs) {
+                if (req.exchangeNeeded())
+                    return true;
+            }
+        }
+
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(DynamicCacheChangeBatch.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 3690f357f1e5e..4eeafed421214 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -65,10 +65,12 @@
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.internal.processors.cache.CacheIteratorConverter;
+import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.query.CacheQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.internal.processors.continuous.AbstractContinuousMessage;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridEmptyIterator;
@@ -1468,19 +1470,7 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
         else {
             String name = e.getKey().name();
 
-            svcName.set(name);
-
-            Collection<ServiceContextImpl> ctxs;
-
-            synchronized (locSvcs) {
-                ctxs = locSvcs.remove(name);
-            }
-
-            if (ctxs != null) {
-                synchronized (ctxs) {
-                    cancel(ctxs, ctxs.size());
-                }
-            }
+            undeploy(name);
 
             // Finish deployment futures if undeployment happened.
             GridFutureAdapter<?> fut = depFuts.remove(name);
@@ -1586,6 +1576,12 @@ private class TopologyListener implements GridLocalEventListener {
                         if (!((CacheAffinityChangeMessage)msg).exchangeNeeded())
                             return;
                     }
+                    else if (msg instanceof DynamicCacheChangeBatch) {
+                        if (!((DynamicCacheChangeBatch)msg).exchangeNeeded())
+                            return;
+                    }
+                    else
+                        return;
                 }
                 else
                     topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
@@ -1771,21 +1767,26 @@ private void processAssignment(CacheEntryEvent<GridServiceAssignmentsKey, GridSe
             }
         }
         // Handle undeployment.
-        else {
-            String name = e.getKey().name();
+        else
+            undeploy(e.getKey().name());
+    }
 
-            svcName.set(name);
 
-            Collection<ServiceContextImpl> ctxs;
+    /**
+     * @param name Name.
+     */
+    private void undeploy(String name) {
+        svcName.set(name);
 
-            synchronized (locSvcs) {
-                ctxs = locSvcs.remove(name);
-            }
+        Collection<ServiceContextImpl> ctxs;
 
-            if (ctxs != null) {
-                synchronized (ctxs) {
-                    cancel(ctxs, ctxs.size());
-                }
+        synchronized (locSvcs) {
+            ctxs = locSvcs.remove(name);
+        }
+
+        if (ctxs != null) {
+            synchronized (ctxs) {
+                cancel(ctxs, ctxs.size());
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceContinuousQueryRedeploy.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceContinuousQueryRedeploy.java
new file mode 100644
index 0000000000000..1a9ef3a084e37
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceContinuousQueryRedeploy.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryListenerException;
+import javax.cache.event.CacheEntryUpdatedListener;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests whether concurrent service cancel and registering ContinuousQuery doesn't causes
+ * service redeployment.
+ */
+public class GridServiceContinuousQueryRedeploy extends GridCommonAbstractTest {
+    /** */
+    private static final String CACHE_NAME = "TEST_CACHE";
+
+    /** */
+    private static final String TEST_KEY = "TEST_KEY";
+
+    /** */
+    private static final String SERVICE_NAME = "service1";
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testServiceRedeploymentAfterCancel() throws Exception {
+        final Ignite ignite = startGrid(0);
+
+        final IgniteCache<Object, Object> managementCache = ignite.getOrCreateCache(CACHE_NAME);
+
+        final ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+        final List<Object> evts = Collections.synchronizedList(new ArrayList<>());
+
+        qry.setLocalListener(new CacheEntryUpdatedListener<Object, Object>() {
+            @Override public void onUpdated(
+                Iterable<CacheEntryEvent<?, ?>> iterable) throws CacheEntryListenerException {
+                for (CacheEntryEvent<?, ?> event : iterable)
+                    evts.add(event);
+            }
+        });
+
+        int iterations = 100;
+
+        while (iterations-- > 0) {
+            QueryCursor quorumCursor = managementCache.query(qry);
+
+            IgniteInternalFuture<?> fut1 = GridTestUtils.runAsync(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    System.out.println("Deploy " + SERVICE_NAME);
+                    deployService(ignite);
+
+                    return null;
+                }
+            });
+
+            IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    System.out.println("Undeploy " + SERVICE_NAME);
+                    ignite.services().cancel(SERVICE_NAME);
+
+                    return null;
+                }
+            });
+
+            fut1.get();
+            fut2.get();
+
+            U.sleep(100);
+
+            assert evts.size() <= 1 : evts.size();
+
+            ignite.services().cancel("service1");
+
+            evts.clear();
+
+            quorumCursor.close();
+        }
+
+    }
+
+    /**
+     * @param ignite Ignite.
+     */
+    private void deployService(final Ignite ignite) {
+        ServiceConfiguration svcCfg = new ServiceConfiguration();
+
+        svcCfg.setService(new ManagementService());
+        svcCfg.setName(SERVICE_NAME);
+        svcCfg.setTotalCount(1);
+        svcCfg.setMaxPerNodeCount(1);
+        svcCfg.setNodeFilter(new IgnitePredicate<ClusterNode>() {
+            @Override public boolean apply(ClusterNode node) {
+                return !node.isClient();
+            }
+        });
+
+        ignite.services().deploy(svcCfg);
+    }
+
+    /**
+     *
+     */
+    public static class ManagementService implements Service {
+        /** */
+        private final String name = UUID.randomUUID().toString();
+
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+            System.out.println(name + " shutdown.");
+        }
+
+        /** {@inheritDoc} */
+        @Override public synchronized void init(ServiceContext ctx) throws Exception {
+            System.out.println(name + " initializing.");
+
+            ignite.cache(CACHE_NAME).put(TEST_KEY, name + " init");
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+            // No-op
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 350b7152c36d9..59777023e8682 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -50,6 +50,7 @@
 import org.apache.ignite.internal.processors.cluster.GridUpdateNotifierSelfTest;
 import org.apache.ignite.internal.processors.port.GridPortProcessorSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceClientNodeTest;
+import org.apache.ignite.internal.processors.service.GridServiceContinuousQueryRedeploy;
 import org.apache.ignite.internal.processors.service.GridServicePackagePrivateSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceProcessorMultiNodeConfigSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceProcessorMultiNodeSelfTest;
@@ -143,6 +144,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridServiceProxyClientReconnectSelfTest.class);
         suite.addTestSuite(IgniteServiceReassignmentTest.class);
         suite.addTestSuite(IgniteServiceProxyTimeoutInitializedTest.class);
+        suite.addTestSuite(GridServiceContinuousQueryRedeploy.class);
 
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class);

From 231984d04d8391ccafb33ce995d942a9033cae44 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 1 Mar 2017 09:49:08 +0700
Subject: [PATCH 168/446] Web console: removed legacy backend port.

(cherry picked from commit 5456caf)
---
 modules/web-console/backend/app/settings.js |  8 +-------
 modules/web-console/backend/index.js        | 10 ----------
 2 files changed, 1 insertion(+), 17 deletions(-)

diff --git a/modules/web-console/backend/app/settings.js b/modules/web-console/backend/app/settings.js
index a79572e6989f0..94f3d90f58664 100644
--- a/modules/web-console/backend/app/settings.js
+++ b/modules/web-console/backend/app/settings.js
@@ -51,13 +51,7 @@ module.exports.factory = function(nconf, fs) {
 
     return {
         agent: {
-            dists: 'agent_dists',
-            port: _normalizePort(nconf.get('agentServer:port') || 3002),
-            SSLOptions: nconf.get('agentServer:ssl') && {
-                key: fs.readFileSync(nconf.get('agentServer:key')),
-                cert: fs.readFileSync(nconf.get('agentServer:cert')),
-                passphrase: nconf.get('agentServer:keyPassphrase')
-            }
+            dists: 'agent_dists'
         },
         server: {
             port: _normalizePort(nconf.get('server:port') || 3000),
diff --git a/modules/web-console/backend/index.js b/modules/web-console/backend/index.js
index 9eeff3516c2ee..27d7298dc5881 100644
--- a/modules/web-console/backend/index.js
+++ b/modules/web-console/backend/index.js
@@ -86,16 +86,6 @@ Promise.all([injector('settings'), injector('app'), injector('agent-manager'), i
         agentMgr.attach(server);
         browserMgr.attach(server);
 
-        // Start legacy agent server.
-        const agentServer = settings.agent.SSLOptions
-            ? https.createServer(settings.agent.SSLOptions) : http.createServer();
-
-        agentServer.listen(settings.agent.port);
-        agentServer.on('error', _onError.bind(null, settings.agent.port));
-        agentServer.on('listening', _onListening.bind(null, agentServer.address()));
-
-        agentMgr.attachLegacy(agentServer);
-
         // Used for automated test.
         if (process.send)
             process.send('running');

From 573624796b171b2420b87657598198f40a91f6bb Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@gridgain.com>
Date: Wed, 1 Mar 2017 22:09:40 +0700
Subject: [PATCH 169/446] Implemented support for enforce join order flag.
 (cherry picked from commit a7f77d4)

---
 .../internal/visor/query/VisorQueryArgV3.java | 51 +++++++++++++++++++
 .../internal/visor/query/VisorQueryJob.java   |  6 +--
 .../resources/META-INF/classnames.properties  |  1 +
 3 files changed, 55 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java
new file mode 100644
index 0000000000000..f32c00a79484b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+/**
+ * Arguments for {@link VisorQueryTask}.
+ */
+public class VisorQueryArgV3 extends VisorQueryArgV2 {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Enforce join order flag. */
+    private final boolean enforceJoinOrder;
+
+    /**
+     * @param cacheName Cache name for query.
+     * @param qryTxt Query text.
+     * @param distributedJoins If {@code true} then distributed joins enabled.
+     * @param enforceJoinOrder If {@code true} then enforce join order.
+     * @param loc Flag whether to execute query locally.
+     * @param pageSize Result batch size.
+     */
+    public VisorQueryArgV3(String cacheName, String qryTxt,
+        boolean distributedJoins, boolean enforceJoinOrder, boolean loc, int pageSize) {
+        super(cacheName, qryTxt, distributedJoins, loc, pageSize);
+
+        this.enforceJoinOrder = enforceJoinOrder;
+    }
+
+    /**
+     * @return Enforce join order flag.
+     */
+    public boolean enforceJoinOrder() {
+        return enforceJoinOrder;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
index c66b2dda8d22c..1ac90ad6a3491 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
@@ -131,9 +131,8 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 if (scanWithFilter) {
                     boolean caseSensitive = qryTxt.startsWith(SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE);
 
-                    String ptrn = caseSensitive
-                        ? qryTxt.substring(SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE.length())
-                        : qryTxt.substring(SCAN_CACHE_WITH_FILTER.length());
+                    String ptrn = qryTxt.substring(
+                        caseSensitive ? SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE.length() : SCAN_CACHE_WITH_FILTER.length());
 
                     filter = new VisorQueryScanSubstringFilter(caseSensitive, ptrn);
                 }
@@ -162,6 +161,7 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 qry.setPageSize(arg.pageSize());
                 qry.setLocal(arg.local());
                 qry.setDistributedJoins(arg instanceof VisorQueryArgV2 && ((VisorQueryArgV2)arg).distributedJoins());
+                qry.setEnforceJoinOrder(arg instanceof VisorQueryArgV3 && ((VisorQueryArgV3)arg).enforceJoinOrder());
 
                 long start = U.currentTimeMillis();
 
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 8a6dc6603c4f1..db486a5600695 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -1792,6 +1792,7 @@ org.apache.ignite.internal.visor.node.VisorSpisConfiguration
 org.apache.ignite.internal.visor.node.VisorTransactionConfiguration
 org.apache.ignite.internal.visor.query.VisorQueryArg
 org.apache.ignite.internal.visor.query.VisorQueryArgV2
+org.apache.ignite.internal.visor.query.VisorQueryArgV3
 org.apache.ignite.internal.visor.query.VisorQueryCleanupTask
 org.apache.ignite.internal.visor.query.VisorQueryCleanupTask$VisorQueryCleanupJob
 org.apache.ignite.internal.visor.query.VisorQueryField

From a30183ac821507fbdaa6f0cc2c6ef25ca2677867 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Mon, 20 Feb 2017 18:23:33 +0700
Subject: [PATCH 170/446] IGNITE-4717 Fixed hangs in VisorCacheClearTask.
 (cherry picked from commit 76f3060)

---
 .../visor/cache/VisorCacheClearTask.java      | 88 +++++--------------
 .../visor/compute/VisorGatewayTask.java       | 30 ++++++-
 2 files changed, 49 insertions(+), 69 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
index 1f1a6fb106bd3..0c8476fa91b10 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.internal.visor.cache;
 
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.internal.processors.task.GridInternal;
@@ -26,7 +25,6 @@
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.resources.JobContextResource;
@@ -90,17 +88,11 @@ private VisorCacheClearJob(String cacheName, boolean debug) {
         }
 
         /**
-         * @param subJob Sub job to execute asynchronously.
+         * @param fut Future for asynchronous cache operation.
          * @param idx Index.
          * @return {@code true} If subJob was not completed and this job should be suspended.
          */
-        private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
-            IgniteCompute compute = ignite.compute(ignite.cluster().forCacheNodes(cacheName)).withAsync();
-
-            compute.call(subJob);
-
-            IgniteFuture<Integer> fut = compute.future();
-
+        private boolean callAsync(IgniteFuture<Integer> fut, int idx) {
             futs[idx] = fut;
 
             if (fut.isDone())
@@ -119,16 +111,28 @@ private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
                 futs = new IgniteFuture[3];
 
             if (futs[0] == null || futs[1] == null || futs[2] == null) {
-                IgniteCache cache = ignite.cache(cacheName);
+                IgniteCache cache = ignite.cache(cacheName).withAsync();
+
+                if (futs[0] == null) {
+                    cache.size(CachePeekMode.PRIMARY);
+
+                    if (callAsync(cache.<Integer>future(), 0))
+                        return null;
+                }
 
-                if (futs[0] == null && callAsync(new VisorCacheSizeCallable(cache), 0))
-                    return null;
+                if (futs[1] == null) {
+                    cache.clear();
 
-                if (futs[1] == null && callAsync(new VisorCacheClearCallable(cache), 1))
-                    return null;
+                    if (callAsync(cache.<Integer>future(), 1))
+                        return null;
+                }
+                
+                if (futs[2] == null) {
+                    cache.size(CachePeekMode.PRIMARY);
 
-                if (futs[2] == null && callAsync(new VisorCacheSizeCallable(cache), 2))
-                    return null;
+                    if (callAsync(cache.<Integer>future(), 2))
+                        return null;
+                }
             }
 
             assert futs[0].isDone() && futs[1].isDone() && futs[2].isDone();
@@ -141,54 +145,4 @@ private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
             return S.toString(VisorCacheClearJob.class, this);
         }
     }
-
-    /**
-     * Callable to get cache size.
-     */
-    @GridInternal
-    private static class VisorCacheSizeCallable implements IgniteCallable<Integer> {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /** */
-        private final IgniteCache cache;
-
-        /**
-         * @param cache Cache to take size from.
-         */
-        private VisorCacheSizeCallable(IgniteCache cache) {
-            this.cache = cache;
-        }
-
-        /** {@inheritDoc} */
-        @Override public Integer call() throws Exception {
-            return cache.size(CachePeekMode.PRIMARY);
-        }
-    }
-
-    /**
-     * Callable to clear cache.
-     */
-    @GridInternal
-    private static class VisorCacheClearCallable implements IgniteCallable<Integer> {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /** */
-        private final IgniteCache cache;
-
-        /**
-         * @param cache Cache to clear.
-         */
-        private VisorCacheClearCallable(IgniteCache cache) {
-            this.cache = cache;
-        }
-
-        /** {@inheritDoc} */
-        @Override public Integer call() throws Exception {
-            cache.clear();
-
-            return 0;
-        }
-    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index 2539a26c558c9..a64ec6d506d6b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -29,21 +29,26 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.compute.ComputeJobResult;
 import org.apache.ignite.compute.ComputeJobResultPolicy;
 import org.apache.ignite.compute.ComputeTask;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.lang.GridTuple3;
+import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.JobContextResource;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -101,9 +106,16 @@ private static class VisorGatewayJob extends ComputeJobAdapter {
         @IgniteInstanceResource
         protected transient IgniteEx ignite;
 
+        /** Auto-inject job context. */
+        @JobContextResource
+        protected transient ComputeJobContext jobCtx;
+
         /** Arguments count. */
         private final int argsCnt;
 
+        /** Future for spawned task. */
+        private transient IgniteFuture fut;
+
         /**
          * Create job with specified argument.
          *
@@ -284,6 +296,9 @@ private static boolean isBuildInObject(Class cls) {
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
         @Override public Object execute() throws IgniteException {
+            if (fut != null)
+                return fut.get();
+
             String nidsArg = argument(0);
             String taskName = argument(1);
 
@@ -355,8 +370,19 @@ else if (isBuildInObject(argCls))
                 }
             }
 
-            return ignite.compute(ignite.cluster().forNodeIds(nids))
-                .execute(taskName, new VisorTaskArgument<>(nids, jobArgs, false));
+            IgniteCompute comp = ignite.compute(ignite.cluster().forNodeIds(nids)).withAsync();
+            
+            comp.execute(taskName, new VisorTaskArgument<>(nids, jobArgs, false));
+
+            fut = comp.future();
+
+            fut.listen(new CI1<IgniteFuture<Object>>() {
+                @Override public void apply(IgniteFuture<Object> f) {
+                    jobCtx.callcc();
+                }
+            });
+
+            return jobCtx.holdcc();
         }
     }
 }

From 5f5cce4f36e6bc0a3468ba1d80fa5f3d158e927f Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@gridgain.com>
Date: Wed, 1 Mar 2017 22:09:40 +0700
Subject: [PATCH 171/446] Implemented support for enforce join order flag.
 (cherry picked from commit a7f77d4)

---
 .../internal/visor/query/VisorQueryArgV3.java | 51 +++++++++++++++++++
 .../internal/visor/query/VisorQueryJob.java   |  6 +--
 .../resources/META-INF/classnames.properties  |  1 +
 3 files changed, 55 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java
new file mode 100644
index 0000000000000..f32c00a79484b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+/**
+ * Arguments for {@link VisorQueryTask}.
+ */
+public class VisorQueryArgV3 extends VisorQueryArgV2 {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Enforce join order flag. */
+    private final boolean enforceJoinOrder;
+
+    /**
+     * @param cacheName Cache name for query.
+     * @param qryTxt Query text.
+     * @param distributedJoins If {@code true} then distributed joins enabled.
+     * @param enforceJoinOrder If {@code true} then enforce join order.
+     * @param loc Flag whether to execute query locally.
+     * @param pageSize Result batch size.
+     */
+    public VisorQueryArgV3(String cacheName, String qryTxt,
+        boolean distributedJoins, boolean enforceJoinOrder, boolean loc, int pageSize) {
+        super(cacheName, qryTxt, distributedJoins, loc, pageSize);
+
+        this.enforceJoinOrder = enforceJoinOrder;
+    }
+
+    /**
+     * @return Enforce join order flag.
+     */
+    public boolean enforceJoinOrder() {
+        return enforceJoinOrder;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
index c66b2dda8d22c..1ac90ad6a3491 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
@@ -131,9 +131,8 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 if (scanWithFilter) {
                     boolean caseSensitive = qryTxt.startsWith(SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE);
 
-                    String ptrn = caseSensitive
-                        ? qryTxt.substring(SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE.length())
-                        : qryTxt.substring(SCAN_CACHE_WITH_FILTER.length());
+                    String ptrn = qryTxt.substring(
+                        caseSensitive ? SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE.length() : SCAN_CACHE_WITH_FILTER.length());
 
                     filter = new VisorQueryScanSubstringFilter(caseSensitive, ptrn);
                 }
@@ -162,6 +161,7 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 qry.setPageSize(arg.pageSize());
                 qry.setLocal(arg.local());
                 qry.setDistributedJoins(arg instanceof VisorQueryArgV2 && ((VisorQueryArgV2)arg).distributedJoins());
+                qry.setEnforceJoinOrder(arg instanceof VisorQueryArgV3 && ((VisorQueryArgV3)arg).enforceJoinOrder());
 
                 long start = U.currentTimeMillis();
 
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index df605dafb2b34..7bebe24f8a51b 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -1819,6 +1819,7 @@ org.apache.ignite.internal.visor.node.VisorSpisConfiguration
 org.apache.ignite.internal.visor.node.VisorTransactionConfiguration
 org.apache.ignite.internal.visor.query.VisorQueryArg
 org.apache.ignite.internal.visor.query.VisorQueryArgV2
+org.apache.ignite.internal.visor.query.VisorQueryArgV3
 org.apache.ignite.internal.visor.query.VisorQueryCleanupTask
 org.apache.ignite.internal.visor.query.VisorQueryCleanupTask$VisorQueryCleanupJob
 org.apache.ignite.internal.visor.query.VisorQueryField

From 7292a5306c90d5c48c520bf9ffad388769250488 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Mon, 20 Feb 2017 18:23:33 +0700
Subject: [PATCH 172/446] IGNITE-4717 Fixed hangs in VisorCacheClearTask.
 (cherry picked from commit 76f3060)

---
 .../visor/cache/VisorCacheClearTask.java      | 88 +++++--------------
 .../visor/compute/VisorGatewayTask.java       | 30 ++++++-
 2 files changed, 49 insertions(+), 69 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
index 1f1a6fb106bd3..0c8476fa91b10 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.internal.visor.cache;
 
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.internal.processors.task.GridInternal;
@@ -26,7 +25,6 @@
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.resources.JobContextResource;
@@ -90,17 +88,11 @@ private VisorCacheClearJob(String cacheName, boolean debug) {
         }
 
         /**
-         * @param subJob Sub job to execute asynchronously.
+         * @param fut Future for asynchronous cache operation.
          * @param idx Index.
          * @return {@code true} If subJob was not completed and this job should be suspended.
          */
-        private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
-            IgniteCompute compute = ignite.compute(ignite.cluster().forCacheNodes(cacheName)).withAsync();
-
-            compute.call(subJob);
-
-            IgniteFuture<Integer> fut = compute.future();
-
+        private boolean callAsync(IgniteFuture<Integer> fut, int idx) {
             futs[idx] = fut;
 
             if (fut.isDone())
@@ -119,16 +111,28 @@ private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
                 futs = new IgniteFuture[3];
 
             if (futs[0] == null || futs[1] == null || futs[2] == null) {
-                IgniteCache cache = ignite.cache(cacheName);
+                IgniteCache cache = ignite.cache(cacheName).withAsync();
+
+                if (futs[0] == null) {
+                    cache.size(CachePeekMode.PRIMARY);
+
+                    if (callAsync(cache.<Integer>future(), 0))
+                        return null;
+                }
 
-                if (futs[0] == null && callAsync(new VisorCacheSizeCallable(cache), 0))
-                    return null;
+                if (futs[1] == null) {
+                    cache.clear();
 
-                if (futs[1] == null && callAsync(new VisorCacheClearCallable(cache), 1))
-                    return null;
+                    if (callAsync(cache.<Integer>future(), 1))
+                        return null;
+                }
+                
+                if (futs[2] == null) {
+                    cache.size(CachePeekMode.PRIMARY);
 
-                if (futs[2] == null && callAsync(new VisorCacheSizeCallable(cache), 2))
-                    return null;
+                    if (callAsync(cache.<Integer>future(), 2))
+                        return null;
+                }
             }
 
             assert futs[0].isDone() && futs[1].isDone() && futs[2].isDone();
@@ -141,54 +145,4 @@ private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
             return S.toString(VisorCacheClearJob.class, this);
         }
     }
-
-    /**
-     * Callable to get cache size.
-     */
-    @GridInternal
-    private static class VisorCacheSizeCallable implements IgniteCallable<Integer> {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /** */
-        private final IgniteCache cache;
-
-        /**
-         * @param cache Cache to take size from.
-         */
-        private VisorCacheSizeCallable(IgniteCache cache) {
-            this.cache = cache;
-        }
-
-        /** {@inheritDoc} */
-        @Override public Integer call() throws Exception {
-            return cache.size(CachePeekMode.PRIMARY);
-        }
-    }
-
-    /**
-     * Callable to clear cache.
-     */
-    @GridInternal
-    private static class VisorCacheClearCallable implements IgniteCallable<Integer> {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /** */
-        private final IgniteCache cache;
-
-        /**
-         * @param cache Cache to clear.
-         */
-        private VisorCacheClearCallable(IgniteCache cache) {
-            this.cache = cache;
-        }
-
-        /** {@inheritDoc} */
-        @Override public Integer call() throws Exception {
-            cache.clear();
-
-            return 0;
-        }
-    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index 2539a26c558c9..a64ec6d506d6b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -29,21 +29,26 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.compute.ComputeJobResult;
 import org.apache.ignite.compute.ComputeJobResultPolicy;
 import org.apache.ignite.compute.ComputeTask;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.lang.GridTuple3;
+import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.JobContextResource;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -101,9 +106,16 @@ private static class VisorGatewayJob extends ComputeJobAdapter {
         @IgniteInstanceResource
         protected transient IgniteEx ignite;
 
+        /** Auto-inject job context. */
+        @JobContextResource
+        protected transient ComputeJobContext jobCtx;
+
         /** Arguments count. */
         private final int argsCnt;
 
+        /** Future for spawned task. */
+        private transient IgniteFuture fut;
+
         /**
          * Create job with specified argument.
          *
@@ -284,6 +296,9 @@ private static boolean isBuildInObject(Class cls) {
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
         @Override public Object execute() throws IgniteException {
+            if (fut != null)
+                return fut.get();
+
             String nidsArg = argument(0);
             String taskName = argument(1);
 
@@ -355,8 +370,19 @@ else if (isBuildInObject(argCls))
                 }
             }
 
-            return ignite.compute(ignite.cluster().forNodeIds(nids))
-                .execute(taskName, new VisorTaskArgument<>(nids, jobArgs, false));
+            IgniteCompute comp = ignite.compute(ignite.cluster().forNodeIds(nids)).withAsync();
+            
+            comp.execute(taskName, new VisorTaskArgument<>(nids, jobArgs, false));
+
+            fut = comp.future();
+
+            fut.listen(new CI1<IgniteFuture<Object>>() {
+                @Override public void apply(IgniteFuture<Object> f) {
+                    jobCtx.callcc();
+                }
+            });
+
+            return jobCtx.holdcc();
         }
     }
 }

From 620235f3ca0fa32c1f4a4dc1a5e0bd5545f6e07e Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Thu, 2 Mar 2017 11:05:19 +0700
Subject: [PATCH 173/446] Minor cleanup.

---
 .../ignite/internal/visor/cache/VisorCacheTypeMetadata.java | 6 ++++++
 .../ignite/internal/visor/node/VisorIgfsConfiguration.java  | 1 -
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
index f17e5889663fc..c87ad05d824e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.cache.store.jdbc.JdbcTypeField;
 import org.apache.ignite.internal.LessNamingBean;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 
@@ -372,4 +373,9 @@ public Collection<String> txtFlds() {
     public Map<String, LinkedHashMap<String, IgniteBiTuple<String, Boolean>>> grps() {
         return grps;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorCacheTypeMetadata.class, this);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index 9f7652b7d5e3b..cb10c1c3bdc38 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -25,7 +25,6 @@
 import org.apache.ignite.configuration.FileSystemConfiguration;
 import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration;
 import org.apache.ignite.igfs.IgfsMode;
-import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
 import org.apache.ignite.internal.LessNamingBean;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;

From 840ab997436dfad8cdbb4ad182de24bceec37962 Mon Sep 17 00:00:00 2001
From: AKuznetsov <akuznetsov@gridgain.com>
Date: Tue, 14 Feb 2017 20:54:31 +0700
Subject: [PATCH 174/446] IGNITE-4436 API for collecting list of running
 queries and cancel them. (cherry picked from commit 4923734)

---
 .../processors/query/GridQueryIndexing.java   |  17 +-
 .../processors/query/GridQueryProcessor.java  |  32 ++-
 .../query/GridRunningQueryInfo.java           | 132 +++++++++++
 .../internal/visor/VisorMultiNodeTask.java    |   2 +-
 .../visor/query/VisorCancelQueriesTask.java   |  72 ++++++
 .../query/VisorCollectRunningQueriesTask.java |  96 ++++++++
 .../visor/query/VisorRunningQuery.java        | 132 +++++++++++
 .../cache/query/GridCacheTwoStepQuery.java    |  18 +-
 .../processors/query/h2/IgniteH2Indexing.java |  86 +++++++-
 .../query/h2/sql/GridSqlQuerySplitter.java    |   4 +-
 .../h2/twostep/GridReduceQueryExecutor.java   |  60 ++++-
 .../cache/CacheSqlQueryValueCopySelfTest.java | 208 +++++++++++++++++-
 .../GridCacheCrossCacheQuerySelfTest.java     |   2 +-
 .../h2/GridIndexingSpiAbstractSelfTest.java   |   7 +
 14 files changed, 826 insertions(+), 42 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index ef39d96f4d5ec..ca047244a97f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -240,8 +240,23 @@ public void store(@Nullable String spaceName, GridQueryTypeDescriptor type, Cach
      */
     public PreparedStatement prepareNativeStatement(String schema, String sql) throws SQLException;
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> runningQueries(long duration);
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries ID's to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries);
+
     /**
      * Cancels all executing queries.
      */
     public void cancelAllQueries();
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index f8919a105b601..10bf75a01113d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -44,7 +44,6 @@
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryField;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryObjectBuilder;
@@ -119,7 +118,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
     private static final int QRY_DETAIL_METRICS_EVICTION_FREQ = 3_000;
 
     /** */
-    private static Set<Class<?>> SQL_TYPES = new HashSet<>(F.<Class<?>>asList(
+    private static final Set<Class<?>> SQL_TYPES = new HashSet<>(F.<Class<?>>asList(
         Integer.class,
         Boolean.class,
         Byte.class,
@@ -919,6 +918,29 @@ public <K, V> Iterator<Cache.Entry<K, V>> queryLocal(
         }
     }
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+        if (moduleEnabled())
+            return idx.runningQueries(duration);
+
+        return Collections.emptyList();
+    }
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries ID's to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries) {
+        if (moduleEnabled())
+            idx.cancelQueries(queries);
+    }
+
     /**
      * @param sqlQry Sql query.
      * @param params Params.
@@ -2722,7 +2744,7 @@ private interface PropertyAccessor {
     }
 
     /** Accessor that deals with fields. */
-    private final static class FieldAccessor implements PropertyAccessor {
+    private static final class FieldAccessor implements PropertyAccessor {
         /** Field to access. */
         private final Field fld;
 
@@ -2765,7 +2787,7 @@ private FieldAccessor(Field fld) {
     }
 
     /** Getter and setter methods based accessor. */
-    private final static class MethodsAccessor implements PropertyAccessor {
+    private static final class MethodsAccessor implements PropertyAccessor {
         /** */
         private final Method getter;
 
@@ -2823,7 +2845,7 @@ private MethodsAccessor(Method getter, Method setter, String propName) {
     }
 
     /** Accessor with getter only. */
-    private final static class ReadOnlyMethodsAccessor implements PropertyAccessor {
+    private static final class ReadOnlyMethodsAccessor implements PropertyAccessor {
         /** */
         private final Method getter;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
new file mode 100644
index 0000000000000..d77c8c01a8ede
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
+
+/**
+ * Query descriptor.
+ */
+public class GridRunningQueryInfo {
+    /** */
+    private final long id;
+
+    /** */
+    private final String qry;
+
+    /** Query type. */
+    private final GridCacheQueryType qryType;
+
+    /** */
+    private final String cache;
+
+    /** */
+    private final long startTime;
+
+    /** */
+    private final GridQueryCancel cancel;
+
+    /** */
+    private final boolean loc;
+
+    /**
+     * @param id Query ID.
+     * @param qry Query text.
+     * @param qryType Query type.
+     * @param cache Cache where query was executed.
+     * @param startTime Query start time.
+     * @param cancel Query cancel.
+     * @param loc Local query flag.
+     */
+    public GridRunningQueryInfo(Long id, String qry, GridCacheQueryType qryType, String cache, long startTime,
+        GridQueryCancel cancel, boolean loc) {
+        this.id = id;
+        this.qry = qry;
+        this.qryType = qryType;
+        this.cache = cache;
+        this.startTime = startTime;
+        this.cancel = cancel;
+        this.loc = loc;
+    }
+
+    /**
+     * @return Query ID.
+     */
+    public Long id() {
+        return id;
+    }
+
+    /**
+     * @return Query text.
+     */
+    public String query() {
+        return qry;
+    }
+
+    /**
+     * @return Query type.
+     */
+    public GridCacheQueryType queryType() {
+        return qryType;
+    }
+
+    /**
+     * @return Cache where query was executed.
+     */
+    public String cache() {
+        return cache;
+    }
+
+    /**
+     * @return Query start time.
+     */
+    public long startTime() {
+        return startTime;
+    }
+
+    /**
+     * @param curTime Current time.
+     * @param duration Duration of long query.
+     * @return {@code true} if this query should be considered as long running query.
+     */
+    public boolean longQuery(long curTime, long duration) {
+        return curTime - startTime > duration;
+    }
+
+    /**
+     * Cancel query.
+     */
+    public void cancel() {
+        if (cancel != null)
+            cancel.cancel();
+    }
+
+    /**
+     * @return {@code true} if query can be cancelled.
+     */
+    public boolean cancelable() {
+        return cancel != null;
+    }
+
+    /**
+     * @return {@code true} if query is local.
+     */
+    public boolean local() {
+        return loc;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
index 57f134698ea56..ece1a17953f39 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
@@ -130,4 +130,4 @@ protected Map<? extends ComputeJob, ClusterNode> map0(List<ClusterNode> subgrid,
                 logFinish(ignite.log(), getClass(), start);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
new file mode 100644
index 0000000000000..a6f2d821f15a0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task to cancel queries.
+ */
+@GridInternal
+public class VisorCancelQueriesTask extends VisorOneNodeTask<Collection<Long>, Void> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCancelQueriesJob job(Collection<Long> arg) {
+        return new VisorCancelQueriesJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected Void reduce0(List<ComputeJobResult> results) throws IgniteException {
+        return null;
+    }
+
+    /**
+     * Job to cancel queries on node.
+     */
+    private static class VisorCancelQueriesJob extends VisorJob<Collection<Long>, Void> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with specified argument.
+         *
+         * @param arg Job argument.
+         * @param debug Flag indicating whether debug information should be printed into node log.
+         */
+        protected VisorCancelQueriesJob(@Nullable Collection<Long> arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Void run(@Nullable Collection<Long> queries) throws IgniteException {
+            ignite.context().query().cancelQueries(queries);
+
+            return null;
+        }
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
new file mode 100644
index 0000000000000..2b40e61e74f50
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task to collect currently running queries.
+ */
+@GridInternal
+public class VisorCollectRunningQueriesTask extends VisorMultiNodeTask<Long, Map<UUID, Collection<VisorRunningQuery>>, Collection<VisorRunningQuery>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCollectRunningQueriesJob job(Long arg) {
+        return new VisorCollectRunningQueriesJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected Map<UUID, Collection<VisorRunningQuery>> reduce0(List<ComputeJobResult> results) throws IgniteException {
+        Map<UUID, Collection<VisorRunningQuery>> map = new HashMap<>();
+
+        for (ComputeJobResult res : results)
+            if (res.getException() == null) {
+                Collection<VisorRunningQuery> queries = res.getData();
+
+                map.put(res.getNode().id(), queries);
+            }
+
+        return map;
+    }
+
+    /**
+     * Job to collect currently running queries from node.
+     */
+    private static class VisorCollectRunningQueriesJob extends VisorJob<Long, Collection<VisorRunningQuery>> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with specified argument.
+         *
+         * @param arg Job argument.
+         * @param debug Flag indicating whether debug information should be printed into node log.
+         */
+        protected VisorCollectRunningQueriesJob(@Nullable Long arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Collection<VisorRunningQuery> run(@Nullable Long duration) throws IgniteException {
+            Collection<GridRunningQueryInfo> queries = ignite.context().query()
+                .runningQueries(duration != null ? duration : 0);
+
+            Collection<VisorRunningQuery> res = new ArrayList<>(queries.size());
+
+            long curTime = U.currentTimeMillis();
+
+            for (GridRunningQueryInfo qry : queries)
+                res.add(new VisorRunningQuery(qry.id(), qry.query(), qry.queryType(), qry.cache(),
+                    qry.startTime(), curTime - qry.startTime(),
+                    qry.cancelable(), qry.local()));
+
+            return res;
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java
new file mode 100644
index 0000000000000..fc6bc7a9222d4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
+
+/**
+ * Descriptor of running query.
+ */
+public class VisorRunningQuery implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private long id;
+
+    /** Query text. */
+    private String qry;
+
+    /** Query type. */
+    private GridCacheQueryType qryType;
+
+    /** Cache name for query. */
+    private String cache;
+
+    /** */
+    private long startTime;
+
+    /** */
+    private long duration;
+
+    /** */
+    private boolean cancellable;
+
+    /** */
+    private boolean loc;
+
+    /**
+     * @param id Query ID.
+     * @param qry Query text.
+     * @param qryType Query type.
+     * @param cache Cache where query was executed.
+     * @param startTime Query start time.
+     * @param duration Query current duration.
+     * @param cancellable {@code true} if query can be canceled.
+     * @param loc {@code true} if query is local.
+     */
+    public VisorRunningQuery(long id, String qry, GridCacheQueryType qryType, String cache,
+        long startTime, long duration,
+        boolean cancellable, boolean loc) {
+        this.id = id;
+        this.qry = qry;
+        this.qryType = qryType;
+        this.cache = cache;
+        this.startTime = startTime;
+        this.duration = duration;
+        this.cancellable = cancellable;
+        this.loc = loc;
+    }
+
+    /**
+     * @return Query ID.
+     */
+    public long getId() {
+        return id;
+    }
+
+    /**
+     * @return Query txt.
+     */
+    public String getQuery() {
+        return qry;
+    }
+
+    /**
+     * @return Query type.
+     */
+    public GridCacheQueryType getQueryType() {
+        return qryType;
+    }
+
+    /**
+     * @return Cache name.
+     */
+    public String getCache() {
+        return cache;
+    }
+
+    /**
+     * @return Query start time.
+     */
+    public long getStartTime() {
+        return startTime;
+    }
+
+    /**
+     * @return Query duration.
+     */
+    public long getDuration() {
+        return duration;
+    }
+
+    /**
+     * @return {@code true} if query can be cancelled.
+     */
+    public boolean isCancelable() {
+        return cancellable;
+    }
+
+    /**
+     * @return {@code true} if query is local.
+     */
+    public boolean isLocal() {
+        return loc;
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
index 8dcba2f2f34dc..f53936fc374a1 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
@@ -45,6 +45,9 @@ public class GridCacheTwoStepQuery {
     /** */
     private boolean explain;
 
+    /** */
+    private String originalSql;
+
     /** */
     private Collection<String> spaces;
 
@@ -67,10 +70,12 @@ public class GridCacheTwoStepQuery {
     private List<Integer> extraCaches;
 
     /**
+     * @param originalSql Original query SQL.
      * @param schemas Schema names in query.
      * @param tbls Tables in query.
      */
-    public GridCacheTwoStepQuery(Set<String> schemas, Set<String> tbls) {
+    public GridCacheTwoStepQuery(String originalSql, Set<String> schemas, Set<String> tbls) {
+        this.originalSql = originalSql;
         this.schemas = schemas;
         this.tbls = tbls;
     }
@@ -195,6 +200,13 @@ public void extraCaches(List<Integer> extraCaches) {
         this.extraCaches = extraCaches;
     }
 
+    /**
+     * @return Original query SQL.
+     */
+    public String originalSql() {
+        return originalSql;
+    }
+
     /**
      * @return Spaces.
      */
@@ -223,7 +235,7 @@ public Set<String> schemas() {
     public GridCacheTwoStepQuery copy(Object[] args) {
         assert !explain;
 
-        GridCacheTwoStepQuery cp = new GridCacheTwoStepQuery(schemas, tbls);
+        GridCacheTwoStepQuery cp = new GridCacheTwoStepQuery(originalSql, schemas, tbls);
 
         cp.caches = caches;
         cp.extraCaches = extraCaches;
@@ -250,4 +262,4 @@ public Set<String> tables() {
     @Override public String toString() {
         return S.toString(GridCacheTwoStepQuery.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index e375867807d0c..8c5c2a364cabf 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -52,6 +52,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import javax.cache.Cache;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
@@ -81,6 +82,7 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
 import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
@@ -176,7 +178,11 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_CLEANUP_PERIOD;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_THREAD_USAGE_TIMEOUT;
+import static org.apache.ignite.IgniteSystemProperties.getInteger;
 import static org.apache.ignite.IgniteSystemProperties.getString;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.TEXT;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.FULLTEXT;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.GEO_SPATIAL;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.SORTED;
@@ -283,9 +289,15 @@ public class IgniteH2Indexing implements GridQueryIndexing {
     /** space name -> schema name */
     private final Map<String, String> space2schema = new ConcurrentHashMap8<>();
 
+    /** */
+    private AtomicLong qryIdGen;
+
     /** */
     private GridSpinBusyLock busyLock;
 
+    /** */
+    private final ConcurrentMap<Long, GridRunningQueryInfo> runs = new ConcurrentHashMap8<>();
+
     /** */
     private final ThreadLocal<ConnectionWrapper> connCache = new ThreadLocal<ConnectionWrapper>() {
         @Nullable @Override public ConnectionWrapper get() {
@@ -771,8 +783,19 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
         IndexingQueryFilter filters) throws IgniteCheckedException {
         TableDescriptor tbl = tableDescriptor(spaceName, type);
 
-        if (tbl != null && tbl.luceneIdx != null)
-            return tbl.luceneIdx.query(qry, filters);
+        if (tbl != null && tbl.luceneIdx != null) {
+            GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, TEXT, spaceName,
+                U.currentTimeMillis(), null, true);
+
+            try {
+                runs.put(run.id(), run);
+
+                return tbl.luceneIdx.query(qry, filters);
+            }
+            finally {
+                runs.remove(run.id());
+            }
+        }
 
         return new GridEmptyCloseableIterator<>();
     }
@@ -830,6 +853,11 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
 
                 GridH2QueryContext.set(ctx);
 
+                GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL_FIELDS,
+                    spaceName, U.currentTimeMillis(), cancel, true);
+
+                runs.putIfAbsent(run.id(), run);
+
                 try {
                     ResultSet rs = executeSqlQueryWithTimer(spaceName, stmt, conn, qry, params, timeout, cancel);
 
@@ -837,6 +865,8 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
                 }
                 finally {
                     GridH2QueryContext.clearThreadLocal();
+
+                    runs.remove(run.id());
                 }
             }
         };
@@ -1086,6 +1116,11 @@ public void setupConnection(Connection conn, boolean distributedJoins, boolean e
 
         GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter).distributedJoins(false));
 
+        GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL, spaceName,
+            U.currentTimeMillis(), null, true);
+
+        runs.put(run.id(), run);
+
         try {
             ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, null);
 
@@ -1093,6 +1128,8 @@ public void setupConnection(Connection conn, boolean distributedJoins, boolean e
         }
         finally {
             GridH2QueryContext.clearThreadLocal();
+
+            runs.remove(run.id());
         }
     }
 
@@ -1232,7 +1269,7 @@ public static Session session(Connection c) {
                             try {
                                 ctx.cache().createMissingCaches();
                             }
-                            catch (IgniteCheckedException e1) {
+                            catch (IgniteCheckedException ignored) {
                                 throw new CacheException("Failed to create missing caches.", e);
                             }
 
@@ -1737,6 +1774,8 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
 
         this.busyLock = busyLock;
 
+        qryIdGen = new AtomicLong();
+
         if (SysProperties.serializeJavaObject) {
             U.warn(log, "Serialization of Java objects in H2 was enabled.");
 
@@ -1787,7 +1826,7 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
             marshaller = ctx.config().getMarshaller();
 
             mapQryExec = new GridMapQueryExecutor(busyLock);
-            rdcQryExec = new GridReduceQueryExecutor(busyLock);
+            rdcQryExec = new GridReduceQueryExecutor(qryIdGen, busyLock);
 
             mapQryExec.start(ctx, this);
             rdcQryExec.start(ctx, this);
@@ -2241,6 +2280,37 @@ private static List<IndexColumn> treeIndexColumns(List<IndexColumn> cols, IndexC
         return cols;
     }
 
+
+    /** {@inheritDoc} */
+    @Override public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+        Collection<GridRunningQueryInfo> res = new ArrayList<>();
+
+        res.addAll(runs.values());
+        res.addAll(rdcQryExec.longRunningQueries(duration));
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancelQueries(Collection<Long> queries) {
+        if (!F.isEmpty(queries)) {
+            for (Long qryId : queries) {
+                GridRunningQueryInfo run = runs.get(qryId);
+
+                if (run != null)
+                    run.cancel();
+            }
+
+            rdcQryExec.cancelQueries(queries);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancelAllQueries() {
+        for (Connection conn : conns)
+            U.close(conn, log);
+    }
+
     /**
      * Wrapper to store connection and flag is schema set or not.
      */
@@ -3151,10 +3221,4 @@ private void updateLastUsage() {
             lastUsage = U.currentTimeMillis();
         }
     }
-
-    /** {@inheritDoc} */
-    @Override public void cancelAllQueries() {
-        for (Connection conn : conns)
-            U.close(conn, log);
-    }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 09952cfb8248c..e16431506967a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -174,7 +174,7 @@ public static GridCacheTwoStepQuery split(
         qry = collectAllTables(qry, schemas, tbls);
 
         // Build resulting two step query.
-        GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(schemas, tbls);
+        GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(qry.getSQL(), schemas, tbls);
 
         // Map query will be direct reference to the original query AST.
         // Thus all the modifications will be performed on the original AST, so we should be careful when
@@ -958,4 +958,4 @@ private static GridSqlOperation op(GridSqlOperationType type, GridSqlElement lef
     private static GridSqlFunction function(GridSqlFunctionType type) {
         return new GridSqlFunction(type);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 1f00ed271dabe..ee9976c22846a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -62,6 +62,7 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
 import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
@@ -98,6 +99,7 @@
 import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
 import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REDUCE;
 
 /**
@@ -120,7 +122,7 @@ public class GridReduceQueryExecutor {
     private IgniteLogger log;
 
     /** */
-    private final AtomicLong reqIdGen = new AtomicLong();
+    private final AtomicLong qryIdGen;
 
     /** */
     private final ConcurrentMap<Long, QueryRun> runs = new ConcurrentHashMap8<>();
@@ -167,9 +169,11 @@ public class GridReduceQueryExecutor {
     };
 
     /**
+     * @param qryIdGen Query ID generator.
      * @param busyLock Busy lock.
      */
-    public GridReduceQueryExecutor(GridSpinBusyLock busyLock) {
+    public GridReduceQueryExecutor(AtomicLong qryIdGen, GridSpinBusyLock busyLock) {
+        this.qryIdGen = qryIdGen;
         this.busyLock = busyLock;
     }
 
@@ -493,11 +497,13 @@ public Iterator<List<?>> query(
                 }
             }
 
-            final long qryReqId = reqIdGen.incrementAndGet();
+            final long qryReqId = qryIdGen.incrementAndGet();
 
             final String space = cctx.name();
 
-            final QueryRun r = new QueryRun(h2.connectionForSpace(space), qry.mapQueries().size(), qry.pageSize());
+            final QueryRun r = new QueryRun(qryReqId, qry.originalSql(), space,
+                h2.connectionForSpace(space), qry.mapQueries().size(), qry.pageSize(),
+                U.currentTimeMillis(), cancel);
 
             AffinityTopologyVersion topVer = h2.readyTopologyVersion();
 
@@ -1302,10 +1308,46 @@ public void onDisconnected(IgniteFuture<?> reconnectFut) {
             e.getValue().disconnected(err);
     }
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of IDs and statements of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> longRunningQueries(long duration) {
+        Collection<GridRunningQueryInfo> res = new ArrayList<>();
+
+        long curTime = U.currentTimeMillis();
+
+        for (QueryRun run : runs.values()) {
+            if (run.qry.longQuery(curTime, duration))
+                res.add(run.qry);
+        }
+
+        return res;
+    }
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries IDs to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries) {
+        for (Long qryId : queries) {
+            QueryRun run = runs.get(qryId);
+
+            if (run != null)
+                run.qry.cancel();
+        }
+    }
+
     /**
      * Query run.
      */
     private static class QueryRun {
+        /** */
+        private final GridRunningQueryInfo qry;
+
         /** */
         private final List<GridMergeIndex> idxs;
 
@@ -1322,11 +1364,17 @@ private static class QueryRun {
         private final AtomicReference<Object> state = new AtomicReference<>();
 
         /**
+         * @param id Query ID.
+         * @param qry Query text.
+         * @param cache Cache where query was executed.
          * @param conn Connection.
          * @param idxsCnt Number of indexes.
          * @param pageSize Page size.
+         * @param startTime Start time.
+         * @param cancel Query cancel handler.
          */
-        private QueryRun(Connection conn, int idxsCnt, int pageSize) {
+        private QueryRun(Long id, String qry, String cache, Connection conn, int idxsCnt, int pageSize, long startTime, GridQueryCancel cancel) {
+            this.qry = new GridRunningQueryInfo(id, qry, SQL_FIELDS, cache, startTime, cancel, false);
             this.conn = (JdbcConnection)conn;
             this.idxs = new ArrayList<>(idxsCnt);
             this.pageSize = pageSize > 0 ? pageSize : GridCacheTwoStepQuery.DFLT_PAGE_SIZE;
@@ -1384,4 +1432,4 @@ private ExplicitPartitionsSpecializer(Map<ClusterNode,IntArray> partsMap) {
             return copy(msg, n, partsMap);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
index e47e893530384..66e7e4abf9cdb 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
@@ -17,15 +17,23 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.Query;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cache.query.annotations.QuerySqlFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -54,6 +62,7 @@ public class CacheSqlQueryValueCopySelfTest extends GridCommonAbstractTest {
 
         cc.setCopyOnRead(true);
         cc.setIndexedTypes(Integer.class, Value.class);
+        cc.setSqlFunctionClasses(TestSQLFunctions.class);
 
         cfg.setCacheConfiguration(cc);
 
@@ -72,7 +81,7 @@ public class CacheSqlQueryValueCopySelfTest extends GridCommonAbstractTest {
         IgniteCache<Integer, Value> cache = grid(0).cache(null);
 
         for (int i = 0; i < KEYS; i++)
-            cache.put(i, new Value("before"));
+            cache.put(i, new Value(i, "before-" + i));
     }
 
     /** {@inheritDoc} */
@@ -195,17 +204,148 @@ public void testLocalSqlFieldsQuery() {
         check(cache);
     }
 
-    /** */
-    private static class Value {
-        /** */
-        private String str;
+    /**
+     * Run specified query in separate thread.
+     *
+     * @param qry Query to execute.
+     */
+    private IgniteInternalFuture<?> runQueryAsync(final Query<?> qry) throws Exception {
+        return multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    log.info(">>> Query started");
+
+                    grid(0).cache(null).query(qry).getAll();
+
+                    log.info(">>> Query finished");
+                }
+                catch (Throwable e) {
+                    e.printStackTrace();
+                }
+            }
+        }, 1, "run-query");
+    }
 
-        /**
-         * @param str String.
-         */
-        public Value(String str) {
-            this.str = str;
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRunningSqlFieldsQuery() throws Exception {
+        IgniteInternalFuture<?> fut = runQueryAsync(new SqlFieldsQuery("select _val, sleep(1000) from Value limit 3"));
+
+        Thread.sleep(500);
+
+        GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+
+        SqlFieldsQuery qry = new SqlFieldsQuery("select _val, sleep(1000) from Value limit 3");
+        qry.setLocal(true);
+
+        fut = runQueryAsync(qry);
+
+        Thread.sleep(500);
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+    }
+
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRunningSqlQuery() throws Exception {
+        IgniteInternalFuture<?> fut = runQueryAsync(new SqlQuery<Integer, Value>(Value.class, "id > sleep(100)"));
+
+        Thread.sleep(500);
+
+        GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+
+        SqlQuery<Integer, Value> qry = new SqlQuery<>(Value.class, "id > sleep(100)");
+        qry.setLocal(true);
+
+        fut = runQueryAsync(qry);
+
+        Thread.sleep(500);
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+    }
+
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCancelingSqlFieldsQuery() throws Exception {
+        runQueryAsync(new SqlFieldsQuery("select * from (select _val, sleep(100) from Value limit 50)"));
+
+        Thread.sleep(500);
+
+        final GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        final Collection<GridRunningQueryInfo> finalQueries = queries;
+
+        for (GridRunningQueryInfo query : finalQueries)
+            qryProc.cancelQueries(Collections.singleton(query.id()));
+
+        int n = 100;
+
+        // Give cluster some time to cancel query and cleanup resources.
+        while (n > 0) {
+            Thread.sleep(100);
+
+            queries = qryProc.runningQueries(0);
+
+            if (queries.isEmpty())
+                break;
+
+            log.info(">>>> Wait for cancel: " + n);
+
+            n--;
         }
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
     }
 
     /**
@@ -218,9 +358,53 @@ private void check(IgniteCache<Integer, Value> cache) {
         for (Cache.Entry<Integer, Value> entry : cache) {
             cnt++;
 
-            assertEquals("before", entry.getValue().str);
+            assertEquals("before-" + entry.getKey(), entry.getValue().str);
         }
 
         assertEquals(KEYS, cnt);
     }
-}
\ No newline at end of file
+
+    /** */
+    private static class Value {
+        /** */
+        @QuerySqlField
+        private int id;
+
+        /** */
+        @QuerySqlField
+        private String str;
+
+        /**
+         * @param id ID.
+         * @param str String.
+         */
+        public Value(int id, String str) {
+            this.id = id;
+            this.str = str;
+        }
+    }
+
+    /**
+     * Utility class with custom SQL functions.
+     */
+    public static class TestSQLFunctions {
+        /**
+         * Sleep function to simulate long running queries.
+         *
+         * @param x Time to sleep.
+         * @return Return specified argument.
+         */
+        @QuerySqlFunction
+        public static long sleep(long x) {
+            if (x >= 0)
+                try {
+                    Thread.sleep(x);
+                }
+                catch (InterruptedException ignored) {
+                    // No-op.
+                }
+
+            return x;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
index 1f10593e61e0c..01fefa3ac95e3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
@@ -477,4 +477,4 @@ public int getStoreId() {
             return storeId;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
index 17151d8730287..09b27d2ca3f77 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
@@ -106,6 +106,9 @@ protected void startIndexing(IgniteH2Indexing spi) throws Exception {
         spi.registerCache(null, cacheCfg("B"));
     }
 
+    /**
+     * @param name Name.
+     */
     private CacheConfiguration cacheCfg(String name) {
         CacheConfiguration<?,?> cfg = new CacheConfiguration<>();
 
@@ -114,6 +117,7 @@ private CacheConfiguration cacheCfg(String name) {
         return cfg;
     }
 
+    /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         idx.stop();
 
@@ -182,6 +186,9 @@ private IgniteH2Indexing getIndexing() {
         return idx;
     }
 
+    /**
+     * @return {@code true} if OFF-HEAP mode should be tested.
+     */
     protected boolean offheap() {
         return false;
     }

From c2b201ed5bf4bea5dd5aa013f685fdfa02544527 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 2 Mar 2017 10:27:13 +0300
Subject: [PATCH 175/446] IGNITE-3386 - Reentrant lock is lost when owner
 leaves topology

---
 .../main/java/org/apache/ignite/Ignite.java   |  1 +
 .../DataStructuresProcessor.java              |  3 +-
 .../datastructures/GridCacheLockImpl.java     | 17 ++++-
 .../internal/GridCacheRecreateLockTest.java   | 62 +++++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |  2 +
 5 files changed, 82 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java
index 0de08d56584a8..7f8974c20fd9e 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignite.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java
@@ -527,6 +527,7 @@ public IgniteSemaphore semaphore(String name, int cnt, boolean failoverSafe, boo
      *      all threads on other nodes waiting to acquire lock are interrupted.
      * @param fair If {@code True}, fair lock will be created.
      * @param create Boolean flag indicating whether data structure should be created if does not exist.
+     *      Will re-create lock if the node that stored the lock left topology and there are no backups left.
      * @return ReentrantLock for the given name.
      * @throws IgniteException If reentrant lock could not be fetched or created.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
index 1cad22f33e5ef..698efd9d7665d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
@@ -1396,7 +1396,8 @@ public IgniteLock reentrantLock(final String name, final boolean failoverSafe, f
                         name,
                         key,
                         reentrantLockView,
-                        dsCacheCtx);
+                        dsCacheCtx,
+                        create);
 
                     dsMap.put(key, reentrantLock0);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
index 3ab7289237a63..1cf78faa5bd4e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
@@ -105,6 +105,9 @@ public final class GridCacheLockImpl implements GridCacheLockEx, Externalizable
     /** Flag indicating that every operation on this lock should be interrupted. */
     private volatile boolean interruptAll;
 
+    /** Re-create flag. */
+    private volatile boolean reCreate;
+
     /**
      * Empty constructor required by {@link Externalizable}.
      */
@@ -522,7 +525,14 @@ protected boolean compareAndSetGlobalState(final int expVal, final int newVal,
                                 GridCacheLockState val = lockView.get(key);
 
                                 if (val == null)
-                                    throw new IgniteCheckedException("Failed to find reentrant lock with given name: " + name);
+                                    if (reCreate) {
+                                        val = new GridCacheLockState(0, ctx.nodeId(), 0, failoverSafe, fair);
+
+                                        lockView.put(key, val);
+                                    }
+                                    else
+                                        throw new IgniteCheckedException("Failed to find reentrant lock with " +
+                                            "the given name: " + name);
 
                                 final long newThreadID = newThread.getId();
 
@@ -1048,12 +1058,14 @@ protected IgniteConditionObject(String name, ConditionObject object) {
      * @param key Reentrant lock key.
      * @param lockView Reentrant lock projection.
      * @param ctx Cache context.
+     * @param reCreate If {@code true} reentrant lock will be re-created in case it is not in cache.
      */
     @SuppressWarnings("unchecked")
     public GridCacheLockImpl(String name,
         GridCacheInternalKey key,
         IgniteInternalCache<GridCacheInternalKey, GridCacheLockState> lockView,
-        GridCacheContext ctx) {
+        GridCacheContext ctx,
+        boolean reCreate) {
         assert name != null;
         assert key != null;
         assert ctx != null;
@@ -1063,6 +1075,7 @@ public GridCacheLockImpl(String name,
         this.key = key;
         this.lockView = lockView;
         this.ctx = ctx;
+        this.reCreate = reCreate;
 
         log = ctx.logger(getClass());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
new file mode 100644
index 0000000000000..ae4ef87151c1e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
@@ -0,0 +1,62 @@
+package org.apache.ignite.internal;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLock;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+
+/**
+ * Create lock after owner node left topology test
+ */
+@GridCommonTest(group = "Kernal Self")
+public class GridCacheRecreateLockTest extends GridCommonAbstractTest {
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void test() throws Exception {
+        final Ignite ignite = startNodeAndLock("node1");
+
+        new Thread(new Runnable() {
+            @Override public void run() {
+                try {
+                    Thread.sleep(2000);
+                }
+                catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+
+                ignite.close();
+            }
+        }).start();
+
+        startNodeAndLock("node2");
+    }
+
+    private Ignite startNodeAndLock(String name) {
+        try {
+            IgniteConfiguration cfg = new IgniteConfiguration();
+            cfg.setGridName(name);
+
+            Ignite ignite = Ignition.start(cfg);
+
+            IgniteLock lock = ignite.reentrantLock("lock", true, true, true);
+
+            System.out.println("acquiring lock");
+
+            lock.lock();
+
+            System.out.println("acquired lock");
+
+            return ignite;
+        }
+        catch (Exception e) {
+            assertTrue(false);
+        }
+
+        return null;
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 8a501fdb801d6..7c8b6a94c8f0d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.internal.GridAffinityNoCacheSelfTest;
 import org.apache.ignite.internal.GridAffinitySelfTest;
 import org.apache.ignite.internal.GridAlwaysFailoverSpiFailSelfTest;
+import org.apache.ignite.internal.GridCacheRecreateLockTest;
 import org.apache.ignite.internal.GridCancelOnGridStopSelfTest;
 import org.apache.ignite.internal.GridCancelUnusedJobSelfTest;
 import org.apache.ignite.internal.GridCancelledJobsMetricsSelfTest;
@@ -152,6 +153,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TaskNodeRestartTest.class);
         suite.addTestSuite(IgniteRoundRobinErrorAfterClientReconnectTest.class);
         suite.addTestSuite(PublicThreadpoolStarvationTest.class);
+        suite.addTestSuite(GridCacheRecreateLockTest.class);
 
         return suite;
     }

From 3310f10b9d48d64280d0ebf2464ad892f3f31c52 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 2 Mar 2017 10:37:00 +0300
Subject: [PATCH 176/446] IGNITE-3386 - Minor code style changes

---
 .../internal/GridCacheRecreateLockTest.java   | 32 ++++++++-----------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
index ae4ef87151c1e..ae850f7789745 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
@@ -13,11 +13,10 @@
  */
 @GridCommonTest(group = "Kernal Self")
 public class GridCacheRecreateLockTest extends GridCommonAbstractTest {
-
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void test() throws Exception {
+    public void testLockOwnerLeavesGrid() throws Exception {
         final Ignite ignite = startNodeAndLock("node1");
 
         new Thread(new Runnable() {
@@ -36,27 +35,24 @@ public void test() throws Exception {
         startNodeAndLock("node2");
     }
 
-    private Ignite startNodeAndLock(String name) {
-        try {
-            IgniteConfiguration cfg = new IgniteConfiguration();
-            cfg.setGridName(name);
-
-            Ignite ignite = Ignition.start(cfg);
+    /**
+     * @param name Grid name.
+     * @return Started Ignite instance.
+     */
+    private Ignite startNodeAndLock(String name) throws Exception {
+        IgniteConfiguration cfg = new IgniteConfiguration();
+        cfg.setGridName(name);
 
-            IgniteLock lock = ignite.reentrantLock("lock", true, true, true);
+        Ignite ignite = Ignition.start(cfg);
 
-            System.out.println("acquiring lock");
+        IgniteLock lock = ignite.reentrantLock("lock", true, true, true);
 
-            lock.lock();
+        System.out.println("acquiring lock");
 
-            System.out.println("acquired lock");
+        lock.lock();
 
-            return ignite;
-        }
-        catch (Exception e) {
-            assertTrue(false);
-        }
+        System.out.println("acquired lock");
 
-        return null;
+        return ignite;
     }
 }
\ No newline at end of file

From 7ad8e79fa1077291c50f2f535ecccde6baee0321 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Tue, 7 Mar 2017 14:32:28 +0300
Subject: [PATCH 177/446] ignite-4577 Add non-reachable addresses at the end of
 addresses list

---
 .../ignite/internal/util/IgniteUtils.java     | 14 ++++++-----
 .../tcp/TcpCommunicationSpi.java              | 25 +++++++++++++++++++
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 3fa3f7b92eccb..ba118cb118cea 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1810,15 +1810,16 @@ public static synchronized boolean isLocalHostChanged() throws IOException {
 
     /**
      * @param addrs Addresses.
+     * @return List of reachable addresses.
      */
-    public static List<InetAddress> filterReachable(List<InetAddress> addrs) {
+    public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
         final int reachTimeout = 2000;
 
         if (addrs.isEmpty())
             return Collections.emptyList();
 
         if (addrs.size() == 1) {
-            InetAddress addr = addrs.get(0);
+            InetAddress addr = F.first(addrs);
 
             if (reachable(addr, reachTimeout))
                 return Collections.singletonList(addr);
@@ -1834,8 +1835,7 @@ public static List<InetAddress> filterReachable(List<InetAddress> addrs) {
 
         for (final InetAddress addr : addrs) {
             futs.add(executor.submit(new Runnable() {
-                @Override
-                public void run() {
+                @Override public void run() {
                     if (reachable(addr, reachTimeout)) {
                         synchronized (res) {
                             res.add(addr);
@@ -1848,11 +1848,13 @@ public void run() {
         for (Future<?> fut : futs) {
             try {
                 fut.get();
-            } catch (InterruptedException e) {
+            }
+            catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
 
                 throw new IgniteException("Thread has been interrupted.", e);
-            } catch (ExecutionException e) {
+            }
+            catch (ExecutionException e) {
                 throw new IgniteException(e);
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 94b7efe078e84..81454f827e58f 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2334,6 +2334,31 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
         if (isExtAddrsExist)
             addrs.addAll(extAddrs);
 
+        Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
+
+        for (InetSocketAddress addr : addrs)
+            allInetAddrs.add(addr.getAddress());
+
+        List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
+
+        if (reachableInetAddrs.size() < allInetAddrs.size()) {
+            LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
+
+            for (InetSocketAddress addr : addrs) {
+                if (reachableInetAddrs.contains(addr.getAddress()))
+                    addrs0.add(addr);
+            }
+            for (InetSocketAddress addr : addrs) {
+                if (!reachableInetAddrs.contains(addr.getAddress()))
+                    addrs0.add(addr);
+            }
+
+            addrs = addrs0;
+        }
+
+        if (log.isDebugEnabled())
+            log.debug("Addresses to connect for node [rmtNode=" + node.id() + ", addrs=" + addrs.toString() + ']');
+
         boolean conn = false;
         GridCommunicationClient client = null;
         IgniteCheckedException errs = null;

From bcb139822afa148a7ea3fbb3eecc274f308070f6 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 10 Mar 2017 15:51:38 +0700
Subject: [PATCH 178/446] IGNITE-4717 VisorClearTask minor fix. (cherry picked
 from commit d4b87f4)

---
 .../visor/cache/VisorCacheClearTask.java      | 57 ++++++++++++++++++-
 1 file changed, 56 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
index 0c8476fa91b10..ce74f1760fa27 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.resources.JobContextResource;
@@ -145,4 +146,58 @@ private boolean callAsync(IgniteFuture<Integer> fut, int idx) {
             return S.toString(VisorCacheClearJob.class, this);
         }
     }
-}
\ No newline at end of file
+
+    /**
+     * Callable to get cache size.
+     *
+     * @deprecated This class needed only for compatibility.
+     */
+    @GridInternal @Deprecated
+    private static class VisorCacheSizeCallable implements IgniteCallable<Integer> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteCache cache;
+
+        /**
+         * @param cache Cache to take size from.
+         */
+        private VisorCacheSizeCallable(IgniteCache cache) {
+            this.cache = cache;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer call() throws Exception {
+            return cache.size(CachePeekMode.PRIMARY);
+        }
+    }
+
+    /**
+     * Callable to clear cache.
+     *
+     * @deprecated This class needed only for compatibility.
+     */
+    @GridInternal @Deprecated
+    private static class VisorCacheClearCallable implements IgniteCallable<Integer> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteCache cache;
+
+        /**
+         * @param cache Cache to clear.
+         */
+        private VisorCacheClearCallable(IgniteCache cache) {
+            this.cache = cache;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer call() throws Exception {
+            cache.clear();
+
+            return 0;
+        }
+    }
+}

From 590b82d817595b82a0706e332c545e4746fafbc2 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 10 Mar 2017 15:51:38 +0700
Subject: [PATCH 179/446] IGNITE-4717 VisorClearTask minor fix. (cherry picked
 from commit d4b87f4)

---
 .../visor/cache/VisorCacheClearTask.java      | 57 ++++++++++++++++++-
 1 file changed, 56 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
index 0c8476fa91b10..ce74f1760fa27 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.resources.JobContextResource;
@@ -145,4 +146,58 @@ private boolean callAsync(IgniteFuture<Integer> fut, int idx) {
             return S.toString(VisorCacheClearJob.class, this);
         }
     }
-}
\ No newline at end of file
+
+    /**
+     * Callable to get cache size.
+     *
+     * @deprecated This class needed only for compatibility.
+     */
+    @GridInternal @Deprecated
+    private static class VisorCacheSizeCallable implements IgniteCallable<Integer> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteCache cache;
+
+        /**
+         * @param cache Cache to take size from.
+         */
+        private VisorCacheSizeCallable(IgniteCache cache) {
+            this.cache = cache;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer call() throws Exception {
+            return cache.size(CachePeekMode.PRIMARY);
+        }
+    }
+
+    /**
+     * Callable to clear cache.
+     *
+     * @deprecated This class needed only for compatibility.
+     */
+    @GridInternal @Deprecated
+    private static class VisorCacheClearCallable implements IgniteCallable<Integer> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteCache cache;
+
+        /**
+         * @param cache Cache to clear.
+         */
+        private VisorCacheClearCallable(IgniteCache cache) {
+            this.cache = cache;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer call() throws Exception {
+            cache.clear();
+
+            return 0;
+        }
+    }
+}

From 0ed4fdacc7cc8cf41b7726fc4a42db1a43241285 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 14 Mar 2017 15:50:03 +0300
Subject: [PATCH 180/446] IGNITE-4761: Fix ServiceProcessor hanging on node
 stop. This closes #1602.

---
 .../service/GridServiceProcessor.java         | 15 ++--
 .../GridServiceProcessorStopSelfTest.java     | 75 +++++++++++++++++++
 2 files changed, 83 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 4eeafed421214..6bcfd65cb59e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -315,6 +315,8 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
 
         busyLock.block();
 
+        U.shutdownNow(GridServiceProcessor.class, depExe, log);
+
         if (!ctx.clientNode())
             ctx.event().removeLocalEventListener(topLsnr);
 
@@ -352,8 +354,6 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
             }
         }
 
-        U.shutdownNow(GridServiceProcessor.class, depExe, log);
-
         Exception err = new IgniteCheckedException("Operation has been cancelled (node is stopping).");
 
         cancelFutures(depFuts, err);
@@ -1399,7 +1399,7 @@ private class ServiceEntriesListener implements CacheEntryUpdatedListener<Object
                 return;
 
             try {
-                depExe.submit(new BusyRunnable() {
+                depExe.submit(new DepRunnable() {
                     @Override public void run0() {
                         onSystemCacheUpdated(deps);
                     }
@@ -1586,7 +1586,7 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                 else
                     topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
 
-                depExe.submit(new BusyRunnable() {
+                depExe.submit(new DepRunnable() {
                     @Override public void run0() {
                         ClusterNode oldest = ctx.discovery().oldestAliveCacheServerNode(topVer);
 
@@ -1794,12 +1794,15 @@ private void undeploy(String name) {
     /**
      *
      */
-    private abstract class BusyRunnable implements Runnable {
+    private abstract class DepRunnable implements Runnable {
         /** {@inheritDoc} */
         @Override public void run() {
             if (!busyLock.enterBusy())
                 return;
 
+            // Won't block ServiceProcessor stopping process.
+            busyLock.leaveBusy();
+
             svcName.set(null);
 
             try {
@@ -1812,8 +1815,6 @@ private abstract class BusyRunnable implements Runnable {
                     throw t;
             }
             finally {
-                busyLock.leaveBusy();
-
                 svcName.set(null);
             }
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
index 92b18abed143c..ea0ba51b60954 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
@@ -20,10 +20,16 @@
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteServices;
 import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.services.Service;
@@ -90,6 +96,75 @@ public void testStopDuringDeployment() throws Exception {
         fut.get();
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStopDuringHangedDeployment() throws Exception {
+        final CountDownLatch depLatch = new CountDownLatch(1);
+
+        final CountDownLatch finishLatch = new CountDownLatch(1);
+
+        final IgniteEx node0 = startGrid(0);
+        final IgniteEx node1 = startGrid(1);
+        final IgniteEx node2 = startGrid(2);
+
+        final IgniteCache<Object, Object> cache = node2.getOrCreateCache(new CacheConfiguration<Object, Object>("def")
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL));
+
+        node0.services().deployNodeSingleton("myService", new TestServiceImpl());
+
+        // Guarantee lock owner will never left topology unexpectedly.
+        final Integer lockKey = keyForNode(node2.affinity("def"), new AtomicInteger(1),
+            node2.cluster().localNode());
+
+        // Lock to hold topology version undone.
+        final Lock lock = cache.lock(lockKey);
+
+        // Try to change topology once service has deployed.
+        IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                depLatch.await();
+
+                node1.close();
+
+                return null;
+            }
+        }, "top-change-thread");
+
+        // Stop node on unstable topology.
+        GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                depLatch.await();
+
+                Thread.sleep(1000);
+
+                node0.close();
+
+                finishLatch.countDown();
+
+                return null;
+            }
+        }, "stopping-node-thread");
+
+        assertNotNull(node0.services().service("myService"));
+
+        // Freeze topology changing
+        lock.lock();
+
+        depLatch.countDown();
+
+        boolean wait = finishLatch.await(15, TimeUnit.SECONDS);
+
+        if (!wait)
+            U.dumpThreads(log);
+
+        assertTrue("Deploy future isn't completed", wait);
+
+        fut.get();
+
+        Ignition.stopAll(true);
+    }
+
     /**
      * Simple map service.
      */

From 6633da64d57eadacf30bf437a9c3a3d205903dcd Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Tue, 14 Mar 2017 16:40:58 +0300
Subject: [PATCH 181/446] IGNITE-3386 Reverted: "Reentrant lock is lost when
 lock owner leaves topology"

---
 .../main/java/org/apache/ignite/Ignite.java   |  1 -
 .../DataStructuresProcessor.java              |  3 +-
 .../datastructures/GridCacheLockImpl.java     | 17 +-----
 .../internal/GridCacheRecreateLockTest.java   | 58 -------------------
 .../IgniteComputeGridTestSuite.java           |  2 -
 5 files changed, 3 insertions(+), 78 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java
index 7f8974c20fd9e..0de08d56584a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignite.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java
@@ -527,7 +527,6 @@ public IgniteSemaphore semaphore(String name, int cnt, boolean failoverSafe, boo
      *      all threads on other nodes waiting to acquire lock are interrupted.
      * @param fair If {@code True}, fair lock will be created.
      * @param create Boolean flag indicating whether data structure should be created if does not exist.
-     *      Will re-create lock if the node that stored the lock left topology and there are no backups left.
      * @return ReentrantLock for the given name.
      * @throws IgniteException If reentrant lock could not be fetched or created.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
index 698efd9d7665d..1cad22f33e5ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
@@ -1396,8 +1396,7 @@ public IgniteLock reentrantLock(final String name, final boolean failoverSafe, f
                         name,
                         key,
                         reentrantLockView,
-                        dsCacheCtx,
-                        create);
+                        dsCacheCtx);
 
                     dsMap.put(key, reentrantLock0);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
index 1cf78faa5bd4e..3ab7289237a63 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
@@ -105,9 +105,6 @@ public final class GridCacheLockImpl implements GridCacheLockEx, Externalizable
     /** Flag indicating that every operation on this lock should be interrupted. */
     private volatile boolean interruptAll;
 
-    /** Re-create flag. */
-    private volatile boolean reCreate;
-
     /**
      * Empty constructor required by {@link Externalizable}.
      */
@@ -525,14 +522,7 @@ protected boolean compareAndSetGlobalState(final int expVal, final int newVal,
                                 GridCacheLockState val = lockView.get(key);
 
                                 if (val == null)
-                                    if (reCreate) {
-                                        val = new GridCacheLockState(0, ctx.nodeId(), 0, failoverSafe, fair);
-
-                                        lockView.put(key, val);
-                                    }
-                                    else
-                                        throw new IgniteCheckedException("Failed to find reentrant lock with " +
-                                            "the given name: " + name);
+                                    throw new IgniteCheckedException("Failed to find reentrant lock with given name: " + name);
 
                                 final long newThreadID = newThread.getId();
 
@@ -1058,14 +1048,12 @@ protected IgniteConditionObject(String name, ConditionObject object) {
      * @param key Reentrant lock key.
      * @param lockView Reentrant lock projection.
      * @param ctx Cache context.
-     * @param reCreate If {@code true} reentrant lock will be re-created in case it is not in cache.
      */
     @SuppressWarnings("unchecked")
     public GridCacheLockImpl(String name,
         GridCacheInternalKey key,
         IgniteInternalCache<GridCacheInternalKey, GridCacheLockState> lockView,
-        GridCacheContext ctx,
-        boolean reCreate) {
+        GridCacheContext ctx) {
         assert name != null;
         assert key != null;
         assert ctx != null;
@@ -1075,7 +1063,6 @@ public GridCacheLockImpl(String name,
         this.key = key;
         this.lockView = lockView;
         this.ctx = ctx;
-        this.reCreate = reCreate;
 
         log = ctx.logger(getClass());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
deleted file mode 100644
index ae850f7789745..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.apache.ignite.internal;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLock;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.testframework.junits.common.GridCommonTest;
-
-/**
- * Create lock after owner node left topology test
- */
-@GridCommonTest(group = "Kernal Self")
-public class GridCacheRecreateLockTest extends GridCommonAbstractTest {
-    /**
-     * @throws IgniteCheckedException If failed.
-     */
-    public void testLockOwnerLeavesGrid() throws Exception {
-        final Ignite ignite = startNodeAndLock("node1");
-
-        new Thread(new Runnable() {
-            @Override public void run() {
-                try {
-                    Thread.sleep(2000);
-                }
-                catch (InterruptedException e) {
-                    e.printStackTrace();
-                }
-
-                ignite.close();
-            }
-        }).start();
-
-        startNodeAndLock("node2");
-    }
-
-    /**
-     * @param name Grid name.
-     * @return Started Ignite instance.
-     */
-    private Ignite startNodeAndLock(String name) throws Exception {
-        IgniteConfiguration cfg = new IgniteConfiguration();
-        cfg.setGridName(name);
-
-        Ignite ignite = Ignition.start(cfg);
-
-        IgniteLock lock = ignite.reentrantLock("lock", true, true, true);
-
-        System.out.println("acquiring lock");
-
-        lock.lock();
-
-        System.out.println("acquired lock");
-
-        return ignite;
-    }
-}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 7c8b6a94c8f0d..8a501fdb801d6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -22,7 +22,6 @@
 import org.apache.ignite.internal.GridAffinityNoCacheSelfTest;
 import org.apache.ignite.internal.GridAffinitySelfTest;
 import org.apache.ignite.internal.GridAlwaysFailoverSpiFailSelfTest;
-import org.apache.ignite.internal.GridCacheRecreateLockTest;
 import org.apache.ignite.internal.GridCancelOnGridStopSelfTest;
 import org.apache.ignite.internal.GridCancelUnusedJobSelfTest;
 import org.apache.ignite.internal.GridCancelledJobsMetricsSelfTest;
@@ -153,7 +152,6 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TaskNodeRestartTest.class);
         suite.addTestSuite(IgniteRoundRobinErrorAfterClientReconnectTest.class);
         suite.addTestSuite(PublicThreadpoolStarvationTest.class);
-        suite.addTestSuite(GridCacheRecreateLockTest.class);
 
         return suite;
     }

From d124004d8b0396a44c26f4c35c263a15880f508c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 17 Mar 2017 14:57:48 +0300
Subject: [PATCH 182/446] IGNITE-4473 - Client should re-try connection attempt
 in case of concurrent network failure.

---
 .../internal/GridKernalGatewayImpl.java       |   8 +-
 .../apache/ignite/internal/IgniteKernal.java  | 120 +++++-
 .../IgniteNeedReconnectException.java         |  40 ++
 .../discovery/GridDiscoveryManager.java       |  24 ++
 .../GridCachePartitionExchangeManager.java    |  25 +-
 .../dht/GridDhtAssignmentFetchFuture.java     |  14 +-
 .../GridDhtPartitionsExchangeFuture.java      |  48 ++-
 .../service/GridServiceProcessor.java         |  86 ++--
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 201 ++++++++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   5 +
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |   8 +
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   9 +
 .../IgniteClientReconnectCacheTest.java       |   7 +-
 .../internal/IgniteClientRejoinTest.java      | 378 ++++++++++++++++++
 .../tcp/TcpClientDiscoverySpiSelfTest.java    |  48 ++-
 .../IgniteClientReconnectTestSuite.java       |   2 +
 16 files changed, 929 insertions(+), 94 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
index fe8c580ca63ea..036954a3a280d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
@@ -44,7 +44,7 @@ public class GridKernalGatewayImpl implements GridKernalGateway, Serializable {
 
     /** */
     @GridToStringExclude
-    private IgniteFutureImpl<?> reconnectFut;
+    private volatile IgniteFutureImpl<?> reconnectFut;
 
     /** */
     private final AtomicReference<GridKernalState> state = new AtomicReference<>(GridKernalState.STOPPED);
@@ -149,6 +149,12 @@ public GridKernalGatewayImpl(String gridName) {
 
     /** {@inheritDoc} */
     @Override public GridFutureAdapter<?> onDisconnected() {
+        if (state.get() == GridKernalState.DISCONNECTED) {
+            assert reconnectFut != null;
+
+            return (GridFutureAdapter<?>)reconnectFut.internalFuture();
+        }
+
         GridFutureAdapter<?> fut = new GridFutureAdapter<>();
 
         reconnectFut = new IgniteFutureImpl<>(fut);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 8fda72fc18419..25f7884c889b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -250,6 +250,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     /** Periodic starvation check interval. */
     private static final long PERIODIC_STARVATION_CHECK_FREQ = 1000 * 30;
 
+    /** Force complete reconnect future. */
+    private static final Object STOP_RECONNECT = new Object();
+
     /** */
     @GridToStringExclude
     private GridKernalContextImpl ctx;
@@ -327,6 +330,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     @GridToStringExclude
     private final AtomicBoolean stopGuard = new AtomicBoolean();
 
+    /** */
+    private final ReconnectState reconnectState = new ReconnectState();
+
     /**
      * No-arg constructor is required by externalization.
      */
@@ -930,6 +936,8 @@ public void start(final IgniteConfiguration cfg,
             // Notify IO manager the second so further components can send and receive messages.
             ctx.io().onKernalStart();
 
+            boolean recon = false;
+
             // Callbacks.
             for (GridComponent comp : ctx) {
                 // Skip discovery manager.
@@ -940,10 +948,24 @@ public void start(final IgniteConfiguration cfg,
                 if (comp instanceof GridIoManager)
                     continue;
 
-                if (!skipDaemon(comp))
-                    comp.onKernalStart();
+                if (!skipDaemon(comp)) {
+                    try {
+                        comp.onKernalStart();
+                    }
+                    catch (IgniteNeedReconnectException e) {
+                        assert ctx.discovery().reconnectSupported();
+
+                        if (log.isDebugEnabled())
+                            log.debug("Failed to start node components on node start, will wait for reconnect: " + e);
+
+                        recon = true;
+                    }
+                }
             }
 
+            if (recon)
+                reconnectState.waitFirstReconnect();
+
             // Register MBeans.
             registerKernalMBean();
             registerLocalNodeMBean();
@@ -3274,6 +3296,8 @@ private void unguard() {
     public void onDisconnected() {
         Throwable err = null;
 
+        reconnectState.waitPreviousReconnect();
+
         GridFutureAdapter<?> reconnectFut = ctx.gateway().onDisconnected();
 
         if (reconnectFut == null) {
@@ -3282,9 +3306,18 @@ public void onDisconnected() {
             return;
         }
 
-        IgniteFuture<?> userFut = new IgniteFutureImpl<>(reconnectFut);
+        IgniteFutureImpl<?> curFut = (IgniteFutureImpl<?>)ctx.cluster().get().clientReconnectFuture();
+
+        IgniteFuture<?> userFut;
 
-        ctx.cluster().get().clientReconnectFuture(userFut);
+        // In case of previous reconnect did not finish keep reconnect future.
+        if (curFut != null && curFut.internalFuture() == reconnectFut)
+            userFut = curFut;
+        else {
+            userFut = new IgniteFutureImpl<>(reconnectFut);
+
+            ctx.cluster().get().clientReconnectFuture(userFut);
+        }
 
         ctx.disconnected(true);
 
@@ -3337,30 +3370,53 @@ public void onReconnected(final boolean clusterRestarted) {
         try {
             ctx.disconnected(false);
 
-            GridCompoundFuture<?, ?> reconnectFut = new GridCompoundFuture<>();
+            GridCompoundFuture curReconnectFut = reconnectState.curReconnectFut = new GridCompoundFuture<>();
+
+            reconnectState.reconnectDone = new GridFutureAdapter<>();
 
             for (GridComponent comp : ctx.components()) {
                 IgniteInternalFuture<?> fut = comp.onReconnected(clusterRestarted);
 
                 if (fut != null)
-                    reconnectFut.add((IgniteInternalFuture)fut);
+                    curReconnectFut.add(fut);
             }
 
-            reconnectFut.add((IgniteInternalFuture)ctx.cache().context().exchange().reconnectExchangeFuture());
+            curReconnectFut.add(ctx.cache().context().exchange().reconnectExchangeFuture());
+
+            curReconnectFut.markInitialized();
 
-            reconnectFut.markInitialized();
+            final GridFutureAdapter reconnectDone = reconnectState.reconnectDone;
 
-            reconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
+            curReconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
                 @Override public void apply(IgniteInternalFuture<?> fut) {
                     try {
-                        fut.get();
+                        Object res = fut.get();
+
+                        if (res == STOP_RECONNECT)
+                            return;
 
                         ctx.gateway().onReconnected();
+
+                        reconnectState.firstReconnectFut.onDone();
                     }
                     catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to reconnect, will stop node", e);
+                        if (!X.hasCause(e, IgniteNeedReconnectException.class,
+                            IgniteClientDisconnectedCheckedException.class)) {
+                            U.error(log, "Failed to reconnect, will stop node.", e);
+
+                            reconnectState.firstReconnectFut.onDone(e);
 
-                        close();
+                            close();
+                        }
+                        else {
+                            assert ctx.discovery().reconnectSupported();
+
+                            U.error(log, "Failed to finish reconnect, will retry [locNodeId=" + ctx.localNodeId() +
+                                ", err=" + e.getMessage() + ']');
+                        }
+                    }
+                    finally {
+                        reconnectDone.onDone();
                     }
                 }
             });
@@ -3519,6 +3575,46 @@ public void dumpDebugInfo() {
         }
     }
 
+    /**
+     *
+     */
+    private class ReconnectState {
+        /** */
+        private final GridFutureAdapter firstReconnectFut = new GridFutureAdapter();
+
+        /** */
+        private GridCompoundFuture<?, Object> curReconnectFut;
+
+        /** */
+        private GridFutureAdapter<?> reconnectDone;
+
+        /**
+         * @throws IgniteCheckedException If failed.
+         */
+        void waitFirstReconnect() throws IgniteCheckedException {
+            firstReconnectFut.get();
+        }
+
+        /**
+         *
+         */
+        void waitPreviousReconnect() {
+            if (curReconnectFut != null && !curReconnectFut.isDone()) {
+                assert reconnectDone != null;
+
+                curReconnectFut.onDone(STOP_RECONNECT);
+
+                try {
+                    reconnectDone.get();
+                }
+                catch (IgniteCheckedException ignote) {
+                    // No-op.
+                }
+            }
+
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IgniteKernal.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
new file mode 100644
index 0000000000000..61ab5762b95ac
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Indicates that node should try reconnect to cluster.
+ */
+public class IgniteNeedReconnectException extends IgniteCheckedException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * @param locNode Local node.
+     * @param cause Cause.
+     */
+    public IgniteNeedReconnectException(ClusterNode locNode, @Nullable Throwable cause) {
+        super("Local node need try to reconnect [locNodeId=" + locNode.id() + ']', cause);
+
+        assert locNode.isClient();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 9aa4db1e0042f..2ec10705bedbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -112,6 +112,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -1890,6 +1891,29 @@ public void failNode(UUID nodeId, @Nullable String warning) {
         }
     }
 
+    /**
+     * @return {@code True} if local node client and discovery SPI supports reconnect.
+     */
+    public boolean reconnectSupported() {
+        DiscoverySpi spi = getSpi();
+
+        return ctx.clientNode() && (spi instanceof TcpDiscoverySpi) &&
+            !(((TcpDiscoverySpi) spi).isClientReconnectDisabled());
+    }
+
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() {
+        assert reconnectSupported();
+
+        DiscoverySpi discoverySpi = getSpi();
+
+        ((TcpDiscoverySpi)discoverySpi).reconnect();
+    }
+
     /**
      * Updates topology version if current version is smaller than updated.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7f11dc45aacb7..92142c0731240 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -43,6 +43,7 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
@@ -447,6 +448,15 @@ else if (m instanceof GridDhtPartitionDemandMessage)
                     else
                         U.warn(log, "Still waiting for initial partition map exchange [fut=" + fut + ']');
                 }
+                catch (IgniteNeedReconnectException e) {
+                    throw e;
+                }
+                catch (Exception e) {
+                    if (fut.reconnectOnError(e))
+                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                    throw e;
+                }
             }
 
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
@@ -1690,6 +1700,12 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                         dumpedObjects++;
                                     }
                                 }
+                                catch (Exception e) {
+                                    if (exchFut.reconnectOnError(e))
+                                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                                    throw e;
+                                }
                             }
 
 
@@ -1829,7 +1845,14 @@ else if (r != null) {
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
                 }
-                catch (IgniteClientDisconnectedCheckedException e) {
+                catch (IgniteClientDisconnectedCheckedException | IgniteNeedReconnectException e) {
+                    assert cctx.discovery().reconnectSupported();
+
+                    U.warn(log,"Local node failed to complete partition map exchange due to " +
+                        "network issues, will try to reconnect to cluster", e);
+
+                    cctx.discovery().reconnect();
+
                     return;
                 }
                 catch (IgniteCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index ab8e863e90e7d..6425bc141245d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -17,15 +17,16 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.Queue;
 import java.util.UUID;
-import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
@@ -34,6 +35,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -202,8 +204,14 @@ private void requestFromNextNode() {
                         "continue to another node): " + node);
                 }
                 catch (IgniteCheckedException e) {
-                    U.error(log0, "Failed to request affinity assignment from remote node (will " +
-                        "continue to another node): " + node, e);
+                    if (ctx.discovery().reconnectSupported() && X.hasCause(e, IOException.class)) {
+                        onDone(new IgniteNeedReconnectException(ctx.localNode(), e));
+
+                        return;
+                    }
+
+                    U.warn(log0, "Failed to request affinity assignment from remote node (will " +
+                        "continue to another node): " + node);
                 }
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index e945de958a339..d4f95e5b55d05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -32,6 +33,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
@@ -39,6 +41,7 @@
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -54,7 +57,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
@@ -65,7 +67,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -506,10 +508,17 @@ public void init() throws IgniteInterruptedCheckedException {
 
             throw e;
         }
+        catch (IgniteNeedReconnectException e) {
+            onDone(e);
+        }
         catch (Throwable e) {
-            U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else {
+                U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
 
-            onDone(e);
+                onDone(e);
+            }
 
             if (e instanceof Error)
                 throw (Error)e;
@@ -1297,7 +1306,10 @@ private void onAllReceived(boolean discoThread) {
             }
         }
         catch (IgniteCheckedException e) {
-            onDone(e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else
+                onDone(e);
         }
     }
 
@@ -1314,8 +1326,15 @@ private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
         }
         catch (IgniteCheckedException e) {
             if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) {
-                log.debug("Failed to send full partition map to node, node left grid " +
-                    "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+                if (log.isDebugEnabled())
+                    log.debug("Failed to send full partition map to node, node left grid " +
+                        "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+
+                return;
+            }
+
+            if (reconnectOnError(e)) {
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
 
                 return;
             }
@@ -1641,6 +1660,12 @@ public void onNodeLeft(final ClusterNode node) {
                             }
                         }
                     }
+                    catch (Exception e) {
+                        if (reconnectOnError(e))
+                            onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+                        else
+                            throw e;
+                    }
                     finally {
                         leaveBusy();
                     }
@@ -1652,6 +1677,15 @@ public void onNodeLeft(final ClusterNode node) {
         }
     }
 
+    /**
+     * @param e Exception.
+     * @return {@code True} if local node should try reconnect in case of error.
+     */
+    public boolean reconnectOnError(Throwable e) {
+        return X.hasCause(e, IOException.class, IgniteClientDisconnectedCheckedException.class) &&
+            cctx.discovery().reconnectSupported();
+    }
+
     /** {@inheritDoc} */
     @Override public int compareTo(GridDhtPartitionsExchangeFuture fut) {
         return exchId.compareTo(fut.exchId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6bcfd65cb59e0..bd815189fb5ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1498,60 +1498,60 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
         }
     }
 
-        /**
-         * Deployment callback.
-         *
-         * @param dep Service deployment.
-         * @param topVer Topology version.
-         */
-        private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
-            // Retry forever.
-            try {
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+    /**
+     * Deployment callback.
+     *
+     * @param dep Service deployment.
+     * @param topVer Topology version.
+     */
+    private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
+        // Retry forever.
+        try {
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                // If topology version changed, reassignment will happen from topology event.
-                if (newTopVer.equals(topVer))
-                    reassign(dep, topVer);
-            }
-            catch (IgniteCheckedException e) {
-                if (!(e instanceof ClusterTopologyCheckedException))
-                    log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
+            // If topology version changed, reassignment will happen from topology event.
+            if (newTopVer.equals(topVer))
+                reassign(dep, topVer);
+        }
+        catch (IgniteCheckedException e) {
+            if (!(e instanceof ClusterTopologyCheckedException))
+                log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
 
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                if (!newTopVer.equals(topVer)) {
-                    assert newTopVer.compareTo(topVer) > 0;
+            if (!newTopVer.equals(topVer)) {
+                assert newTopVer.compareTo(topVer) > 0;
 
-                    // Reassignment will happen from topology event.
-                    return;
-                }
+                // Reassignment will happen from topology event.
+                return;
+            }
 
-                ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
-                    private IgniteUuid id = IgniteUuid.randomUuid();
+            ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
+                private IgniteUuid id = IgniteUuid.randomUuid();
 
-                    private long start = System.currentTimeMillis();
+                private long start = System.currentTimeMillis();
 
-                    @Override public IgniteUuid timeoutId() {
-                        return id;
-                    }
+                @Override public IgniteUuid timeoutId() {
+                    return id;
+                }
 
-                    @Override public long endTime() {
-                        return start + RETRY_TIMEOUT;
-                    }
+                @Override public long endTime() {
+                    return start + RETRY_TIMEOUT;
+                }
 
-                    @Override public void onTimeout() {
-                        if (!busyLock.enterBusy())
-                            return;
+                @Override public void onTimeout() {
+                    if (!busyLock.enterBusy())
+                        return;
 
-                        try {
-                            // Try again.
-                            onDeployment(dep, topVer);
-                        }
-                        finally {
-                            busyLock.leaveBusy();
-                        }
+                    try {
+                        // Try again.
+                        onDeployment(dep, topVer);
                     }
-                });
+                    finally {
+                        busyLock.leaveBusy();
+                    }
+                }
+            });
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 95e2cda4f6907..02ba56a884b33 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -129,6 +129,9 @@ class ClientImpl extends TcpDiscoveryImpl {
     /** */
     private static final Object SPI_RECONNECT_FAILED = "SPI_RECONNECT_FAILED";
 
+    /** */
+    private static final Object SPI_RECONNECT = "SPI_RECONNECT";
+
     /** Remote nodes. */
     private final ConcurrentMap<UUID, TcpDiscoveryNode> rmtNodes = new ConcurrentHashMap8<>();
 
@@ -808,6 +811,11 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
             log);
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        msgWorker.addMessage(SPI_RECONNECT);
+    }
+
     /** {@inheritDoc} */
     @Override public void brakeConnection() {
         SocketStream sockStream = msgWorker.currSock;
@@ -879,9 +887,12 @@ private class SocketReader extends IgniteSpiThread {
         /** */
         private UUID rmtNodeId;
 
+        /** */
+        private CountDownLatch stopReadLatch;
+
         /**
          */
-        protected SocketReader() {
+        SocketReader() {
             super(spi.ignite().name(), "tcp-client-disco-sock-reader", log);
         }
 
@@ -889,7 +900,7 @@ protected SocketReader() {
          * @param sockStream Socket.
          * @param rmtNodeId Rmt node id.
          */
-        public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
+        void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             synchronized (mux) {
                 this.sockStream = sockStream;
 
@@ -899,6 +910,31 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             }
         }
 
+        /**
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceStopRead() throws InterruptedException {
+            CountDownLatch stopReadLatch;
+
+            synchronized (mux) {
+                SocketStream stream = sockStream;
+
+                if (stream == null)
+                    return;
+
+                this.stopReadLatch = stopReadLatch = new CountDownLatch(1);
+
+                U.closeQuiet(stream.socket());
+
+                this.sockStream = null;
+                this.rmtNodeId = null;
+
+                mux.notifyAll();
+            }
+
+            stopReadLatch.await();
+        }
+
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException {
             while (!isInterrupted()) {
@@ -906,6 +942,12 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
                 UUID rmtNodeId;
 
                 synchronized (mux) {
+                    if (stopReadLatch != null) {
+                        stopReadLatch.countDown();
+
+                        stopReadLatch = null;
+                    }
+
                     if (this.sockStream == null) {
                         mux.wait();
 
@@ -1007,18 +1049,21 @@ private class SocketWriter extends IgniteSpiThread {
         private final Queue<TcpDiscoveryAbstractMessage> queue = new ArrayDeque<>();
 
         /** */
-        private final long socketTimeout;
+        private final long sockTimeout;
 
         /** */
         private TcpDiscoveryAbstractMessage unackedMsg;
 
+        /** */
+        private CountDownLatch forceLeaveLatch;
+
         /**
          *
          */
-        protected SocketWriter() {
+        SocketWriter() {
             super(spi.ignite().name(), "tcp-client-disco-sock-writer", log);
 
-            socketTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
+            sockTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
                 spi.getSocketTimeout();
         }
 
@@ -1033,6 +1078,29 @@ private void sendMessage(TcpDiscoveryAbstractMessage msg) {
             }
         }
 
+        /**
+         * Sends {@link TcpDiscoveryNodeLeftMessage} and closes socket.
+         *
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceLeave() throws InterruptedException {
+            CountDownLatch forceLeaveLatch;
+
+            synchronized (mux) {
+                // If writer was stopped.
+                if (sock == null)
+                    return;
+
+                this.forceLeaveLatch = forceLeaveLatch = new CountDownLatch(1);
+
+                unackedMsg = null;
+
+                mux.notifyAll();
+            }
+
+            forceLeaveLatch.await();
+        }
+
         /**
          * @param sock Socket.
          * @param clientAck {@code True} is server supports client message acknowlede.
@@ -1089,13 +1157,41 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                         continue;
                     }
 
-                    msg = queue.poll();
+                    if (forceLeaveLatch != null) {
+                        msg = new TcpDiscoveryNodeLeftMessage(getLocalNodeId());
 
-                    if (msg == null) {
-                        mux.wait();
+                        msg.client(true);
+
+                        try {
+                            spi.writeToSocket(
+                                sock,
+                                msg,
+                                sockTimeout);
+                        }
+                        catch (IOException | IgniteCheckedException e) {
+                            if (log.isDebugEnabled()) {
+                                log.debug("Failed to send TcpDiscoveryNodeLeftMessage on force leave [msg=" + msg +
+                                    ", err=" + e.getMessage() + ']');
+                            }
+                        }
+
+                        U.closeQuiet(sock);
+
+                        this.sock = null;
+
+                        clear();
 
                         continue;
                     }
+                    else {
+                        msg = queue.poll();
+
+                        if (msg == null) {
+                            mux.wait();
+
+                            continue;
+                        }
+                    }
                 }
 
                 for (IgniteInClosure<TcpDiscoveryAbstractMessage> msgLsnr : spi.sndMsgLsnrs)
@@ -1115,7 +1211,7 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     spi.writeToSocket(
                         sock,
                         msg,
-                        socketTimeout);
+                        sockTimeout);
 
                     msg = null;
 
@@ -1165,10 +1261,30 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     synchronized (mux) {
                         if (sock == this.sock)
                             this.sock = null; // Connection has dead.
+
+                        clear();
                     }
                 }
             }
         }
+
+        /**
+         *
+         */
+        private void clear() {
+            assert Thread.holdsLock(mux);
+
+            queue.clear();
+            unackedMsg = null;
+
+            CountDownLatch forceLeaveLatch = this.forceLeaveLatch;
+
+            if (forceLeaveLatch != null) {
+                this.forceLeaveLatch = null;
+
+                forceLeaveLatch.countDown();
+            }
+        }
     }
 
     /**
@@ -1413,6 +1529,38 @@ else if (msg == SPI_STOP) {
                         else
                             leaveLatch.countDown();
                     }
+                    else if (msg == SPI_RECONNECT) {
+                        if (state == CONNECTED) {
+                            if (reconnector != null) {
+                                reconnector.cancel();
+                                reconnector.join();
+
+                                reconnector = null;
+                            }
+
+                            sockWriter.forceLeave();
+                            sockReader.forceStopRead();
+
+                            currSock = null;
+
+                            queue.clear();
+
+                            onDisconnected();
+
+                            notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
+
+                            UUID newId = UUID.randomUUID();
+
+                            U.quietAndWarn(log, "Local node will try to reconnect to cluster with new id due " +
+                                "to network problems [newId=" + newId +
+                                ", prevId=" + locNode.id() +
+                                ", locNode=" + locNode+ ']');
+
+                            locNode.onClientDisconnected(newId);
+
+                            tryJoin();
+                        }
+                    }
                     else if (msg instanceof TcpDiscoveryNodeFailedMessage &&
                         ((TcpDiscoveryNodeFailedMessage)msg).failedNodeId().equals(locNode.id())) {
                         TcpDiscoveryNodeFailedMessage msg0 = (TcpDiscoveryNodeFailedMessage)msg;
@@ -1495,20 +1643,7 @@ else if (msg == SPI_RECONNECT_FAILED) {
                                         ", failMsg=" + forceFailMsg + ']');
                                 }
 
-                                state = DISCONNECTED;
-
-                                nodeAdded = false;
-
-                                IgniteClientDisconnectedCheckedException err =
-                                    new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
-                                    "client node disconnected.");
-
-                                for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
-                                    GridFutureAdapter<Boolean> fut = e.getValue();
-
-                                    if (pingFuts.remove(e.getKey(), fut))
-                                        fut.onDone(err);
-                                }
+                                onDisconnected();
 
                                 notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
                             }
@@ -1603,6 +1738,26 @@ else if (discoMsg instanceof TcpDiscoveryCheckFailedMessage)
             }
         }
 
+        /**
+         *
+         */
+        private void onDisconnected() {
+            state = DISCONNECTED;
+
+            nodeAdded = false;
+
+            IgniteClientDisconnectedCheckedException err =
+                new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
+                    "client node disconnected.");
+
+            for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
+                GridFutureAdapter<Boolean> fut = e.getValue();
+
+                if (pingFuts.remove(e.getKey(), fut))
+                    fut.onDone(err);
+            }
+        }
+
         /**
          * @throws InterruptedException If interrupted.
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 4600be094c840..afd1c2ba7fc66 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1609,6 +1609,11 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
         throw new UnsupportedOperationException();
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        throw new UnsupportedOperationException("Reconnect is not supported for server.");
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteSpiThread workerThread() {
         return msgWorker;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index f199c20e17edb..84c2ff28afef0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -258,6 +259,13 @@ protected static String threadStatus(Thread t) {
         return t.isAlive() ? "alive" : "dead";
     }
 
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public abstract void reconnect() throws IgniteSpiException;
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      * <p>
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 00ae97d1e302c..a2a47feb31486 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1926,6 +1926,15 @@ boolean isSslEnabled() {
         return ignite().configuration().getSslContextFactory() != null;
     }
 
+    /**
+     * Force reconnect to cluster.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() throws IgniteSpiException {
+        impl.reconnect();
+    }
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 0f0165b0f04f9..6cdf465978907 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -700,9 +700,12 @@ public void testReconnectInitialExchangeInProgress() throws Exception {
                 try {
                     Ignition.start(optimize(getConfiguration(getTestGridName(SRV_CNT))));
 
-                    fail();
+                    // Commented due to IGNITE-4473, because
+                    // IgniteClientDisconnectedException won't
+                    // be thrown, but client will reconnect.
+//                    fail();
 
-                    return false;
+                    return true;
                 }
                 catch (IgniteClientDisconnectedException e) {
                     log.info("Expected start error: " + e);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
new file mode 100644
index 0000000000000..a5d42e9a12cba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests client to be able restore connection to cluster if coordination is not available.
+ */
+public class IgniteClientRejoinTest extends GridCommonAbstractTest {
+    /** Block. */
+    private volatile boolean block;
+
+    /** Block all. */
+    private volatile boolean blockAll;
+
+    /** Coordinator. */
+    private volatile ClusterNode crd;
+
+    /** Client reconnect disabled. */
+    private boolean clientReconnectDisabled;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK", "true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.clearProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        clientReconnectDisabled = false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("client")) {
+            cfg.setCommunicationSpi(new TcpCommunicationSpi());
+
+            TcpDiscoverySpi spi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+            DiscoverySpi dspi = new DiscoverySpi();
+
+            dspi.setIpFinder(spi.getIpFinder());
+
+            cfg.setDiscoverySpi(dspi);
+
+            dspi.setJoinTimeout(60_000);
+            dspi.setClientReconnectDisabled(clientReconnectDisabled);
+
+            cfg.setClientMode(true);
+        }
+
+        // TODO: IGNITE-4833
+        cfg.setPeerClassLoadingEnabled(false);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectAfterStart() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        List<Ignite> clientNodes = new ArrayList<>();
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++)
+            clientNodes.add(startGrid("client" + i));
+
+        blockAll = true;
+
+        GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                U.sleep(5_000);
+
+                block = true;
+                blockAll = false;
+
+                System.out.println(">>> Allow with blocked coordinator.");
+
+                latch.countDown();
+
+                return null;
+            }
+        });
+
+        IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                latch.await();
+
+                U.sleep((new Random().nextInt(15) + 30) * 1000);
+
+                block = false;
+
+                System.out.println(">>> Allow coordinator.");
+
+                return null;
+            }
+        });
+
+        fut.get();
+
+        for (Ignite client : clientNodes) {
+            while (true) {
+                try {
+                    IgniteCache<Integer, Integer> cache = client.getOrCreateCache("some");
+
+                    for (int i = 0; i < 100; i++)
+                        cache.put(i, i);
+
+                    for (int i = 0; i < 100; i++)
+                        assertEquals((Integer)i, cache.get(i));
+
+                    cache.clear();
+
+                    break;
+                }
+                catch (IgniteClientDisconnectedException e) {
+                    e.reconnectFuture().get();
+                }
+            }
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnect() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        GridTestUtils.runAsync(new Callable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                latch.countDown();
+
+                Random rnd = new Random();
+
+                U.sleep((rnd.nextInt(15) + 15) * 1000);
+
+                block = false;
+
+                System.out.println(">>> ALLOW connection to coordinator.");
+
+                return true;
+            }
+        });
+
+        for (IgniteInternalFuture<Ignite> clientFut : futs) {
+            Ignite client = clientFut.get();
+
+            IgniteCache<Integer, Integer> cache = client.getOrCreateCache(client.name());
+
+            for (int i = 0; i < 100; i++)
+                cache.put(i, i);
+
+            for (int i = 0; i < 100; i++)
+                assert i == cache.get(i);
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectDisabled() throws Exception {
+        clientReconnectDisabled = true;
+
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        latch.countDown();
+
+        for (final IgniteInternalFuture<Ignite> clientFut : futs) {
+            //noinspection ThrowableNotThrown
+            GridTestUtils.assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    clientFut.get();
+
+                    return null;
+                }
+            }, IgniteCheckedException.class, null);
+        }
+
+        assertEquals(0, srv1.cluster().forClients().nodes().size());
+        assertEquals(0, srv2.cluster().forClients().nodes().size());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 3 * 60_000;
+    }
+
+    /**
+     *
+     */
+    private class TcpCommunicationSpi extends org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     *
+     */
+    private class DiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, byte[] data,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(TcpDiscoveryAbstractMessage msg, Socket sock, int res,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(msg, sock, res, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Socket openSocket(Socket sock, InetSocketAddress remAddr,
+            IgniteSpiOperationTimeoutHelper timeoutHelper) throws IOException, IgniteSpiOperationTimeoutException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            return super.openSocket(sock, remAddr, timeoutHelper);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 331b581bb7bec..0483a1ce8fbc6 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -32,6 +32,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.IgniteState;
@@ -43,6 +44,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -1788,8 +1790,7 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
         clientNodeIds.add(client.cluster().localNode().id());
 
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override
-            public boolean apply() {
+            @Override public boolean apply() {
                 return srv.cluster().nodes().size() == 2;
             }
         }, awaitTime());
@@ -1799,6 +1800,49 @@ public boolean apply() {
         assertFalse(err.get());
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testForceClientReconnect() throws Exception {
+        startServerNodes(1);
+
+        startClientNodes(1);
+
+        Ignite srv = G.ignite("server-0");
+        IgniteKernal client = (IgniteKernal)G.ignite("client-0");
+
+        UUID clientId = F.first(clientNodeIds);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        srv.events().enableLocal(EVT_NODE_JOINED);
+
+        srv.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                latch.countDown();
+
+                return false;
+            }
+        }, EVT_NODE_JOINED);
+
+        client.context().discovery().reconnect();
+
+        assert latch.await(10, TimeUnit.SECONDS);
+
+        while (true) {
+            try {
+                UUID newId = client.localNode().id();
+
+                assert !clientId.equals(newId) : clientId;
+
+                break;
+            }
+            catch (IgniteClientDisconnectedException e) {
+                e.reconnectFuture().get(10_000);
+            }
+        }
+    }
+
     /**
      * @param ignite Ignite.
      * @throws Exception If failed.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index ea8e37bada1d0..67d88e1e29eac 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.IgniteClientReconnectServicesTest;
 import org.apache.ignite.internal.IgniteClientReconnectStopTest;
 import org.apache.ignite.internal.IgniteClientReconnectStreamerTest;
+import org.apache.ignite.internal.IgniteClientRejoinTest;
 
 /**
  *
@@ -52,6 +53,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteClientReconnectServicesTest.class);
         suite.addTestSuite(IgniteClientReconnectStreamerTest.class);
         suite.addTestSuite(IgniteClientReconnectFailoverTest.class);
+        suite.addTestSuite(IgniteClientRejoinTest.class);
 
         return suite;
     }

From a2b4751f5eefd70a5a1aa26652c9671240125f78 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 17 Mar 2017 14:57:48 +0300
Subject: [PATCH 183/446] IGNITE-4473 - Client should re-try connection attempt
 in case of concurrent network failure.

(cherry picked from commit d124004)
---
 .../internal/GridKernalGatewayImpl.java       |   8 +-
 .../apache/ignite/internal/IgniteKernal.java  | 120 +++++-
 .../IgniteNeedReconnectException.java         |  40 ++
 .../discovery/GridDiscoveryManager.java       |  24 ++
 .../GridCachePartitionExchangeManager.java    |  25 +-
 .../dht/GridDhtAssignmentFetchFuture.java     |  14 +-
 .../GridDhtPartitionsExchangeFuture.java      |  48 ++-
 .../service/GridServiceProcessor.java         |  86 ++--
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 201 ++++++++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   5 +
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |   8 +
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   9 +
 .../IgniteClientReconnectCacheTest.java       |   7 +-
 .../internal/IgniteClientRejoinTest.java      | 378 ++++++++++++++++++
 .../tcp/TcpClientDiscoverySpiSelfTest.java    |  48 ++-
 .../IgniteClientReconnectTestSuite.java       |   2 +
 16 files changed, 929 insertions(+), 94 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
index fe8c580ca63ea..036954a3a280d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
@@ -44,7 +44,7 @@ public class GridKernalGatewayImpl implements GridKernalGateway, Serializable {
 
     /** */
     @GridToStringExclude
-    private IgniteFutureImpl<?> reconnectFut;
+    private volatile IgniteFutureImpl<?> reconnectFut;
 
     /** */
     private final AtomicReference<GridKernalState> state = new AtomicReference<>(GridKernalState.STOPPED);
@@ -149,6 +149,12 @@ public GridKernalGatewayImpl(String gridName) {
 
     /** {@inheritDoc} */
     @Override public GridFutureAdapter<?> onDisconnected() {
+        if (state.get() == GridKernalState.DISCONNECTED) {
+            assert reconnectFut != null;
+
+            return (GridFutureAdapter<?>)reconnectFut.internalFuture();
+        }
+
         GridFutureAdapter<?> fut = new GridFutureAdapter<>();
 
         reconnectFut = new IgniteFutureImpl<>(fut);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 8fda72fc18419..25f7884c889b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -250,6 +250,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     /** Periodic starvation check interval. */
     private static final long PERIODIC_STARVATION_CHECK_FREQ = 1000 * 30;
 
+    /** Force complete reconnect future. */
+    private static final Object STOP_RECONNECT = new Object();
+
     /** */
     @GridToStringExclude
     private GridKernalContextImpl ctx;
@@ -327,6 +330,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     @GridToStringExclude
     private final AtomicBoolean stopGuard = new AtomicBoolean();
 
+    /** */
+    private final ReconnectState reconnectState = new ReconnectState();
+
     /**
      * No-arg constructor is required by externalization.
      */
@@ -930,6 +936,8 @@ public void start(final IgniteConfiguration cfg,
             // Notify IO manager the second so further components can send and receive messages.
             ctx.io().onKernalStart();
 
+            boolean recon = false;
+
             // Callbacks.
             for (GridComponent comp : ctx) {
                 // Skip discovery manager.
@@ -940,10 +948,24 @@ public void start(final IgniteConfiguration cfg,
                 if (comp instanceof GridIoManager)
                     continue;
 
-                if (!skipDaemon(comp))
-                    comp.onKernalStart();
+                if (!skipDaemon(comp)) {
+                    try {
+                        comp.onKernalStart();
+                    }
+                    catch (IgniteNeedReconnectException e) {
+                        assert ctx.discovery().reconnectSupported();
+
+                        if (log.isDebugEnabled())
+                            log.debug("Failed to start node components on node start, will wait for reconnect: " + e);
+
+                        recon = true;
+                    }
+                }
             }
 
+            if (recon)
+                reconnectState.waitFirstReconnect();
+
             // Register MBeans.
             registerKernalMBean();
             registerLocalNodeMBean();
@@ -3274,6 +3296,8 @@ private void unguard() {
     public void onDisconnected() {
         Throwable err = null;
 
+        reconnectState.waitPreviousReconnect();
+
         GridFutureAdapter<?> reconnectFut = ctx.gateway().onDisconnected();
 
         if (reconnectFut == null) {
@@ -3282,9 +3306,18 @@ public void onDisconnected() {
             return;
         }
 
-        IgniteFuture<?> userFut = new IgniteFutureImpl<>(reconnectFut);
+        IgniteFutureImpl<?> curFut = (IgniteFutureImpl<?>)ctx.cluster().get().clientReconnectFuture();
+
+        IgniteFuture<?> userFut;
 
-        ctx.cluster().get().clientReconnectFuture(userFut);
+        // In case of previous reconnect did not finish keep reconnect future.
+        if (curFut != null && curFut.internalFuture() == reconnectFut)
+            userFut = curFut;
+        else {
+            userFut = new IgniteFutureImpl<>(reconnectFut);
+
+            ctx.cluster().get().clientReconnectFuture(userFut);
+        }
 
         ctx.disconnected(true);
 
@@ -3337,30 +3370,53 @@ public void onReconnected(final boolean clusterRestarted) {
         try {
             ctx.disconnected(false);
 
-            GridCompoundFuture<?, ?> reconnectFut = new GridCompoundFuture<>();
+            GridCompoundFuture curReconnectFut = reconnectState.curReconnectFut = new GridCompoundFuture<>();
+
+            reconnectState.reconnectDone = new GridFutureAdapter<>();
 
             for (GridComponent comp : ctx.components()) {
                 IgniteInternalFuture<?> fut = comp.onReconnected(clusterRestarted);
 
                 if (fut != null)
-                    reconnectFut.add((IgniteInternalFuture)fut);
+                    curReconnectFut.add(fut);
             }
 
-            reconnectFut.add((IgniteInternalFuture)ctx.cache().context().exchange().reconnectExchangeFuture());
+            curReconnectFut.add(ctx.cache().context().exchange().reconnectExchangeFuture());
+
+            curReconnectFut.markInitialized();
 
-            reconnectFut.markInitialized();
+            final GridFutureAdapter reconnectDone = reconnectState.reconnectDone;
 
-            reconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
+            curReconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
                 @Override public void apply(IgniteInternalFuture<?> fut) {
                     try {
-                        fut.get();
+                        Object res = fut.get();
+
+                        if (res == STOP_RECONNECT)
+                            return;
 
                         ctx.gateway().onReconnected();
+
+                        reconnectState.firstReconnectFut.onDone();
                     }
                     catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to reconnect, will stop node", e);
+                        if (!X.hasCause(e, IgniteNeedReconnectException.class,
+                            IgniteClientDisconnectedCheckedException.class)) {
+                            U.error(log, "Failed to reconnect, will stop node.", e);
+
+                            reconnectState.firstReconnectFut.onDone(e);
 
-                        close();
+                            close();
+                        }
+                        else {
+                            assert ctx.discovery().reconnectSupported();
+
+                            U.error(log, "Failed to finish reconnect, will retry [locNodeId=" + ctx.localNodeId() +
+                                ", err=" + e.getMessage() + ']');
+                        }
+                    }
+                    finally {
+                        reconnectDone.onDone();
                     }
                 }
             });
@@ -3519,6 +3575,46 @@ public void dumpDebugInfo() {
         }
     }
 
+    /**
+     *
+     */
+    private class ReconnectState {
+        /** */
+        private final GridFutureAdapter firstReconnectFut = new GridFutureAdapter();
+
+        /** */
+        private GridCompoundFuture<?, Object> curReconnectFut;
+
+        /** */
+        private GridFutureAdapter<?> reconnectDone;
+
+        /**
+         * @throws IgniteCheckedException If failed.
+         */
+        void waitFirstReconnect() throws IgniteCheckedException {
+            firstReconnectFut.get();
+        }
+
+        /**
+         *
+         */
+        void waitPreviousReconnect() {
+            if (curReconnectFut != null && !curReconnectFut.isDone()) {
+                assert reconnectDone != null;
+
+                curReconnectFut.onDone(STOP_RECONNECT);
+
+                try {
+                    reconnectDone.get();
+                }
+                catch (IgniteCheckedException ignote) {
+                    // No-op.
+                }
+            }
+
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IgniteKernal.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
new file mode 100644
index 0000000000000..61ab5762b95ac
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Indicates that node should try reconnect to cluster.
+ */
+public class IgniteNeedReconnectException extends IgniteCheckedException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * @param locNode Local node.
+     * @param cause Cause.
+     */
+    public IgniteNeedReconnectException(ClusterNode locNode, @Nullable Throwable cause) {
+        super("Local node need try to reconnect [locNodeId=" + locNode.id() + ']', cause);
+
+        assert locNode.isClient();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 9aa4db1e0042f..2ec10705bedbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -112,6 +112,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -1890,6 +1891,29 @@ public void failNode(UUID nodeId, @Nullable String warning) {
         }
     }
 
+    /**
+     * @return {@code True} if local node client and discovery SPI supports reconnect.
+     */
+    public boolean reconnectSupported() {
+        DiscoverySpi spi = getSpi();
+
+        return ctx.clientNode() && (spi instanceof TcpDiscoverySpi) &&
+            !(((TcpDiscoverySpi) spi).isClientReconnectDisabled());
+    }
+
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() {
+        assert reconnectSupported();
+
+        DiscoverySpi discoverySpi = getSpi();
+
+        ((TcpDiscoverySpi)discoverySpi).reconnect();
+    }
+
     /**
      * Updates topology version if current version is smaller than updated.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7f11dc45aacb7..92142c0731240 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -43,6 +43,7 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
@@ -447,6 +448,15 @@ else if (m instanceof GridDhtPartitionDemandMessage)
                     else
                         U.warn(log, "Still waiting for initial partition map exchange [fut=" + fut + ']');
                 }
+                catch (IgniteNeedReconnectException e) {
+                    throw e;
+                }
+                catch (Exception e) {
+                    if (fut.reconnectOnError(e))
+                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                    throw e;
+                }
             }
 
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
@@ -1690,6 +1700,12 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                         dumpedObjects++;
                                     }
                                 }
+                                catch (Exception e) {
+                                    if (exchFut.reconnectOnError(e))
+                                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                                    throw e;
+                                }
                             }
 
 
@@ -1829,7 +1845,14 @@ else if (r != null) {
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
                 }
-                catch (IgniteClientDisconnectedCheckedException e) {
+                catch (IgniteClientDisconnectedCheckedException | IgniteNeedReconnectException e) {
+                    assert cctx.discovery().reconnectSupported();
+
+                    U.warn(log,"Local node failed to complete partition map exchange due to " +
+                        "network issues, will try to reconnect to cluster", e);
+
+                    cctx.discovery().reconnect();
+
                     return;
                 }
                 catch (IgniteCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index ab8e863e90e7d..6425bc141245d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -17,15 +17,16 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.Queue;
 import java.util.UUID;
-import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
@@ -34,6 +35,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -202,8 +204,14 @@ private void requestFromNextNode() {
                         "continue to another node): " + node);
                 }
                 catch (IgniteCheckedException e) {
-                    U.error(log0, "Failed to request affinity assignment from remote node (will " +
-                        "continue to another node): " + node, e);
+                    if (ctx.discovery().reconnectSupported() && X.hasCause(e, IOException.class)) {
+                        onDone(new IgniteNeedReconnectException(ctx.localNode(), e));
+
+                        return;
+                    }
+
+                    U.warn(log0, "Failed to request affinity assignment from remote node (will " +
+                        "continue to another node): " + node);
                 }
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index e945de958a339..d4f95e5b55d05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -32,6 +33,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
@@ -39,6 +41,7 @@
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -54,7 +57,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
@@ -65,7 +67,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -506,10 +508,17 @@ public void init() throws IgniteInterruptedCheckedException {
 
             throw e;
         }
+        catch (IgniteNeedReconnectException e) {
+            onDone(e);
+        }
         catch (Throwable e) {
-            U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else {
+                U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
 
-            onDone(e);
+                onDone(e);
+            }
 
             if (e instanceof Error)
                 throw (Error)e;
@@ -1297,7 +1306,10 @@ private void onAllReceived(boolean discoThread) {
             }
         }
         catch (IgniteCheckedException e) {
-            onDone(e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else
+                onDone(e);
         }
     }
 
@@ -1314,8 +1326,15 @@ private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
         }
         catch (IgniteCheckedException e) {
             if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) {
-                log.debug("Failed to send full partition map to node, node left grid " +
-                    "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+                if (log.isDebugEnabled())
+                    log.debug("Failed to send full partition map to node, node left grid " +
+                        "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+
+                return;
+            }
+
+            if (reconnectOnError(e)) {
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
 
                 return;
             }
@@ -1641,6 +1660,12 @@ public void onNodeLeft(final ClusterNode node) {
                             }
                         }
                     }
+                    catch (Exception e) {
+                        if (reconnectOnError(e))
+                            onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+                        else
+                            throw e;
+                    }
                     finally {
                         leaveBusy();
                     }
@@ -1652,6 +1677,15 @@ public void onNodeLeft(final ClusterNode node) {
         }
     }
 
+    /**
+     * @param e Exception.
+     * @return {@code True} if local node should try reconnect in case of error.
+     */
+    public boolean reconnectOnError(Throwable e) {
+        return X.hasCause(e, IOException.class, IgniteClientDisconnectedCheckedException.class) &&
+            cctx.discovery().reconnectSupported();
+    }
+
     /** {@inheritDoc} */
     @Override public int compareTo(GridDhtPartitionsExchangeFuture fut) {
         return exchId.compareTo(fut.exchId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6bcfd65cb59e0..bd815189fb5ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1498,60 +1498,60 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
         }
     }
 
-        /**
-         * Deployment callback.
-         *
-         * @param dep Service deployment.
-         * @param topVer Topology version.
-         */
-        private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
-            // Retry forever.
-            try {
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+    /**
+     * Deployment callback.
+     *
+     * @param dep Service deployment.
+     * @param topVer Topology version.
+     */
+    private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
+        // Retry forever.
+        try {
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                // If topology version changed, reassignment will happen from topology event.
-                if (newTopVer.equals(topVer))
-                    reassign(dep, topVer);
-            }
-            catch (IgniteCheckedException e) {
-                if (!(e instanceof ClusterTopologyCheckedException))
-                    log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
+            // If topology version changed, reassignment will happen from topology event.
+            if (newTopVer.equals(topVer))
+                reassign(dep, topVer);
+        }
+        catch (IgniteCheckedException e) {
+            if (!(e instanceof ClusterTopologyCheckedException))
+                log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
 
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                if (!newTopVer.equals(topVer)) {
-                    assert newTopVer.compareTo(topVer) > 0;
+            if (!newTopVer.equals(topVer)) {
+                assert newTopVer.compareTo(topVer) > 0;
 
-                    // Reassignment will happen from topology event.
-                    return;
-                }
+                // Reassignment will happen from topology event.
+                return;
+            }
 
-                ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
-                    private IgniteUuid id = IgniteUuid.randomUuid();
+            ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
+                private IgniteUuid id = IgniteUuid.randomUuid();
 
-                    private long start = System.currentTimeMillis();
+                private long start = System.currentTimeMillis();
 
-                    @Override public IgniteUuid timeoutId() {
-                        return id;
-                    }
+                @Override public IgniteUuid timeoutId() {
+                    return id;
+                }
 
-                    @Override public long endTime() {
-                        return start + RETRY_TIMEOUT;
-                    }
+                @Override public long endTime() {
+                    return start + RETRY_TIMEOUT;
+                }
 
-                    @Override public void onTimeout() {
-                        if (!busyLock.enterBusy())
-                            return;
+                @Override public void onTimeout() {
+                    if (!busyLock.enterBusy())
+                        return;
 
-                        try {
-                            // Try again.
-                            onDeployment(dep, topVer);
-                        }
-                        finally {
-                            busyLock.leaveBusy();
-                        }
+                    try {
+                        // Try again.
+                        onDeployment(dep, topVer);
                     }
-                });
+                    finally {
+                        busyLock.leaveBusy();
+                    }
+                }
+            });
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 95e2cda4f6907..02ba56a884b33 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -129,6 +129,9 @@ class ClientImpl extends TcpDiscoveryImpl {
     /** */
     private static final Object SPI_RECONNECT_FAILED = "SPI_RECONNECT_FAILED";
 
+    /** */
+    private static final Object SPI_RECONNECT = "SPI_RECONNECT";
+
     /** Remote nodes. */
     private final ConcurrentMap<UUID, TcpDiscoveryNode> rmtNodes = new ConcurrentHashMap8<>();
 
@@ -808,6 +811,11 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
             log);
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        msgWorker.addMessage(SPI_RECONNECT);
+    }
+
     /** {@inheritDoc} */
     @Override public void brakeConnection() {
         SocketStream sockStream = msgWorker.currSock;
@@ -879,9 +887,12 @@ private class SocketReader extends IgniteSpiThread {
         /** */
         private UUID rmtNodeId;
 
+        /** */
+        private CountDownLatch stopReadLatch;
+
         /**
          */
-        protected SocketReader() {
+        SocketReader() {
             super(spi.ignite().name(), "tcp-client-disco-sock-reader", log);
         }
 
@@ -889,7 +900,7 @@ protected SocketReader() {
          * @param sockStream Socket.
          * @param rmtNodeId Rmt node id.
          */
-        public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
+        void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             synchronized (mux) {
                 this.sockStream = sockStream;
 
@@ -899,6 +910,31 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             }
         }
 
+        /**
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceStopRead() throws InterruptedException {
+            CountDownLatch stopReadLatch;
+
+            synchronized (mux) {
+                SocketStream stream = sockStream;
+
+                if (stream == null)
+                    return;
+
+                this.stopReadLatch = stopReadLatch = new CountDownLatch(1);
+
+                U.closeQuiet(stream.socket());
+
+                this.sockStream = null;
+                this.rmtNodeId = null;
+
+                mux.notifyAll();
+            }
+
+            stopReadLatch.await();
+        }
+
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException {
             while (!isInterrupted()) {
@@ -906,6 +942,12 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
                 UUID rmtNodeId;
 
                 synchronized (mux) {
+                    if (stopReadLatch != null) {
+                        stopReadLatch.countDown();
+
+                        stopReadLatch = null;
+                    }
+
                     if (this.sockStream == null) {
                         mux.wait();
 
@@ -1007,18 +1049,21 @@ private class SocketWriter extends IgniteSpiThread {
         private final Queue<TcpDiscoveryAbstractMessage> queue = new ArrayDeque<>();
 
         /** */
-        private final long socketTimeout;
+        private final long sockTimeout;
 
         /** */
         private TcpDiscoveryAbstractMessage unackedMsg;
 
+        /** */
+        private CountDownLatch forceLeaveLatch;
+
         /**
          *
          */
-        protected SocketWriter() {
+        SocketWriter() {
             super(spi.ignite().name(), "tcp-client-disco-sock-writer", log);
 
-            socketTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
+            sockTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
                 spi.getSocketTimeout();
         }
 
@@ -1033,6 +1078,29 @@ private void sendMessage(TcpDiscoveryAbstractMessage msg) {
             }
         }
 
+        /**
+         * Sends {@link TcpDiscoveryNodeLeftMessage} and closes socket.
+         *
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceLeave() throws InterruptedException {
+            CountDownLatch forceLeaveLatch;
+
+            synchronized (mux) {
+                // If writer was stopped.
+                if (sock == null)
+                    return;
+
+                this.forceLeaveLatch = forceLeaveLatch = new CountDownLatch(1);
+
+                unackedMsg = null;
+
+                mux.notifyAll();
+            }
+
+            forceLeaveLatch.await();
+        }
+
         /**
          * @param sock Socket.
          * @param clientAck {@code True} is server supports client message acknowlede.
@@ -1089,13 +1157,41 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                         continue;
                     }
 
-                    msg = queue.poll();
+                    if (forceLeaveLatch != null) {
+                        msg = new TcpDiscoveryNodeLeftMessage(getLocalNodeId());
 
-                    if (msg == null) {
-                        mux.wait();
+                        msg.client(true);
+
+                        try {
+                            spi.writeToSocket(
+                                sock,
+                                msg,
+                                sockTimeout);
+                        }
+                        catch (IOException | IgniteCheckedException e) {
+                            if (log.isDebugEnabled()) {
+                                log.debug("Failed to send TcpDiscoveryNodeLeftMessage on force leave [msg=" + msg +
+                                    ", err=" + e.getMessage() + ']');
+                            }
+                        }
+
+                        U.closeQuiet(sock);
+
+                        this.sock = null;
+
+                        clear();
 
                         continue;
                     }
+                    else {
+                        msg = queue.poll();
+
+                        if (msg == null) {
+                            mux.wait();
+
+                            continue;
+                        }
+                    }
                 }
 
                 for (IgniteInClosure<TcpDiscoveryAbstractMessage> msgLsnr : spi.sndMsgLsnrs)
@@ -1115,7 +1211,7 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     spi.writeToSocket(
                         sock,
                         msg,
-                        socketTimeout);
+                        sockTimeout);
 
                     msg = null;
 
@@ -1165,10 +1261,30 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     synchronized (mux) {
                         if (sock == this.sock)
                             this.sock = null; // Connection has dead.
+
+                        clear();
                     }
                 }
             }
         }
+
+        /**
+         *
+         */
+        private void clear() {
+            assert Thread.holdsLock(mux);
+
+            queue.clear();
+            unackedMsg = null;
+
+            CountDownLatch forceLeaveLatch = this.forceLeaveLatch;
+
+            if (forceLeaveLatch != null) {
+                this.forceLeaveLatch = null;
+
+                forceLeaveLatch.countDown();
+            }
+        }
     }
 
     /**
@@ -1413,6 +1529,38 @@ else if (msg == SPI_STOP) {
                         else
                             leaveLatch.countDown();
                     }
+                    else if (msg == SPI_RECONNECT) {
+                        if (state == CONNECTED) {
+                            if (reconnector != null) {
+                                reconnector.cancel();
+                                reconnector.join();
+
+                                reconnector = null;
+                            }
+
+                            sockWriter.forceLeave();
+                            sockReader.forceStopRead();
+
+                            currSock = null;
+
+                            queue.clear();
+
+                            onDisconnected();
+
+                            notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
+
+                            UUID newId = UUID.randomUUID();
+
+                            U.quietAndWarn(log, "Local node will try to reconnect to cluster with new id due " +
+                                "to network problems [newId=" + newId +
+                                ", prevId=" + locNode.id() +
+                                ", locNode=" + locNode+ ']');
+
+                            locNode.onClientDisconnected(newId);
+
+                            tryJoin();
+                        }
+                    }
                     else if (msg instanceof TcpDiscoveryNodeFailedMessage &&
                         ((TcpDiscoveryNodeFailedMessage)msg).failedNodeId().equals(locNode.id())) {
                         TcpDiscoveryNodeFailedMessage msg0 = (TcpDiscoveryNodeFailedMessage)msg;
@@ -1495,20 +1643,7 @@ else if (msg == SPI_RECONNECT_FAILED) {
                                         ", failMsg=" + forceFailMsg + ']');
                                 }
 
-                                state = DISCONNECTED;
-
-                                nodeAdded = false;
-
-                                IgniteClientDisconnectedCheckedException err =
-                                    new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
-                                    "client node disconnected.");
-
-                                for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
-                                    GridFutureAdapter<Boolean> fut = e.getValue();
-
-                                    if (pingFuts.remove(e.getKey(), fut))
-                                        fut.onDone(err);
-                                }
+                                onDisconnected();
 
                                 notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
                             }
@@ -1603,6 +1738,26 @@ else if (discoMsg instanceof TcpDiscoveryCheckFailedMessage)
             }
         }
 
+        /**
+         *
+         */
+        private void onDisconnected() {
+            state = DISCONNECTED;
+
+            nodeAdded = false;
+
+            IgniteClientDisconnectedCheckedException err =
+                new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
+                    "client node disconnected.");
+
+            for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
+                GridFutureAdapter<Boolean> fut = e.getValue();
+
+                if (pingFuts.remove(e.getKey(), fut))
+                    fut.onDone(err);
+            }
+        }
+
         /**
          * @throws InterruptedException If interrupted.
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 4600be094c840..afd1c2ba7fc66 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1609,6 +1609,11 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
         throw new UnsupportedOperationException();
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        throw new UnsupportedOperationException("Reconnect is not supported for server.");
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteSpiThread workerThread() {
         return msgWorker;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index f199c20e17edb..84c2ff28afef0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -258,6 +259,13 @@ protected static String threadStatus(Thread t) {
         return t.isAlive() ? "alive" : "dead";
     }
 
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public abstract void reconnect() throws IgniteSpiException;
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      * <p>
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 00ae97d1e302c..a2a47feb31486 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1926,6 +1926,15 @@ boolean isSslEnabled() {
         return ignite().configuration().getSslContextFactory() != null;
     }
 
+    /**
+     * Force reconnect to cluster.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() throws IgniteSpiException {
+        impl.reconnect();
+    }
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 0f0165b0f04f9..6cdf465978907 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -700,9 +700,12 @@ public void testReconnectInitialExchangeInProgress() throws Exception {
                 try {
                     Ignition.start(optimize(getConfiguration(getTestGridName(SRV_CNT))));
 
-                    fail();
+                    // Commented due to IGNITE-4473, because
+                    // IgniteClientDisconnectedException won't
+                    // be thrown, but client will reconnect.
+//                    fail();
 
-                    return false;
+                    return true;
                 }
                 catch (IgniteClientDisconnectedException e) {
                     log.info("Expected start error: " + e);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
new file mode 100644
index 0000000000000..a5d42e9a12cba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests client to be able restore connection to cluster if coordination is not available.
+ */
+public class IgniteClientRejoinTest extends GridCommonAbstractTest {
+    /** Block. */
+    private volatile boolean block;
+
+    /** Block all. */
+    private volatile boolean blockAll;
+
+    /** Coordinator. */
+    private volatile ClusterNode crd;
+
+    /** Client reconnect disabled. */
+    private boolean clientReconnectDisabled;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK", "true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.clearProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        clientReconnectDisabled = false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("client")) {
+            cfg.setCommunicationSpi(new TcpCommunicationSpi());
+
+            TcpDiscoverySpi spi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+            DiscoverySpi dspi = new DiscoverySpi();
+
+            dspi.setIpFinder(spi.getIpFinder());
+
+            cfg.setDiscoverySpi(dspi);
+
+            dspi.setJoinTimeout(60_000);
+            dspi.setClientReconnectDisabled(clientReconnectDisabled);
+
+            cfg.setClientMode(true);
+        }
+
+        // TODO: IGNITE-4833
+        cfg.setPeerClassLoadingEnabled(false);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectAfterStart() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        List<Ignite> clientNodes = new ArrayList<>();
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++)
+            clientNodes.add(startGrid("client" + i));
+
+        blockAll = true;
+
+        GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                U.sleep(5_000);
+
+                block = true;
+                blockAll = false;
+
+                System.out.println(">>> Allow with blocked coordinator.");
+
+                latch.countDown();
+
+                return null;
+            }
+        });
+
+        IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                latch.await();
+
+                U.sleep((new Random().nextInt(15) + 30) * 1000);
+
+                block = false;
+
+                System.out.println(">>> Allow coordinator.");
+
+                return null;
+            }
+        });
+
+        fut.get();
+
+        for (Ignite client : clientNodes) {
+            while (true) {
+                try {
+                    IgniteCache<Integer, Integer> cache = client.getOrCreateCache("some");
+
+                    for (int i = 0; i < 100; i++)
+                        cache.put(i, i);
+
+                    for (int i = 0; i < 100; i++)
+                        assertEquals((Integer)i, cache.get(i));
+
+                    cache.clear();
+
+                    break;
+                }
+                catch (IgniteClientDisconnectedException e) {
+                    e.reconnectFuture().get();
+                }
+            }
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnect() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        GridTestUtils.runAsync(new Callable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                latch.countDown();
+
+                Random rnd = new Random();
+
+                U.sleep((rnd.nextInt(15) + 15) * 1000);
+
+                block = false;
+
+                System.out.println(">>> ALLOW connection to coordinator.");
+
+                return true;
+            }
+        });
+
+        for (IgniteInternalFuture<Ignite> clientFut : futs) {
+            Ignite client = clientFut.get();
+
+            IgniteCache<Integer, Integer> cache = client.getOrCreateCache(client.name());
+
+            for (int i = 0; i < 100; i++)
+                cache.put(i, i);
+
+            for (int i = 0; i < 100; i++)
+                assert i == cache.get(i);
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectDisabled() throws Exception {
+        clientReconnectDisabled = true;
+
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        latch.countDown();
+
+        for (final IgniteInternalFuture<Ignite> clientFut : futs) {
+            //noinspection ThrowableNotThrown
+            GridTestUtils.assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    clientFut.get();
+
+                    return null;
+                }
+            }, IgniteCheckedException.class, null);
+        }
+
+        assertEquals(0, srv1.cluster().forClients().nodes().size());
+        assertEquals(0, srv2.cluster().forClients().nodes().size());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 3 * 60_000;
+    }
+
+    /**
+     *
+     */
+    private class TcpCommunicationSpi extends org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     *
+     */
+    private class DiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, byte[] data,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(TcpDiscoveryAbstractMessage msg, Socket sock, int res,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(msg, sock, res, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Socket openSocket(Socket sock, InetSocketAddress remAddr,
+            IgniteSpiOperationTimeoutHelper timeoutHelper) throws IOException, IgniteSpiOperationTimeoutException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            return super.openSocket(sock, remAddr, timeoutHelper);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 331b581bb7bec..0483a1ce8fbc6 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -32,6 +32,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.IgniteState;
@@ -43,6 +44,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -1788,8 +1790,7 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
         clientNodeIds.add(client.cluster().localNode().id());
 
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override
-            public boolean apply() {
+            @Override public boolean apply() {
                 return srv.cluster().nodes().size() == 2;
             }
         }, awaitTime());
@@ -1799,6 +1800,49 @@ public boolean apply() {
         assertFalse(err.get());
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testForceClientReconnect() throws Exception {
+        startServerNodes(1);
+
+        startClientNodes(1);
+
+        Ignite srv = G.ignite("server-0");
+        IgniteKernal client = (IgniteKernal)G.ignite("client-0");
+
+        UUID clientId = F.first(clientNodeIds);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        srv.events().enableLocal(EVT_NODE_JOINED);
+
+        srv.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                latch.countDown();
+
+                return false;
+            }
+        }, EVT_NODE_JOINED);
+
+        client.context().discovery().reconnect();
+
+        assert latch.await(10, TimeUnit.SECONDS);
+
+        while (true) {
+            try {
+                UUID newId = client.localNode().id();
+
+                assert !clientId.equals(newId) : clientId;
+
+                break;
+            }
+            catch (IgniteClientDisconnectedException e) {
+                e.reconnectFuture().get(10_000);
+            }
+        }
+    }
+
     /**
      * @param ignite Ignite.
      * @throws Exception If failed.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index ea8e37bada1d0..67d88e1e29eac 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.IgniteClientReconnectServicesTest;
 import org.apache.ignite.internal.IgniteClientReconnectStopTest;
 import org.apache.ignite.internal.IgniteClientReconnectStreamerTest;
+import org.apache.ignite.internal.IgniteClientRejoinTest;
 
 /**
  *
@@ -52,6 +53,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteClientReconnectServicesTest.class);
         suite.addTestSuite(IgniteClientReconnectStreamerTest.class);
         suite.addTestSuite(IgniteClientReconnectFailoverTest.class);
+        suite.addTestSuite(IgniteClientRejoinTest.class);
 
         return suite;
     }

From 946419314b567c604a15ae4f9658d89bc350127b Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 17 Mar 2017 14:57:48 +0300
Subject: [PATCH 184/446] IGNITE-4473 - Client should re-try connection attempt
 in case of concurrent network failure.

(cherry picked from commit d124004)
---
 .../internal/GridKernalGatewayImpl.java       |   8 +-
 .../apache/ignite/internal/IgniteKernal.java  | 120 +++++-
 .../IgniteNeedReconnectException.java         |  40 ++
 .../discovery/GridDiscoveryManager.java       |  24 ++
 .../GridCachePartitionExchangeManager.java    |  25 +-
 .../dht/GridDhtAssignmentFetchFuture.java     |  14 +-
 .../GridDhtPartitionsExchangeFuture.java      |  48 ++-
 .../service/GridServiceProcessor.java         |  86 ++--
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 201 ++++++++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   5 +
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |   8 +
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   9 +
 .../IgniteClientReconnectCacheTest.java       |   7 +-
 .../internal/IgniteClientRejoinTest.java      | 378 ++++++++++++++++++
 .../tcp/TcpClientDiscoverySpiSelfTest.java    |  48 ++-
 .../IgniteClientReconnectTestSuite.java       |   2 +
 16 files changed, 929 insertions(+), 94 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
index fe8c580ca63ea..036954a3a280d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
@@ -44,7 +44,7 @@ public class GridKernalGatewayImpl implements GridKernalGateway, Serializable {
 
     /** */
     @GridToStringExclude
-    private IgniteFutureImpl<?> reconnectFut;
+    private volatile IgniteFutureImpl<?> reconnectFut;
 
     /** */
     private final AtomicReference<GridKernalState> state = new AtomicReference<>(GridKernalState.STOPPED);
@@ -149,6 +149,12 @@ public GridKernalGatewayImpl(String gridName) {
 
     /** {@inheritDoc} */
     @Override public GridFutureAdapter<?> onDisconnected() {
+        if (state.get() == GridKernalState.DISCONNECTED) {
+            assert reconnectFut != null;
+
+            return (GridFutureAdapter<?>)reconnectFut.internalFuture();
+        }
+
         GridFutureAdapter<?> fut = new GridFutureAdapter<>();
 
         reconnectFut = new IgniteFutureImpl<>(fut);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 4972d1f306e2e..063fe255af95f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -250,6 +250,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     /** Periodic starvation check interval. */
     private static final long PERIODIC_STARVATION_CHECK_FREQ = 1000 * 30;
 
+    /** Force complete reconnect future. */
+    private static final Object STOP_RECONNECT = new Object();
+
     /** */
     @GridToStringExclude
     private GridKernalContextImpl ctx;
@@ -327,6 +330,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     @GridToStringExclude
     private final AtomicBoolean stopGuard = new AtomicBoolean();
 
+    /** */
+    private final ReconnectState reconnectState = new ReconnectState();
+
     /**
      * No-arg constructor is required by externalization.
      */
@@ -936,6 +942,8 @@ public void start(
             // Notify IO manager the second so further components can send and receive messages.
             ctx.io().onKernalStart();
 
+            boolean recon = false;
+
             // Callbacks.
             for (GridComponent comp : ctx) {
                 // Skip discovery manager.
@@ -946,10 +954,24 @@ public void start(
                 if (comp instanceof GridIoManager)
                     continue;
 
-                if (!skipDaemon(comp))
-                    comp.onKernalStart();
+                if (!skipDaemon(comp)) {
+                    try {
+                        comp.onKernalStart();
+                    }
+                    catch (IgniteNeedReconnectException e) {
+                        assert ctx.discovery().reconnectSupported();
+
+                        if (log.isDebugEnabled())
+                            log.debug("Failed to start node components on node start, will wait for reconnect: " + e);
+
+                        recon = true;
+                    }
+                }
             }
 
+            if (recon)
+                reconnectState.waitFirstReconnect();
+
             // Register MBeans.
             registerKernalMBean();
             registerLocalNodeMBean();
@@ -3309,6 +3331,8 @@ private void unguard() {
     public void onDisconnected() {
         Throwable err = null;
 
+        reconnectState.waitPreviousReconnect();
+
         GridFutureAdapter<?> reconnectFut = ctx.gateway().onDisconnected();
 
         if (reconnectFut == null) {
@@ -3317,9 +3341,18 @@ public void onDisconnected() {
             return;
         }
 
-        IgniteFuture<?> userFut = new IgniteFutureImpl<>(reconnectFut);
+        IgniteFutureImpl<?> curFut = (IgniteFutureImpl<?>)ctx.cluster().get().clientReconnectFuture();
+
+        IgniteFuture<?> userFut;
 
-        ctx.cluster().get().clientReconnectFuture(userFut);
+        // In case of previous reconnect did not finish keep reconnect future.
+        if (curFut != null && curFut.internalFuture() == reconnectFut)
+            userFut = curFut;
+        else {
+            userFut = new IgniteFutureImpl<>(reconnectFut);
+
+            ctx.cluster().get().clientReconnectFuture(userFut);
+        }
 
         ctx.disconnected(true);
 
@@ -3372,30 +3405,53 @@ public void onReconnected(final boolean clusterRestarted) {
         try {
             ctx.disconnected(false);
 
-            GridCompoundFuture<?, ?> reconnectFut = new GridCompoundFuture<>();
+            GridCompoundFuture curReconnectFut = reconnectState.curReconnectFut = new GridCompoundFuture<>();
+
+            reconnectState.reconnectDone = new GridFutureAdapter<>();
 
             for (GridComponent comp : ctx.components()) {
                 IgniteInternalFuture<?> fut = comp.onReconnected(clusterRestarted);
 
                 if (fut != null)
-                    reconnectFut.add((IgniteInternalFuture)fut);
+                    curReconnectFut.add(fut);
             }
 
-            reconnectFut.add((IgniteInternalFuture)ctx.cache().context().exchange().reconnectExchangeFuture());
+            curReconnectFut.add(ctx.cache().context().exchange().reconnectExchangeFuture());
+
+            curReconnectFut.markInitialized();
 
-            reconnectFut.markInitialized();
+            final GridFutureAdapter reconnectDone = reconnectState.reconnectDone;
 
-            reconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
+            curReconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
                 @Override public void apply(IgniteInternalFuture<?> fut) {
                     try {
-                        fut.get();
+                        Object res = fut.get();
+
+                        if (res == STOP_RECONNECT)
+                            return;
 
                         ctx.gateway().onReconnected();
+
+                        reconnectState.firstReconnectFut.onDone();
                     }
                     catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to reconnect, will stop node", e);
+                        if (!X.hasCause(e, IgniteNeedReconnectException.class,
+                            IgniteClientDisconnectedCheckedException.class)) {
+                            U.error(log, "Failed to reconnect, will stop node.", e);
+
+                            reconnectState.firstReconnectFut.onDone(e);
 
-                        close();
+                            close();
+                        }
+                        else {
+                            assert ctx.discovery().reconnectSupported();
+
+                            U.error(log, "Failed to finish reconnect, will retry [locNodeId=" + ctx.localNodeId() +
+                                ", err=" + e.getMessage() + ']');
+                        }
+                    }
+                    finally {
+                        reconnectDone.onDone();
                     }
                 }
             });
@@ -3574,6 +3630,46 @@ public IgniteInternalFuture sendIoTest(List<ClusterNode> nodes, byte[] payload,
         return ctx.io().sendIoTest(nodes, payload, procFromNioThread);
     }
 
+    /**
+     *
+     */
+    private class ReconnectState {
+        /** */
+        private final GridFutureAdapter firstReconnectFut = new GridFutureAdapter();
+
+        /** */
+        private GridCompoundFuture<?, Object> curReconnectFut;
+
+        /** */
+        private GridFutureAdapter<?> reconnectDone;
+
+        /**
+         * @throws IgniteCheckedException If failed.
+         */
+        void waitFirstReconnect() throws IgniteCheckedException {
+            firstReconnectFut.get();
+        }
+
+        /**
+         *
+         */
+        void waitPreviousReconnect() {
+            if (curReconnectFut != null && !curReconnectFut.isDone()) {
+                assert reconnectDone != null;
+
+                curReconnectFut.onDone(STOP_RECONNECT);
+
+                try {
+                    reconnectDone.get();
+                }
+                catch (IgniteCheckedException ignote) {
+                    // No-op.
+                }
+            }
+
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IgniteKernal.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
new file mode 100644
index 0000000000000..61ab5762b95ac
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Indicates that node should try reconnect to cluster.
+ */
+public class IgniteNeedReconnectException extends IgniteCheckedException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * @param locNode Local node.
+     * @param cause Cause.
+     */
+    public IgniteNeedReconnectException(ClusterNode locNode, @Nullable Throwable cause) {
+        super("Local node need try to reconnect [locNodeId=" + locNode.id() + ']', cause);
+
+        assert locNode.isClient();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 9aa4db1e0042f..2ec10705bedbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -112,6 +112,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -1890,6 +1891,29 @@ public void failNode(UUID nodeId, @Nullable String warning) {
         }
     }
 
+    /**
+     * @return {@code True} if local node client and discovery SPI supports reconnect.
+     */
+    public boolean reconnectSupported() {
+        DiscoverySpi spi = getSpi();
+
+        return ctx.clientNode() && (spi instanceof TcpDiscoverySpi) &&
+            !(((TcpDiscoverySpi) spi).isClientReconnectDisabled());
+    }
+
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() {
+        assert reconnectSupported();
+
+        DiscoverySpi discoverySpi = getSpi();
+
+        ((TcpDiscoverySpi)discoverySpi).reconnect();
+    }
+
     /**
      * Updates topology version if current version is smaller than updated.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7cf75fee86be3..6f2c62608670f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -43,6 +43,7 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
@@ -448,6 +449,15 @@ else if (m instanceof GridDhtPartitionDemandMessage)
                     else
                         U.warn(log, "Still waiting for initial partition map exchange [fut=" + fut + ']');
                 }
+                catch (IgniteNeedReconnectException e) {
+                    throw e;
+                }
+                catch (Exception e) {
+                    if (fut.reconnectOnError(e))
+                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                    throw e;
+                }
             }
 
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
@@ -1697,6 +1707,12 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                         dumpedObjects++;
                                     }
                                 }
+                                catch (Exception e) {
+                                    if (exchFut.reconnectOnError(e))
+                                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                                    throw e;
+                                }
                             }
 
 
@@ -1836,7 +1852,14 @@ else if (r != null) {
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
                 }
-                catch (IgniteClientDisconnectedCheckedException e) {
+                catch (IgniteClientDisconnectedCheckedException | IgniteNeedReconnectException e) {
+                    assert cctx.discovery().reconnectSupported();
+
+                    U.warn(log,"Local node failed to complete partition map exchange due to " +
+                        "network issues, will try to reconnect to cluster", e);
+
+                    cctx.discovery().reconnect();
+
                     return;
                 }
                 catch (IgniteCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index ab8e863e90e7d..6425bc141245d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -17,15 +17,16 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.Queue;
 import java.util.UUID;
-import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
@@ -34,6 +35,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -202,8 +204,14 @@ private void requestFromNextNode() {
                         "continue to another node): " + node);
                 }
                 catch (IgniteCheckedException e) {
-                    U.error(log0, "Failed to request affinity assignment from remote node (will " +
-                        "continue to another node): " + node, e);
+                    if (ctx.discovery().reconnectSupported() && X.hasCause(e, IOException.class)) {
+                        onDone(new IgniteNeedReconnectException(ctx.localNode(), e));
+
+                        return;
+                    }
+
+                    U.warn(log0, "Failed to request affinity assignment from remote node (will " +
+                        "continue to another node): " + node);
                 }
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index e945de958a339..d4f95e5b55d05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -32,6 +33,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
@@ -39,6 +41,7 @@
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -54,7 +57,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
@@ -65,7 +67,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -506,10 +508,17 @@ public void init() throws IgniteInterruptedCheckedException {
 
             throw e;
         }
+        catch (IgniteNeedReconnectException e) {
+            onDone(e);
+        }
         catch (Throwable e) {
-            U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else {
+                U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
 
-            onDone(e);
+                onDone(e);
+            }
 
             if (e instanceof Error)
                 throw (Error)e;
@@ -1297,7 +1306,10 @@ private void onAllReceived(boolean discoThread) {
             }
         }
         catch (IgniteCheckedException e) {
-            onDone(e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else
+                onDone(e);
         }
     }
 
@@ -1314,8 +1326,15 @@ private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
         }
         catch (IgniteCheckedException e) {
             if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) {
-                log.debug("Failed to send full partition map to node, node left grid " +
-                    "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+                if (log.isDebugEnabled())
+                    log.debug("Failed to send full partition map to node, node left grid " +
+                        "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+
+                return;
+            }
+
+            if (reconnectOnError(e)) {
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
 
                 return;
             }
@@ -1641,6 +1660,12 @@ public void onNodeLeft(final ClusterNode node) {
                             }
                         }
                     }
+                    catch (Exception e) {
+                        if (reconnectOnError(e))
+                            onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+                        else
+                            throw e;
+                    }
                     finally {
                         leaveBusy();
                     }
@@ -1652,6 +1677,15 @@ public void onNodeLeft(final ClusterNode node) {
         }
     }
 
+    /**
+     * @param e Exception.
+     * @return {@code True} if local node should try reconnect in case of error.
+     */
+    public boolean reconnectOnError(Throwable e) {
+        return X.hasCause(e, IOException.class, IgniteClientDisconnectedCheckedException.class) &&
+            cctx.discovery().reconnectSupported();
+    }
+
     /** {@inheritDoc} */
     @Override public int compareTo(GridDhtPartitionsExchangeFuture fut) {
         return exchId.compareTo(fut.exchId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index f7f370ee56f56..730c6a4aad390 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1498,60 +1498,60 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
         }
     }
 
-        /**
-         * Deployment callback.
-         *
-         * @param dep Service deployment.
-         * @param topVer Topology version.
-         */
-        private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
-            // Retry forever.
-            try {
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+    /**
+     * Deployment callback.
+     *
+     * @param dep Service deployment.
+     * @param topVer Topology version.
+     */
+    private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
+        // Retry forever.
+        try {
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                // If topology version changed, reassignment will happen from topology event.
-                if (newTopVer.equals(topVer))
-                    reassign(dep, topVer);
-            }
-            catch (IgniteCheckedException e) {
-                if (!(e instanceof ClusterTopologyCheckedException))
-                    log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
+            // If topology version changed, reassignment will happen from topology event.
+            if (newTopVer.equals(topVer))
+                reassign(dep, topVer);
+        }
+        catch (IgniteCheckedException e) {
+            if (!(e instanceof ClusterTopologyCheckedException))
+                log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
 
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                if (!newTopVer.equals(topVer)) {
-                    assert newTopVer.compareTo(topVer) > 0;
+            if (!newTopVer.equals(topVer)) {
+                assert newTopVer.compareTo(topVer) > 0;
 
-                    // Reassignment will happen from topology event.
-                    return;
-                }
+                // Reassignment will happen from topology event.
+                return;
+            }
 
-                ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
-                    private IgniteUuid id = IgniteUuid.randomUuid();
+            ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
+                private IgniteUuid id = IgniteUuid.randomUuid();
 
-                    private long start = System.currentTimeMillis();
+                private long start = System.currentTimeMillis();
 
-                    @Override public IgniteUuid timeoutId() {
-                        return id;
-                    }
+                @Override public IgniteUuid timeoutId() {
+                    return id;
+                }
 
-                    @Override public long endTime() {
-                        return start + RETRY_TIMEOUT;
-                    }
+                @Override public long endTime() {
+                    return start + RETRY_TIMEOUT;
+                }
 
-                    @Override public void onTimeout() {
-                        if (!busyLock.enterBusy())
-                            return;
+                @Override public void onTimeout() {
+                    if (!busyLock.enterBusy())
+                        return;
 
-                        try {
-                            // Try again.
-                            onDeployment(dep, topVer);
-                        }
-                        finally {
-                            busyLock.leaveBusy();
-                        }
+                    try {
+                        // Try again.
+                        onDeployment(dep, topVer);
                     }
-                });
+                    finally {
+                        busyLock.leaveBusy();
+                    }
+                }
+            });
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 95e2cda4f6907..02ba56a884b33 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -129,6 +129,9 @@ class ClientImpl extends TcpDiscoveryImpl {
     /** */
     private static final Object SPI_RECONNECT_FAILED = "SPI_RECONNECT_FAILED";
 
+    /** */
+    private static final Object SPI_RECONNECT = "SPI_RECONNECT";
+
     /** Remote nodes. */
     private final ConcurrentMap<UUID, TcpDiscoveryNode> rmtNodes = new ConcurrentHashMap8<>();
 
@@ -808,6 +811,11 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
             log);
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        msgWorker.addMessage(SPI_RECONNECT);
+    }
+
     /** {@inheritDoc} */
     @Override public void brakeConnection() {
         SocketStream sockStream = msgWorker.currSock;
@@ -879,9 +887,12 @@ private class SocketReader extends IgniteSpiThread {
         /** */
         private UUID rmtNodeId;
 
+        /** */
+        private CountDownLatch stopReadLatch;
+
         /**
          */
-        protected SocketReader() {
+        SocketReader() {
             super(spi.ignite().name(), "tcp-client-disco-sock-reader", log);
         }
 
@@ -889,7 +900,7 @@ protected SocketReader() {
          * @param sockStream Socket.
          * @param rmtNodeId Rmt node id.
          */
-        public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
+        void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             synchronized (mux) {
                 this.sockStream = sockStream;
 
@@ -899,6 +910,31 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             }
         }
 
+        /**
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceStopRead() throws InterruptedException {
+            CountDownLatch stopReadLatch;
+
+            synchronized (mux) {
+                SocketStream stream = sockStream;
+
+                if (stream == null)
+                    return;
+
+                this.stopReadLatch = stopReadLatch = new CountDownLatch(1);
+
+                U.closeQuiet(stream.socket());
+
+                this.sockStream = null;
+                this.rmtNodeId = null;
+
+                mux.notifyAll();
+            }
+
+            stopReadLatch.await();
+        }
+
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException {
             while (!isInterrupted()) {
@@ -906,6 +942,12 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
                 UUID rmtNodeId;
 
                 synchronized (mux) {
+                    if (stopReadLatch != null) {
+                        stopReadLatch.countDown();
+
+                        stopReadLatch = null;
+                    }
+
                     if (this.sockStream == null) {
                         mux.wait();
 
@@ -1007,18 +1049,21 @@ private class SocketWriter extends IgniteSpiThread {
         private final Queue<TcpDiscoveryAbstractMessage> queue = new ArrayDeque<>();
 
         /** */
-        private final long socketTimeout;
+        private final long sockTimeout;
 
         /** */
         private TcpDiscoveryAbstractMessage unackedMsg;
 
+        /** */
+        private CountDownLatch forceLeaveLatch;
+
         /**
          *
          */
-        protected SocketWriter() {
+        SocketWriter() {
             super(spi.ignite().name(), "tcp-client-disco-sock-writer", log);
 
-            socketTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
+            sockTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
                 spi.getSocketTimeout();
         }
 
@@ -1033,6 +1078,29 @@ private void sendMessage(TcpDiscoveryAbstractMessage msg) {
             }
         }
 
+        /**
+         * Sends {@link TcpDiscoveryNodeLeftMessage} and closes socket.
+         *
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceLeave() throws InterruptedException {
+            CountDownLatch forceLeaveLatch;
+
+            synchronized (mux) {
+                // If writer was stopped.
+                if (sock == null)
+                    return;
+
+                this.forceLeaveLatch = forceLeaveLatch = new CountDownLatch(1);
+
+                unackedMsg = null;
+
+                mux.notifyAll();
+            }
+
+            forceLeaveLatch.await();
+        }
+
         /**
          * @param sock Socket.
          * @param clientAck {@code True} is server supports client message acknowlede.
@@ -1089,13 +1157,41 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                         continue;
                     }
 
-                    msg = queue.poll();
+                    if (forceLeaveLatch != null) {
+                        msg = new TcpDiscoveryNodeLeftMessage(getLocalNodeId());
 
-                    if (msg == null) {
-                        mux.wait();
+                        msg.client(true);
+
+                        try {
+                            spi.writeToSocket(
+                                sock,
+                                msg,
+                                sockTimeout);
+                        }
+                        catch (IOException | IgniteCheckedException e) {
+                            if (log.isDebugEnabled()) {
+                                log.debug("Failed to send TcpDiscoveryNodeLeftMessage on force leave [msg=" + msg +
+                                    ", err=" + e.getMessage() + ']');
+                            }
+                        }
+
+                        U.closeQuiet(sock);
+
+                        this.sock = null;
+
+                        clear();
 
                         continue;
                     }
+                    else {
+                        msg = queue.poll();
+
+                        if (msg == null) {
+                            mux.wait();
+
+                            continue;
+                        }
+                    }
                 }
 
                 for (IgniteInClosure<TcpDiscoveryAbstractMessage> msgLsnr : spi.sndMsgLsnrs)
@@ -1115,7 +1211,7 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     spi.writeToSocket(
                         sock,
                         msg,
-                        socketTimeout);
+                        sockTimeout);
 
                     msg = null;
 
@@ -1165,10 +1261,30 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     synchronized (mux) {
                         if (sock == this.sock)
                             this.sock = null; // Connection has dead.
+
+                        clear();
                     }
                 }
             }
         }
+
+        /**
+         *
+         */
+        private void clear() {
+            assert Thread.holdsLock(mux);
+
+            queue.clear();
+            unackedMsg = null;
+
+            CountDownLatch forceLeaveLatch = this.forceLeaveLatch;
+
+            if (forceLeaveLatch != null) {
+                this.forceLeaveLatch = null;
+
+                forceLeaveLatch.countDown();
+            }
+        }
     }
 
     /**
@@ -1413,6 +1529,38 @@ else if (msg == SPI_STOP) {
                         else
                             leaveLatch.countDown();
                     }
+                    else if (msg == SPI_RECONNECT) {
+                        if (state == CONNECTED) {
+                            if (reconnector != null) {
+                                reconnector.cancel();
+                                reconnector.join();
+
+                                reconnector = null;
+                            }
+
+                            sockWriter.forceLeave();
+                            sockReader.forceStopRead();
+
+                            currSock = null;
+
+                            queue.clear();
+
+                            onDisconnected();
+
+                            notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
+
+                            UUID newId = UUID.randomUUID();
+
+                            U.quietAndWarn(log, "Local node will try to reconnect to cluster with new id due " +
+                                "to network problems [newId=" + newId +
+                                ", prevId=" + locNode.id() +
+                                ", locNode=" + locNode+ ']');
+
+                            locNode.onClientDisconnected(newId);
+
+                            tryJoin();
+                        }
+                    }
                     else if (msg instanceof TcpDiscoveryNodeFailedMessage &&
                         ((TcpDiscoveryNodeFailedMessage)msg).failedNodeId().equals(locNode.id())) {
                         TcpDiscoveryNodeFailedMessage msg0 = (TcpDiscoveryNodeFailedMessage)msg;
@@ -1495,20 +1643,7 @@ else if (msg == SPI_RECONNECT_FAILED) {
                                         ", failMsg=" + forceFailMsg + ']');
                                 }
 
-                                state = DISCONNECTED;
-
-                                nodeAdded = false;
-
-                                IgniteClientDisconnectedCheckedException err =
-                                    new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
-                                    "client node disconnected.");
-
-                                for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
-                                    GridFutureAdapter<Boolean> fut = e.getValue();
-
-                                    if (pingFuts.remove(e.getKey(), fut))
-                                        fut.onDone(err);
-                                }
+                                onDisconnected();
 
                                 notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
                             }
@@ -1603,6 +1738,26 @@ else if (discoMsg instanceof TcpDiscoveryCheckFailedMessage)
             }
         }
 
+        /**
+         *
+         */
+        private void onDisconnected() {
+            state = DISCONNECTED;
+
+            nodeAdded = false;
+
+            IgniteClientDisconnectedCheckedException err =
+                new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
+                    "client node disconnected.");
+
+            for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
+                GridFutureAdapter<Boolean> fut = e.getValue();
+
+                if (pingFuts.remove(e.getKey(), fut))
+                    fut.onDone(err);
+            }
+        }
+
         /**
          * @throws InterruptedException If interrupted.
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 89a9efa481eaa..dfe614b715026 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1609,6 +1609,11 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
         throw new UnsupportedOperationException();
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        throw new UnsupportedOperationException("Reconnect is not supported for server.");
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteSpiThread workerThread() {
         return msgWorker;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index f199c20e17edb..84c2ff28afef0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -258,6 +259,13 @@ protected static String threadStatus(Thread t) {
         return t.isAlive() ? "alive" : "dead";
     }
 
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public abstract void reconnect() throws IgniteSpiException;
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      * <p>
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 00ae97d1e302c..a2a47feb31486 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1926,6 +1926,15 @@ boolean isSslEnabled() {
         return ignite().configuration().getSslContextFactory() != null;
     }
 
+    /**
+     * Force reconnect to cluster.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() throws IgniteSpiException {
+        impl.reconnect();
+    }
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 0f0165b0f04f9..6cdf465978907 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -700,9 +700,12 @@ public void testReconnectInitialExchangeInProgress() throws Exception {
                 try {
                     Ignition.start(optimize(getConfiguration(getTestGridName(SRV_CNT))));
 
-                    fail();
+                    // Commented due to IGNITE-4473, because
+                    // IgniteClientDisconnectedException won't
+                    // be thrown, but client will reconnect.
+//                    fail();
 
-                    return false;
+                    return true;
                 }
                 catch (IgniteClientDisconnectedException e) {
                     log.info("Expected start error: " + e);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
new file mode 100644
index 0000000000000..a5d42e9a12cba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests client to be able restore connection to cluster if coordination is not available.
+ */
+public class IgniteClientRejoinTest extends GridCommonAbstractTest {
+    /** Block. */
+    private volatile boolean block;
+
+    /** Block all. */
+    private volatile boolean blockAll;
+
+    /** Coordinator. */
+    private volatile ClusterNode crd;
+
+    /** Client reconnect disabled. */
+    private boolean clientReconnectDisabled;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK", "true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.clearProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        clientReconnectDisabled = false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("client")) {
+            cfg.setCommunicationSpi(new TcpCommunicationSpi());
+
+            TcpDiscoverySpi spi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+            DiscoverySpi dspi = new DiscoverySpi();
+
+            dspi.setIpFinder(spi.getIpFinder());
+
+            cfg.setDiscoverySpi(dspi);
+
+            dspi.setJoinTimeout(60_000);
+            dspi.setClientReconnectDisabled(clientReconnectDisabled);
+
+            cfg.setClientMode(true);
+        }
+
+        // TODO: IGNITE-4833
+        cfg.setPeerClassLoadingEnabled(false);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectAfterStart() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        List<Ignite> clientNodes = new ArrayList<>();
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++)
+            clientNodes.add(startGrid("client" + i));
+
+        blockAll = true;
+
+        GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                U.sleep(5_000);
+
+                block = true;
+                blockAll = false;
+
+                System.out.println(">>> Allow with blocked coordinator.");
+
+                latch.countDown();
+
+                return null;
+            }
+        });
+
+        IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                latch.await();
+
+                U.sleep((new Random().nextInt(15) + 30) * 1000);
+
+                block = false;
+
+                System.out.println(">>> Allow coordinator.");
+
+                return null;
+            }
+        });
+
+        fut.get();
+
+        for (Ignite client : clientNodes) {
+            while (true) {
+                try {
+                    IgniteCache<Integer, Integer> cache = client.getOrCreateCache("some");
+
+                    for (int i = 0; i < 100; i++)
+                        cache.put(i, i);
+
+                    for (int i = 0; i < 100; i++)
+                        assertEquals((Integer)i, cache.get(i));
+
+                    cache.clear();
+
+                    break;
+                }
+                catch (IgniteClientDisconnectedException e) {
+                    e.reconnectFuture().get();
+                }
+            }
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnect() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        GridTestUtils.runAsync(new Callable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                latch.countDown();
+
+                Random rnd = new Random();
+
+                U.sleep((rnd.nextInt(15) + 15) * 1000);
+
+                block = false;
+
+                System.out.println(">>> ALLOW connection to coordinator.");
+
+                return true;
+            }
+        });
+
+        for (IgniteInternalFuture<Ignite> clientFut : futs) {
+            Ignite client = clientFut.get();
+
+            IgniteCache<Integer, Integer> cache = client.getOrCreateCache(client.name());
+
+            for (int i = 0; i < 100; i++)
+                cache.put(i, i);
+
+            for (int i = 0; i < 100; i++)
+                assert i == cache.get(i);
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectDisabled() throws Exception {
+        clientReconnectDisabled = true;
+
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        latch.countDown();
+
+        for (final IgniteInternalFuture<Ignite> clientFut : futs) {
+            //noinspection ThrowableNotThrown
+            GridTestUtils.assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    clientFut.get();
+
+                    return null;
+                }
+            }, IgniteCheckedException.class, null);
+        }
+
+        assertEquals(0, srv1.cluster().forClients().nodes().size());
+        assertEquals(0, srv2.cluster().forClients().nodes().size());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 3 * 60_000;
+    }
+
+    /**
+     *
+     */
+    private class TcpCommunicationSpi extends org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     *
+     */
+    private class DiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, byte[] data,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(TcpDiscoveryAbstractMessage msg, Socket sock, int res,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(msg, sock, res, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Socket openSocket(Socket sock, InetSocketAddress remAddr,
+            IgniteSpiOperationTimeoutHelper timeoutHelper) throws IOException, IgniteSpiOperationTimeoutException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            return super.openSocket(sock, remAddr, timeoutHelper);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 331b581bb7bec..0483a1ce8fbc6 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -32,6 +32,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.IgniteState;
@@ -43,6 +44,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -1788,8 +1790,7 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
         clientNodeIds.add(client.cluster().localNode().id());
 
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override
-            public boolean apply() {
+            @Override public boolean apply() {
                 return srv.cluster().nodes().size() == 2;
             }
         }, awaitTime());
@@ -1799,6 +1800,49 @@ public boolean apply() {
         assertFalse(err.get());
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testForceClientReconnect() throws Exception {
+        startServerNodes(1);
+
+        startClientNodes(1);
+
+        Ignite srv = G.ignite("server-0");
+        IgniteKernal client = (IgniteKernal)G.ignite("client-0");
+
+        UUID clientId = F.first(clientNodeIds);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        srv.events().enableLocal(EVT_NODE_JOINED);
+
+        srv.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                latch.countDown();
+
+                return false;
+            }
+        }, EVT_NODE_JOINED);
+
+        client.context().discovery().reconnect();
+
+        assert latch.await(10, TimeUnit.SECONDS);
+
+        while (true) {
+            try {
+                UUID newId = client.localNode().id();
+
+                assert !clientId.equals(newId) : clientId;
+
+                break;
+            }
+            catch (IgniteClientDisconnectedException e) {
+                e.reconnectFuture().get(10_000);
+            }
+        }
+    }
+
     /**
      * @param ignite Ignite.
      * @throws Exception If failed.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index ea8e37bada1d0..67d88e1e29eac 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.IgniteClientReconnectServicesTest;
 import org.apache.ignite.internal.IgniteClientReconnectStopTest;
 import org.apache.ignite.internal.IgniteClientReconnectStreamerTest;
+import org.apache.ignite.internal.IgniteClientRejoinTest;
 
 /**
  *
@@ -52,6 +53,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteClientReconnectServicesTest.class);
         suite.addTestSuite(IgniteClientReconnectStreamerTest.class);
         suite.addTestSuite(IgniteClientReconnectFailoverTest.class);
+        suite.addTestSuite(IgniteClientRejoinTest.class);
 
         return suite;
     }

From c4de164392ddc114c88d5a6eba0ff0b13d32542f Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Mon, 20 Mar 2017 16:31:15 +0300
Subject: [PATCH 185/446] IGNITE-518: Unmuted tests that was fixed in
 ignite-4036. This closes #1636.

---
 .../expiry/IgniteCacheExpiryPolicyAbstractTest.java   | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index b234631e4002a..0b1b8c5fff3a4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -162,7 +162,8 @@ public void testCreateUpdate0() throws Exception {
         assertEquals(0, pSize);
     }
 
-    /**     * @throws Exception If failed.
+    /**
+     * @throws Exception If failed.
      */
     public void testZeroOnCreate() throws Exception {
         factory = CreatedExpiryPolicy.factoryOf(Duration.ZERO);
@@ -849,8 +850,6 @@ private void createUpdate(Integer key, @Nullable TransactionConcurrency txConcur
      * @throws Exception If failed.
      */
     public void testNearCreateUpdate() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-518");
-
         if (cacheMode() != PARTITIONED)
             return;
 
@@ -974,8 +973,6 @@ private void nearPutAll() throws Exception {
      * @throws Exception If failed.
      */
     public void testNearAccess() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-518");
-
         if (cacheMode() != PARTITIONED)
             return;
 
@@ -1023,10 +1020,10 @@ public void testNearAccess() throws Exception {
      * @throws Exception If failed.
      */
     public void testNearExpiresOnClient() throws Exception {
-        if(cacheMode() != PARTITIONED)
+        if (cacheMode() != PARTITIONED)
             return;
 
-        factory =  CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS,1));
+        factory = CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 1));
 
         nearCache = true;
 

From cad46799f34888381137e903899d73805ad11b13 Mon Sep 17 00:00:00 2001
From: Dmitriy Shabalin <dshabalin@gridgain.com>
Date: Thu, 2 Mar 2017 16:05:15 +0700
Subject: [PATCH 186/446] IGNITE-4659 Migration to Webpack 2. Upgrade template
 engine from jade to pug. (cherry picked from commit 1080e68)

---
 modules/web-console/frontend/.eslintrc        |   1 +
 modules/web-console/frontend/.gitignore       |   1 +
 .../web-console/frontend/app/app.config.js    |  12 +-
 modules/web-console/frontend/app/app.js       |   7 +-
 ...og.jade => activities-user-dialog.tpl.pug} |   0
 .../activities-user-dialog/index.js           |   2 +-
 ...epicker.jade => form-field-datepicker.pug} |   8 +-
 .../list-of-registered-users/index.js         |   2 +-
 .../list-of-registered-users.controller.js    |  11 +-
 ....jade => list-of-registered-users.tpl.pug} |   4 +-
 .../ui-grid-header/ui-grid-header.scss        |   1 +
 ...rid-header.jade => ui-grid-header.tpl.pug} |   0
 ...rid-settings.jade => ui-grid-settings.pug} |   0
 .../directives/centered/centered.directive.js |   2 +-
 .../centered/{centered.css => centered.scss}  |   0
 .../information/information.directive.js      |   4 +-
 .../{information.jade => information.pug}     |   0
 .../ui-ace-docker/ui-ace-docker.directive.js  |   4 +-
 .../{ui-ace-docker.jade => ui-ace-docker.pug} |   0
 .../ui-ace-java/ui-ace-java.directive.js      |   4 +-
 .../{ui-ace-java.jade => ui-ace-java.pug}     |   0
 .../ui-ace-pojos/ui-ace-pojos.directive.js    |   4 +-
 .../{ui-ace-pojos.jade => ui-ace-pojos.pug}   |   2 +-
 .../ui-ace-pom/ui-ace-pom.directive.js        |   4 +-
 .../{ui-ace-pom.jade => ui-ace-pom.pug}       |   0
 .../ui-ace-sharp/ui-ace-sharp.directive.js    |   4 +-
 .../{ui-ace-sharp.jade => ui-ace-sharp.pug}   |   0
 .../ui-ace-spring/ui-ace-spring.directive.js  |   4 +-
 .../{ui-ace-spring.jade => ui-ace-spring.pug} |   0
 .../frontend/app/helpers/jade/form.jade       |  28 --
 .../frontend/app/helpers/jade/form.pug        |  28 ++
 ...-checkbox.jade => form-field-checkbox.pug} |  10 +-
 ...-datalist.jade => form-field-datalist.pug} |  10 +-
 ...rm-field-down.jade => form-field-down.pug} |   2 +-
 ...-dropdown.jade => form-field-dropdown.pug} |  12 +-
 ...-feedback.jade => form-field-feedback.pug} |  10 +-
 ...-field-label.jade => form-field-label.pug} |   6 +-
 ...ield-number.jade => form-field-number.pug} |   8 +-
 ...-password.jade => form-field-password.pug} |   8 +-
 ...rm-field-text.jade => form-field-text.pug} |  16 +-
 .../{form-field-up.jade => form-field-up.pug} |   2 +-
 .../form/{form-group.jade => form-group.pug}  |   0
 .../helpers/jade/{mixins.jade => mixins.pug}  |  68 ++--
 .../app/modules/agent/agent.module.js         |   4 +-
 .../modules/branding/header-logo.directive.js |   4 +-
 .../{header-logo.jade => header-logo.pug}     |   0
 .../branding/powered-by-apache.directive.js   |   4 +-
 ...d-by-apache.jade => powered-by-apache.pug} |   0
 .../frontend/app/modules/demo/Demo.module.js  |   3 +-
 .../app/modules/dialog/dialog.factory.js      |   2 +-
 .../dialog/{dialog.jade => dialog.tpl.pug}    |   0
 .../GettingStarted.provider.js                |   3 +-
 .../app/modules/loading/loading.directive.js  |   8 +-
 .../loading/{loading.jade => loading.pug}     |   0
 .../loading/{loading.css => loading.scss}     |   0
 .../app/modules/nodes/Nodes.service.js        |   4 +-
 ...nodes-dialog.jade => nodes-dialog.tpl.pug} |   0
 .../app/modules/sql/notebook.controller.js    |   4 +-
 .../app/modules/sql/sql.controller.js         |  15 +-
 .../frontend/app/modules/sql/sql.module.js    |  15 +-
 .../app/modules/states/admin.state.js         |   4 +-
 .../app/modules/states/configuration.state.js |  23 +-
 .../caches/{affinity.jade => affinity.pug}    |  24 +-
 ...-near-cache.jade => client-near-cache.pug} |  14 +-
 .../{concurrency.jade => concurrency.pug}     |  18 +-
 .../caches/{general.jade => general.pug}      |  24 +-
 .../caches/{memory.jade => memory.pug}        |  27 +-
 ...ache-client.jade => near-cache-client.pug} |  18 +-
 ...ache-server.jade => near-cache-server.pug} |  18 +-
 .../{node-filter.jade => node-filter.pug}     |  24 +-
 .../caches/{query.jade => query.pug}          |  40 +--
 .../caches/{rebalance.jade => rebalance.pug}  |  24 +-
 .../{statistics.jade => statistics.pug}       |  12 +-
 .../caches/{store.jade => store.pug}          | 104 +++----
 .../clusters/{atomic.jade => atomic.pug}      |  14 +-
 .../{attributes.jade => attributes.pug}       |  16 +-
 .../clusters/{binary.jade => binary.pug}      |  14 +-
 .../{cache-key-cfg.jade => cache-key-cfg.pug} |  14 +-
 .../{checkpoint.jade => checkpoint.pug}       |  20 +-
 .../clusters/checkpoint/{fs.jade => fs.pug}   |  14 +-
 .../checkpoint/{jdbc.jade => jdbc.pug}        |   2 +-
 .../clusters/checkpoint/{s3.jade => s3.pug}   |   2 +-
 .../{collision.jade => collision.pug}         |  26 +-
 .../collision/{custom.jade => custom.pug}     |   4 +-
 .../{fifo-queue.jade => fifo-queue.pug}       |   6 +-
 .../{job-stealing.jade => job-stealing.pug}   |  22 +-
 ...priority-queue.jade => priority-queue.pug} |  16 +-
 .../{communication.jade => communication.pug} |  58 ++--
 .../{connector.jade => connector.pug}         |  46 +--
 .../{deployment.jade => deployment.pug}       |  72 ++---
 .../{discovery.jade => discovery.pug}         |  54 ++--
 .../clusters/{events.jade => events.pug}      |  18 +-
 .../clusters/{failover.jade => failover.pug}  |  12 +-
 .../clusters/{general.jade => general.pug}    |  42 +--
 .../discovery/{cloud.jade => cloud.pug}       |  48 +--
 .../discovery/{google.jade => google.pug}     |  12 +-
 .../general/discovery/{jdbc.jade => jdbc.pug} |   8 +-
 .../{multicast.jade => multicast.pug}         |  30 +-
 .../general/discovery/{s3.jade => s3.pug}     |   6 +-
 .../discovery/{shared.jade => shared.pug}     |   4 +-
 .../general/discovery/{vm.jade => vm.pug}     |  20 +-
 .../{zookeeper.jade => zookeeper.pug}         |  48 +--
 ...f.jade => bounded-exponential-backoff.pug} |   8 +-
 .../retrypolicy/{custom.jade => custom.pug}   |   6 +-
 ...l-backoff.jade => exponential-backoff.pug} |   8 +-
 .../retrypolicy/{forever.jade => forever.pug} |   4 +-
 .../retrypolicy/{n-times.jade => n-times.pug} |   6 +-
 .../{one-time.jade => one-time.pug}           |   4 +-
 .../{until-elapsed.jade => until-elapsed.pug} |   6 +-
 .../clusters/{igfs.jade => igfs.pug}          |  10 +-
 ...load-balancing.jade => load-balancing.pug} |  12 +-
 .../clusters/{logger.jade => logger.pug}      |  22 +-
 .../logger/{custom.jade => custom.pug}        |   4 +-
 .../clusters/logger/{log4j.jade => log4j.pug} |   8 +-
 .../logger/{log4j2.jade => log4j2.pug}        |   6 +-
 .../{marshaller.jade => marshaller.pug}       |  26 +-
 .../clusters/{metrics.jade => metrics.pug}    |  16 +-
 .../clusters/{odbc.jade => odbc.pug}          |  12 +-
 .../clusters/{ssl.jade => ssl.pug}            |  44 +--
 .../clusters/{swap.jade => swap.pug}          |  28 +-
 .../clusters/{thread.jade => thread.pug}      |  18 +-
 .../clusters/{time.jade => time.pug}          |  16 +-
 .../{transactions.jade => transactions.pug}   |  20 +-
 .../domains/{general.jade => general.pug}     |  14 +-
 .../domains/{query.jade => query.pug}         |  60 ++--
 .../domains/{store.jade => store.pug}         |  60 ++--
 .../igfs/{dual.jade => dual.pug}              |  14 +-
 .../{fragmentizer.jade => fragmentizer.pug}   |  16 +-
 .../igfs/{general.jade => general.pug}        |   8 +-
 .../configuration/igfs/{ipc.jade => ipc.pug}  |  24 +-
 .../igfs/{misc.jade => misc.pug}              |  56 ++--
 .../igfs/{secondary.jade => secondary.pug}    |  18 +-
 .../summary/summary.controller.js             |   5 +
 .../app/modules/states/errors.state.js        |   4 +-
 .../app/modules/states/password.state.js      |   6 +-
 .../app/modules/states/profile.state.js       |   4 +-
 .../app/modules/states/signin.state.js        |   2 +-
 .../frontend/app/services/Clone.service.js    |   4 +-
 .../frontend/app/services/Confirm.service.js  |   4 +-
 .../app/services/ConfirmBatch.service.js      |   4 +-
 .../frontend/controllers/caches-controller.js |   4 +-
 .../controllers/domains-controller.js         |   4 +-
 .../frontend/gulpfile.babel.js/paths.js       |  28 +-
 .../frontend/gulpfile.babel.js/tasks/build.js |   2 +-
 .../gulpfile.babel.js/tasks/bundle.js         |   8 +-
 .../frontend/gulpfile.babel.js/tasks/watch.js |   5 +-
 .../gulpfile.babel.js/webpack/common.js       | 292 +++++++++---------
 .../webpack/environments/development.js       |  90 +++---
 .../webpack/environments/production.js        |  41 ++-
 .../gulpfile.babel.js/webpack/index.js        |   4 +-
 modules/web-console/frontend/package.json     |  60 ++--
 .../frontend/test/e2e/exampe.test.js          |   4 +-
 .../frontend/test/karma.conf.babel.js         |   7 +-
 .../frontend/test/protractor.conf.js          |  16 +-
 .../test/unit/JavaTransformer.test.js         |   2 +-
 .../frontend/test/unit/JavaTypes.test.js      | 137 ++++----
 .../test/unit/SharpTransformer.test.js        |   2 +-
 .../test/unit/SpringTransformer.test.js       |   2 +-
 .../frontend/test/unit/SqlTypes.test.js       |   7 +-
 .../frontend/test/unit/UserAuth.test.js       |   4 +-
 .../frontend/test/unit/Version.test.js        |   1 +
 .../test/unit/defaultName.filter.test.js      |  27 +-
 .../frontend/views/{403.jade => 403.pug}      |   0
 .../frontend/views/{404.jade => 404.pug}      |   0
 .../frontend/views/{base.jade => base.pug}    |   4 +-
 .../{caches.jade => caches.tpl.pug}           |  24 +-
 .../{clusters.jade => clusters.tpl.pug}       |  50 +--
 ...ins-import.jade => domains-import.tpl.pug} |   4 +-
 .../{domains.jade => domains.tpl.pug}         |   8 +-
 .../configuration/{igfs.jade => igfs.tpl.pug} |  14 +-
 .../{sidebar.jade => sidebar.tpl.pug}         |   0
 ...jade => summary-project-structure.tpl.pug} |   1 +
 .../{summary-tabs.jade => summary-tabs.pug}   |   2 +-
 .../{summary.jade => summary.tpl.pug}         |   8 +-
 .../includes/{footer.jade => footer.pug}      |   0
 .../includes/{header.jade => header.pug}      |   0
 .../frontend/views/{index.jade => index.pug}  |   0
 .../views/{reset.jade => reset.tpl.pug}       |   0
 .../settings/{admin.jade => admin.tpl.pug}    |   0
 .../{profile.jade => profile.tpl.pug}         |   0
 .../views/{signin.jade => signin.tpl.pug}     |   0
 ...e-metadata.jade => cache-metadata.tpl.pug} |   0
 ...t-settings.jade => chart-settings.tpl.pug} |   0
 ...notebook-new.jade => notebook-new.tpl.pug} |   0
 ...graph-rate.jade => paragraph-rate.tpl.pug} |   0
 .../views/sql/{sql.jade => sql.tpl.pug}       |  18 +-
 ...t-download.jade => agent-download.tpl.pug} |   0
 .../templates/{alert.jade => alert.tpl.pug}   |   0
 ...tch-confirm.jade => batch-confirm.tpl.pug} |   0
 .../templates/{clone.jade => clone.tpl.pug}   |   2 +-
 .../{confirm.jade => confirm.tpl.pug}         |   0
 .../{demo-info.jade => demo-info.tpl.pug}     |   0
 .../{dropdown.jade => dropdown.tpl.pug}       |   0
 ...g-started.jade => getting-started.tpl.pug} |   0
 .../{message.jade => message.tpl.pug}         |   0
 .../frontend/views/templates/pagination.jade  |  32 --
 .../templates/{select.jade => select.tpl.pug} |   0
 ...on-error.jade => validation-error.tpl.pug} |   0
 198 files changed, 1442 insertions(+), 1436 deletions(-)
 rename modules/web-console/frontend/app/components/activities-user-dialog/{activities-user-dialog.jade => activities-user-dialog.tpl.pug} (100%)
 rename modules/web-console/frontend/app/components/form-field-datepicker/{form-field-datepicker.jade => form-field-datepicker.pug} (90%)
 rename modules/web-console/frontend/app/components/list-of-registered-users/{list-of-registered-users.jade => list-of-registered-users.tpl.pug} (98%)
 rename modules/web-console/frontend/app/components/ui-grid-header/{ui-grid-header.jade => ui-grid-header.tpl.pug} (100%)
 rename modules/web-console/frontend/app/components/ui-grid-settings/{ui-grid-settings.jade => ui-grid-settings.pug} (100%)
 rename modules/web-console/frontend/app/directives/centered/{centered.css => centered.scss} (100%)
 rename modules/web-console/frontend/app/directives/information/{information.jade => information.pug} (100%)
 rename modules/web-console/frontend/app/directives/ui-ace-docker/{ui-ace-docker.jade => ui-ace-docker.pug} (100%)
 rename modules/web-console/frontend/app/directives/ui-ace-java/{ui-ace-java.jade => ui-ace-java.pug} (100%)
 rename modules/web-console/frontend/app/directives/ui-ace-pojos/{ui-ace-pojos.jade => ui-ace-pojos.pug} (97%)
 rename modules/web-console/frontend/app/directives/ui-ace-pom/{ui-ace-pom.jade => ui-ace-pom.pug} (100%)
 rename modules/web-console/frontend/app/directives/ui-ace-sharp/{ui-ace-sharp.jade => ui-ace-sharp.pug} (100%)
 rename modules/web-console/frontend/app/directives/ui-ace-spring/{ui-ace-spring.jade => ui-ace-spring.pug} (100%)
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form.jade
 create mode 100644 modules/web-console/frontend/app/helpers/jade/form.pug
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-checkbox.jade => form-field-checkbox.pug} (83%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-datalist.jade => form-field-datalist.pug} (86%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-down.jade => form-field-down.pug} (95%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-dropdown.jade => form-field-dropdown.pug} (80%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-feedback.jade => form-field-feedback.pug} (83%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-label.jade => form-field-label.pug} (88%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-number.jade => form-field-number.pug} (91%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-password.jade => form-field-password.pug} (91%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-text.jade => form-field-text.pug} (86%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-up.jade => form-field-up.pug} (95%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-group.jade => form-group.pug} (100%)
 rename modules/web-console/frontend/app/helpers/jade/{mixins.jade => mixins.pug} (91%)
 rename modules/web-console/frontend/app/modules/branding/{header-logo.jade => header-logo.pug} (100%)
 rename modules/web-console/frontend/app/modules/branding/{powered-by-apache.jade => powered-by-apache.pug} (100%)
 rename modules/web-console/frontend/app/modules/dialog/{dialog.jade => dialog.tpl.pug} (100%)
 rename modules/web-console/frontend/app/modules/loading/{loading.jade => loading.pug} (100%)
 rename modules/web-console/frontend/app/modules/loading/{loading.css => loading.scss} (100%)
 rename modules/web-console/frontend/app/modules/nodes/{nodes-dialog.jade => nodes-dialog.tpl.pug} (100%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{affinity.jade => affinity.pug} (77%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{client-near-cache.jade => client-near-cache.pug} (81%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{concurrency.jade => concurrency.pug} (79%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{general.jade => general.pug} (77%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{memory.jade => memory.pug} (86%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{near-cache-client.jade => near-cache-client.pug} (76%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{near-cache-server.jade => near-cache-server.pug} (77%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{node-filter.jade => node-filter.pug} (71%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{query.jade => query.pug} (76%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{rebalance.jade => rebalance.pug} (75%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{statistics.jade => statistics.pug} (76%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{store.jade => store.pug} (72%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{atomic.jade => atomic.pug} (83%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{attributes.jade => attributes.pug} (80%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{binary.jade => binary.pug} (92%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{cache-key-cfg.jade => cache-key-cfg.pug} (83%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{checkpoint.jade => checkpoint.pug} (86%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/{fs.jade => fs.pug} (85%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/{jdbc.jade => jdbc.pug} (98%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/{s3.jade => s3.pug} (99%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{collision.jade => collision.pug} (75%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/collision/{custom.jade => custom.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/collision/{fifo-queue.jade => fifo-queue.pug} (78%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/collision/{job-stealing.jade => job-stealing.pug} (73%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/collision/{priority-queue.jade => priority-queue.pug} (64%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{communication.jade => communication.pug} (56%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{connector.jade => connector.pug} (71%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{deployment.jade => deployment.pug} (78%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{discovery.jade => discovery.pug} (50%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{events.jade => events.pug} (78%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{failover.jade => failover.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{general.jade => general.pug} (71%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{cloud.jade => cloud.pug} (78%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{google.jade => google.pug} (68%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{jdbc.jade => jdbc.pug} (82%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{multicast.jade => multicast.pug} (79%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{s3.jade => s3.pug} (79%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{shared.jade => shared.pug} (89%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{vm.jade => vm.pug} (83%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{zookeeper.jade => zookeeper.pug} (70%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{bounded-exponential-backoff.jade => bounded-exponential-backoff.pug} (84%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{custom.jade => custom.pug} (89%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{exponential-backoff.jade => exponential-backoff.pug} (83%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{forever.jade => forever.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{n-times.jade => n-times.pug} (86%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{one-time.jade => one-time.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{until-elapsed.jade => until-elapsed.pug} (86%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{igfs.jade => igfs.pug} (84%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{load-balancing.jade => load-balancing.pug} (94%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{logger.jade => logger.pug} (80%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/logger/{custom.jade => custom.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/logger/{log4j.jade => log4j.pug} (87%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/logger/{log4j2.jade => log4j2.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{marshaller.jade => marshaller.pug} (76%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{metrics.jade => metrics.pug} (73%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{odbc.jade => odbc.pug} (83%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{ssl.jade => ssl.pug} (73%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{swap.jade => swap.pug} (73%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{thread.jade => thread.pug} (68%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{time.jade => time.pug} (73%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{transactions.jade => transactions.pug} (78%)
 rename modules/web-console/frontend/app/modules/states/configuration/domains/{general.jade => general.pug} (79%)
 rename modules/web-console/frontend/app/modules/states/configuration/domains/{query.jade => query.pug} (77%)
 rename modules/web-console/frontend/app/modules/states/configuration/domains/{store.jade => store.pug} (74%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{dual.jade => dual.pug} (75%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{fragmentizer.jade => fragmentizer.pug} (70%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{general.jade => general.pug} (89%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{ipc.jade => ipc.pug} (66%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{misc.jade => misc.pug} (66%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{secondary.jade => secondary.pug} (69%)
 rename modules/web-console/frontend/views/{403.jade => 403.pug} (100%)
 rename modules/web-console/frontend/views/{404.jade => 404.pug} (100%)
 rename modules/web-console/frontend/views/{base.jade => base.pug} (94%)
 rename modules/web-console/frontend/views/configuration/{caches.jade => caches.tpl.pug} (89%)
 rename modules/web-console/frontend/views/configuration/{clusters.jade => clusters.tpl.pug} (83%)
 rename modules/web-console/frontend/views/configuration/{domains-import.jade => domains-import.tpl.pug} (99%)
 rename modules/web-console/frontend/views/configuration/{domains.jade => domains.tpl.pug} (97%)
 rename modules/web-console/frontend/views/configuration/{igfs.jade => igfs.tpl.pug} (94%)
 rename modules/web-console/frontend/views/configuration/{sidebar.jade => sidebar.tpl.pug} (100%)
 rename modules/web-console/frontend/views/configuration/{summary-project-structure.jade => summary-project-structure.tpl.pug} (99%)
 rename modules/web-console/frontend/views/configuration/{summary-tabs.jade => summary-tabs.pug} (97%)
 rename modules/web-console/frontend/views/configuration/{summary.jade => summary.tpl.pug} (93%)
 rename modules/web-console/frontend/views/includes/{footer.jade => footer.pug} (100%)
 rename modules/web-console/frontend/views/includes/{header.jade => header.pug} (100%)
 rename modules/web-console/frontend/views/{index.jade => index.pug} (100%)
 rename modules/web-console/frontend/views/{reset.jade => reset.tpl.pug} (100%)
 rename modules/web-console/frontend/views/settings/{admin.jade => admin.tpl.pug} (100%)
 rename modules/web-console/frontend/views/settings/{profile.jade => profile.tpl.pug} (100%)
 rename modules/web-console/frontend/views/{signin.jade => signin.tpl.pug} (100%)
 rename modules/web-console/frontend/views/sql/{cache-metadata.jade => cache-metadata.tpl.pug} (100%)
 rename modules/web-console/frontend/views/sql/{chart-settings.jade => chart-settings.tpl.pug} (100%)
 rename modules/web-console/frontend/views/sql/{notebook-new.jade => notebook-new.tpl.pug} (100%)
 rename modules/web-console/frontend/views/sql/{paragraph-rate.jade => paragraph-rate.tpl.pug} (100%)
 rename modules/web-console/frontend/views/sql/{sql.jade => sql.tpl.pug} (95%)
 rename modules/web-console/frontend/views/templates/{agent-download.jade => agent-download.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{alert.jade => alert.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{batch-confirm.jade => batch-confirm.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{clone.jade => clone.tpl.pug} (98%)
 rename modules/web-console/frontend/views/templates/{confirm.jade => confirm.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{demo-info.jade => demo-info.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{dropdown.jade => dropdown.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{getting-started.jade => getting-started.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{message.jade => message.tpl.pug} (100%)
 delete mode 100644 modules/web-console/frontend/views/templates/pagination.jade
 rename modules/web-console/frontend/views/templates/{select.jade => select.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{validation-error.jade => validation-error.tpl.pug} (100%)

diff --git a/modules/web-console/frontend/.eslintrc b/modules/web-console/frontend/.eslintrc
index 988cfa07c0888..958c6d11ef40f 100644
--- a/modules/web-console/frontend/.eslintrc
+++ b/modules/web-console/frontend/.eslintrc
@@ -32,6 +32,7 @@ globals:
     $generatorOptional: true
     saveAs: true
     process: true
+    require: true
 
 rules:
     arrow-parens: [1, "always"]
diff --git a/modules/web-console/frontend/.gitignore b/modules/web-console/frontend/.gitignore
index 46bca1383a528..27138f6a6875c 100644
--- a/modules/web-console/frontend/.gitignore
+++ b/modules/web-console/frontend/.gitignore
@@ -1,5 +1,6 @@
 *.idea
 *.log
+*.log.*
 .npmrc
 build/*
 node_modules
diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js
index 39d761f418eef..3ca5c3b8eaef2 100644
--- a/modules/web-console/frontend/app/app.config.js
+++ b/modules/web-console/frontend/app/app.config.js
@@ -16,6 +16,7 @@
  */
 
 import _ from 'lodash';
+import angular from 'angular';
 
 const nonNil = _.negate(_.isNil);
 const nonEmpty = _.negate(_.isEmpty);
@@ -25,7 +26,10 @@ _.mixin({
     nonEmpty
 });
 
-import alertTemplateUrl from '../views/templates/alert.jade';
+import alertTemplateUrl from 'views/templates/alert.tpl.pug';
+import selectTemplateUrl from 'views/templates/select.tpl.pug';
+import dropdownTemplateUrl from 'views/templates/dropdown.tpl.pug';
+import validationTemplateUrl from 'views/templates/validation-error.tpl.pug';
 
 const igniteConsoleCfg = angular.module('ignite-console.config', ['ngAnimate', 'mgcrea.ngStrap']);
 
@@ -48,7 +52,7 @@ igniteConsoleCfg.config(['$popoverProvider', ($popoverProvider) => {
         trigger: 'manual',
         placement: 'right',
         container: 'body',
-        templateUrl: '/templates/validation-error.html'
+        templateUrl: validationTemplateUrl
     });
 }]);
 
@@ -70,7 +74,7 @@ igniteConsoleCfg.config(['$selectProvider', ($selectProvider) => {
         maxLength: '5',
         allText: 'Select All',
         noneText: 'Clear All',
-        templateUrl: '/templates/select.html',
+        templateUrl: selectTemplateUrl,
         iconCheckmark: 'fa fa-check',
         caretHtml: ''
     });
@@ -91,7 +95,7 @@ igniteConsoleCfg.config(['$alertProvider', ($alertProvider) => {
 // AngularStrap dropdowns () configuration.
 igniteConsoleCfg.config(['$dropdownProvider', ($dropdownProvider) => {
     angular.extend($dropdownProvider.defaults, {
-        templateUrl: 'templates/dropdown.html'
+        templateUrl: dropdownTemplateUrl
     });
 }]);
 
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 9958cb53ae539..5e3bb07434b2c 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -19,7 +19,6 @@ import '../public/stylesheets/style.scss';
 import '../app/components/ui-grid-header/ui-grid-header.scss';
 import '../app/components/ui-grid-settings/ui-grid-settings.scss';
 import '../app/components/form-field-datepicker/form-field-datepicker.scss';
-import './helpers/jade/mixins.jade';
 
 import './app.config';
 
@@ -122,7 +121,7 @@ import IgniteActivitiesUserDialog from './components/activities-user-dialog';
 // Inject external modules.
 import 'ignite_modules_temp/index';
 
-import baseTemplate from '../views/base.jade';
+import baseTemplate from 'views/base.pug';
 
 angular
 .module('ignite-console', [
@@ -242,12 +241,12 @@ angular
         .state('base', {
             url: '',
             abstract: true,
-            templateUrl: baseTemplate
+            template: baseTemplate
         })
         .state('settings', {
             url: '/settings',
             abstract: true,
-            templateUrl: baseTemplate
+            template: baseTemplate
         });
 
     $urlRouterProvider.otherwise('/404');
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
rename to modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/index.js b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
index dca6ba92ede6d..2f8fdeff00473 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/index.js
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
@@ -16,7 +16,7 @@
  */
 
  import controller from './activities-user-dialog.controller';
- import templateUrl from './activities-user-dialog.jade';
+ import templateUrl from './activities-user-dialog.tpl.pug';
 
  export default ['$modal', ($modal) => ({ show = true, user }) => {
      const ActivitiesUserDialog = $modal({
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
similarity index 90%
rename from modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
rename to modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
index 2578cf4b17189..c9d382cb36cc4 100644
--- a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
+++ b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
@@ -17,15 +17,15 @@
 mixin ignite-form-field-datepicker(label, model, name, disabled, required, placeholder, tip)
     mixin form-field-input()
         input.form-control(
-            id='{{ #{name} }}Input'
-            name='{{ #{name} }}'
+            id=`{{ ${name} }}Input`
+            name=`{{ ${name} }}`
 
             placeholder=placeholder
             
             data-ng-model=model
 
-            data-ng-required=required && '#{required}'
-            data-ng-disabled=disabled && '#{disabled}'
+            data-ng-required=required && `${required}`
+            data-ng-disabled=disabled && `${disabled}`
 
             bs-datepicker
             data-date-format='MMM yyyy'
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/index.js b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
index 32a34f41c02d6..22a89da3b6a24 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/index.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './list-of-registered-users.jade';
+import templateUrl from './list-of-registered-users.tpl.pug';
 import controller from './list-of-registered-users.controller';
 
 export default [() => {
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 272681a5f576b..33fe819068c78 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import headerTemplate from 'app/components/ui-grid-header/ui-grid-header.jade';
+import headerTemplate from 'app/components/ui-grid-header/ui-grid-header.tpl.pug';
 
 import columnDefs from './list-of-registered-users.column-defs';
 import categories from './list-of-registered-users.categories';
@@ -30,9 +30,9 @@ const rowTemplate = `<div
   ui-grid-cell/>`;
 
 export default class IgniteListOfRegisteredUsersCtrl {
-    static $inject = ['$scope', '$state', '$filter', '$templateCache', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
+    static $inject = ['$scope', '$state', '$filter', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
 
-    constructor($scope, $state, $filter, $templateCache, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
+    constructor($scope, $state, $filter, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
         const $ctrl = this;
 
         const companySelectOptions = [];
@@ -113,7 +113,7 @@ export default class IgniteListOfRegisteredUsersCtrl {
             columnVirtualizationThreshold: 30,
             columnDefs,
             categories,
-            headerTemplate: $templateCache.get(headerTemplate),
+            headerTemplate,
             rowTemplate,
             enableFiltering: true,
             enableRowSelection: true,
@@ -157,6 +157,9 @@ export default class IgniteListOfRegisteredUsersCtrl {
             AdminData.loadUsers(params)
                 .then((data) => $ctrl.gridOptions.data = data)
                 .then((data) => {
+                    companySelectOptions.length = 0;
+                    countrySelectOptions.length = 0;
+
                     companySelectOptions.push(...usersToFilterOptions('company'));
                     countrySelectOptions.push(...usersToFilterOptions('countryCode'));
 
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
similarity index 98%
rename from modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
rename to modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
index 119591036d9da..52975b98dc0ee 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
@@ -14,8 +14,8 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
-include /app/components/form-field-datepicker/form-field-datepicker.jade
+include /app/helpers/jade/mixins
+include /app/components/form-field-datepicker/form-field-datepicker.pug
 
 mixin grid-settings()
     i.fa.fa-bars(data-animation='am-flip-x' bs-dropdown='' aria-haspopup='true' aria-expanded='expanded' data-auto-close='1' data-trigger='click')
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
index c39050469db9c..c6e7bdf241e98 100644
--- a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
@@ -50,6 +50,7 @@
     .ui-grid-header-span {
         position: relative;
         border-right: 0;
+        background: #f5f5f5;
 
         .ng-hide + .ui-grid-header-cell-row .ui-grid-header-cell {
             height: 58px;
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
rename to modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug
diff --git a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.jade b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.pug
similarity index 100%
rename from modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.jade
rename to modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.pug
diff --git a/modules/web-console/frontend/app/directives/centered/centered.directive.js b/modules/web-console/frontend/app/directives/centered/centered.directive.js
index 4abd086e6a5cb..77bbb940fe937 100644
--- a/modules/web-console/frontend/app/directives/centered/centered.directive.js
+++ b/modules/web-console/frontend/app/directives/centered/centered.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import './centered.css';
+import './centered.scss';
 
 export default ['centered', [() => {
     return {
diff --git a/modules/web-console/frontend/app/directives/centered/centered.css b/modules/web-console/frontend/app/directives/centered/centered.scss
similarity index 100%
rename from modules/web-console/frontend/app/directives/centered/centered.css
rename to modules/web-console/frontend/app/directives/centered/centered.scss
diff --git a/modules/web-console/frontend/app/directives/information/information.directive.js b/modules/web-console/frontend/app/directives/information/information.directive.js
index a9a2f8c27fde9..6f304ef8a9b0b 100644
--- a/modules/web-console/frontend/app/directives/information/information.directive.js
+++ b/modules/web-console/frontend/app/directives/information/information.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './information.jade';
+import template from './information.pug';
 
 export default ['igniteInformation', [() => {
     return {
@@ -23,7 +23,7 @@ export default ['igniteInformation', [() => {
             title: '@'
         },
         restrict: 'E',
-        templateUrl,
+        template,
         replace: true,
         transclude: true
     };
diff --git a/modules/web-console/frontend/app/directives/information/information.jade b/modules/web-console/frontend/app/directives/information/information.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/information/information.jade
rename to modules/web-console/frontend/app/directives/information/information.pug
diff --git a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js
index 08e4f76e3bbea..9042acb767f53 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './ui-ace-docker.jade';
+import template from './ui-ace-docker.pug';
 import controller from './ui-ace-docker.controller';
 
 export default ['igniteUiAceDocker', [() => {
@@ -38,7 +38,7 @@ export default ['igniteUiAceDocker', [() => {
             data: '=ngModel'
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['?^igniteUiAceTabs']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.jade b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.jade
rename to modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.pug
diff --git a/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js b/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js
index c21bff7279b59..5f43b2351e1f4 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './ui-ace-java.jade';
+import template from './ui-ace-java.pug';
 import controller from './ui-ace-java.controller';
 
 export default ['igniteUiAceJava', [() => {
@@ -55,7 +55,7 @@ export default ['igniteUiAceJava', [() => {
             client: '@'
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['igniteUiAceJava', '?^igniteUiAceTabs', '?^form', '?ngModel']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.jade b/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.jade
rename to modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.pug
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js
index 7c224b79eb39e..8a8d047de561c 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './ui-ace-pojos.jade';
+import template from './ui-ace-pojos.pug';
 import controller from './ui-ace-pojos.controller';
 
 export default ['igniteUiAcePojos', [() => {
@@ -38,7 +38,7 @@ export default ['igniteUiAcePojos', [() => {
             pojos: '=ngModel'
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['?^igniteUiAceTabs']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.jade b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.pug
similarity index 97%
rename from modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.jade
rename to modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.pug
index 581b8c1451601..4bee5e5f43035 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.jade
+++ b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.pug
@@ -15,7 +15,7 @@
     limitations under the License.
 
 mixin check-tooltip(message)
-    i.tipLabel.icon-help(bs-tooltip='"#{message}"')
+    i.tipLabel.icon-help(bs-tooltip=`"${message}"`)
 
 .panel-details-noborder
     .details-row
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js
index 2a7a87890ace9..664d3a000fd4a 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './ui-ace-pom.jade';
+import template from './ui-ace-pom.pug';
 import controller from './ui-ace-pom.controller';
 
 export default ['igniteUiAcePom', [() => {
@@ -33,7 +33,7 @@ export default ['igniteUiAcePom', [() => {
             cluster: '='
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['?^igniteUiAceTabs']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.jade b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.jade
rename to modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.pug
diff --git a/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js b/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js
index 5d9ad7988b23f..5a37b80742ea8 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './ui-ace-sharp.jade';
+import template from './ui-ace-sharp.pug';
 import controller from './ui-ace-sharp.controller';
 
 export default ['igniteUiAceSharp', ['IgniteSharpTransformer', (generator) => {
@@ -125,7 +125,7 @@ export default ['igniteUiAceSharp', ['IgniteSharpTransformer', (generator) => {
             data: '=?ngModel'
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['igniteUiAceSharp', '?^igniteUiAceTabs', '?^form', '?ngModel']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.jade b/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.jade
rename to modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.pug
diff --git a/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js b/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js
index 42d25b65f3993..de3f434dddca2 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js
@@ -17,7 +17,7 @@
 
 import _ from 'lodash';
 
-import templateUrl from './ui-ace-spring.jade';
+import template from './ui-ace-spring.pug';
 import controller from './ui-ace-spring.controller';
 
 export default ['igniteUiAceSpring', [() => {
@@ -58,7 +58,7 @@ export default ['igniteUiAceSpring', [() => {
             client: '@'
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['igniteUiAceSpring', '?^igniteUiAceTabs', '?^form', '?ngModel']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.jade b/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.jade
rename to modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.pug
diff --git a/modules/web-console/frontend/app/helpers/jade/form.jade b/modules/web-console/frontend/app/helpers/jade/form.jade
deleted file mode 100644
index c0c558d9cb925..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form.jade
+++ /dev/null
@@ -1,28 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-include ./form/form-field-feedback.jade
-include ./form/form-field-label.jade
-include ./form/form-field-text.jade
-include ./form/form-field-password.jade
-include ./form/form-field-dropdown.jade
-include ./form/form-field-datalist.jade
-include ./form/form-field-checkbox.jade
-include ./form/form-field-number.jade
-include ./form/form-field-up.jade
-include ./form/form-field-down.jade
-
-include ./form/form-group.jade
diff --git a/modules/web-console/frontend/app/helpers/jade/form.pug b/modules/web-console/frontend/app/helpers/jade/form.pug
new file mode 100644
index 0000000000000..6fddbf6b97687
--- /dev/null
+++ b/modules/web-console/frontend/app/helpers/jade/form.pug
@@ -0,0 +1,28 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include ./form/form-field-feedback
+include ./form/form-field-label
+include ./form/form-field-text
+include ./form/form-field-password
+include ./form/form-field-dropdown
+include ./form/form-field-datalist
+include ./form/form-field-checkbox
+include ./form/form-field-number
+include ./form/form-field-up
+include ./form/form-field-down
+
+include ./form/form-group
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug
similarity index 83%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug
index 222ecfee76317..68a2dd2ece56f 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug
@@ -16,19 +16,19 @@
 
 mixin form-field-checkbox(label, model, name, disabled, required, tip)
     div.checkbox.col-xs-12.col-sm-12.col-md-12
-        label(id='{{ #{name} }}Label')
+        label(id=`{{ ${name} }}Label`)
             .input-tip
                 if block
                     block
                 else
                     input(
-                        id='{{ #{name} }}Input'
-                        name='{{ #{name} }}'
+                        id=`{{ ${name} }}Input`
+                        name=`{{ ${name} }}`
                         type='checkbox'
 
                         data-ng-model=model
-                        data-ng-required=required && '#{required}'
-                        data-ng-disabled=disabled && '#{disabled}'
+                        data-ng-required=required && `${required}`
+                        data-ng-disabled=disabled && `${disabled}`
 
                         data-ng-focus='tableReset()'
 
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.pug
similarity index 86%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.pug
index 4c1970e14bfa0..e4b71afb4315b 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.pug
@@ -19,17 +19,17 @@ mixin form-field-datalist(label, model, name, disabled, required, placeholder, o
 
     mixin form-field-input()
         input.form-control(
-            id='{{ #{name} }}Input'
-            name='{{ #{name} }}'
+            id=`{{ ${name} }}Input`
+            name=`{{ ${name} }}`
             placeholder=placeholder
            
             data-ng-model=model
 
-            data-ng-required=required && '#{required}'
-            data-ng-disabled=disabled && '#{disabled}' || '!#{options}.length'
+            data-ng-required=required && `${required}`
+            data-ng-disabled=disabled && `${disabled}` || `!${options}.length`
 
             bs-typeahead
-            bs-options='item for item in #{options}'
+            bs-options=`item for item in ${options}`
             container='body'
             data-min-length='1'
             ignite-retain-selection
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-down.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-down.pug
similarity index 95%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-down.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-down.pug
index cd10ebee1f889..9361650fb0ba5 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-down.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-down.pug
@@ -15,4 +15,4 @@
     limitations under the License.
 
 mixin ignite-form-field-down()
-    i.tipField.fa.fa-arrow-down(ignite-form-field-down ng-click="$ctrl.down()")&attributes(attributes)
+    i.tipField.fa.fa-arrow-down(ignite-form-field-down ng-click='$ctrl.down()')&attributes(attributes)
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug
similarity index 80%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug
index 33af6d1d8785a..d7bd0fd2943ea 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug
@@ -17,18 +17,18 @@
 mixin ignite-form-field-dropdown(label, model, name, disabled, required, multiple, placeholder, placeholderEmpty, options, tip)
     mixin form-field-input()
         button.select-toggle.form-control(
-            id='{{ #{name} }}Input'
-            name='{{ #{name} }}'
+            id=`{{ ${name} }}Input`
+            name=`{{ ${name} }}`
 
-            data-placeholder=placeholderEmpty ? '{{ #{options}.length > 0 ? "#{placeholder}" : "#{placeholderEmpty}" }}' : placeholder
+            data-placeholder=placeholderEmpty ? `{{ ${options}.length > 0 ? '${placeholder}' : '${placeholderEmpty}' }}` : placeholder
             
             data-ng-model=model
 
-            data-ng-required=required && '#{required}'
-            data-ng-disabled=disabled && '#{disabled}' || '!#{options}.length'
+            data-ng-required=required && `${required}`
+            data-ng-disabled=disabled && `${disabled}` || `!${options}.length`
 
             bs-select
-            bs-options='item.value as item.label for item in #{options}'
+            bs-options=`item.value as item.label for item in ${options}`
 
             data-multiple=multiple ? '1' : false
             data-container='body > .wrapper'
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
similarity index 83%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
index ac1f0802683c4..c70e7a3b02e6b 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
@@ -15,13 +15,13 @@
     limitations under the License.
 
 mixin form-field-feedback(name, error, message)
-    -var __field = form + '[' + name + ']'
-    -var __error = __field + '.$error.' + error
-    -var __pristine = __field + '.$pristine'
+    -var __field = `${form}[${name}]`
+    -var __error = `${__field}.$error.${error}`
+    -var __pristine = `${__field}.$pristine`
 
     i.fa.fa-exclamation-triangle.form-field-feedback(
-        ng-if='!#{__pristine} && #{__error}'
-        name='{{ #{name} }}'
+        ng-if=`!${__pristine} && ${__error}`
+        name=`{{ ${name} }}`
 
         bs-tooltip
         data-title=message
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-label.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-label.pug
similarity index 88%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-label.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-label.pug
index d6aef8192cc4b..d0275c94ee010 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-label.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-label.pug
@@ -16,8 +16,8 @@
 
 mixin ignite-form-field__label(label, name, required)
     label.ignite-form-field__label(
-        id='{{ #{name} }}Label'
-        for='{{ #{name} }}Input'
-        class="{{ #{required} ? 'required' : '' }}"
+        id=`{{ ${name} }}Label`
+        for=`{{ ${name} }}Input`
+        class=`{{ ${required} ? 'required' : '' }}`
     )
         span !{label}
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-number.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-number.pug
similarity index 91%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-number.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-number.pug
index 58b0dcd654291..dd41849d1e7ce 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-number.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-number.pug
@@ -17,8 +17,8 @@
 mixin ignite-form-field-number(label, model, name, disabled, required, placeholder, min, max, step, tip)
     mixin form-field-input()
         input.form-control(
-            id='{{ #{name} }}Input'
-            name='{{ #{name} }}'
+            id=`{{ ${name} }}Input`
+            name=`{{ ${name} }}`
             placeholder=placeholder
             type='number'
 
@@ -28,8 +28,8 @@ mixin ignite-form-field-number(label, model, name, disabled, required, placehold
 
             data-ng-model=model
 
-            data-ng-required=required && '#{required}'
-            data-ng-disabled=disabled && '#{disabled}'
+            data-ng-required=required && `${required}`
+            data-ng-disabled=disabled && `${disabled}`
             data-ng-focus='tableReset()'
 
             data-ignite-form-panel-field=''
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-password.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
similarity index 91%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-password.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
index e5e7bc83f3114..a12982f4fb6e8 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-password.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
@@ -16,15 +16,15 @@
 
 mixin ignite-form-field-password-input(name, model, disabled, required, placeholder)
     input.form-control(
-        id='{{ #{name} }}Input'
-        name='{{ #{name} }}'
+        id=`{{ ${name} }}Input`
+        name=`{{ ${name} }}`
         placeholder=placeholder
         type='password'
 
         data-ng-model=model
 
-        data-ng-required=required && '#{required}'
-        data-ng-disabled=disabled && '#{disabled}'
+        data-ng-required=required && `${required}`
+        data-ng-disabled=disabled && `${disabled}`
         data-ng-focus='tableReset()'
 
         data-ignite-form-panel-field=''
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
similarity index 86%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-text.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
index 1f93d3b8ede50..76ea6e609d6e1 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
@@ -16,15 +16,15 @@
 
 mixin ignite-form-field-input(name, model, disabled, required, placeholder)
     input.form-control(
-        id='{{ #{name} }}Input'
-        name='{{ #{name} }}'
+        id=`{{ ${name} }}Input`
+        name=`{{ ${name} }}`
         placeholder=placeholder
         type='text'
 
         data-ng-model=model
 
-        data-ng-required=required && '#{required}'
-        data-ng-disabled=disabled && '#{disabled}'
+        data-ng-required=required && `${required}`
+        data-ng-disabled=disabled && `${disabled}`
         data-ng-focus='tableReset()'
 
         data-ignite-form-panel-field=''
@@ -32,15 +32,15 @@ mixin ignite-form-field-input(name, model, disabled, required, placeholder)
 
 mixin ignite-form-field-url-input(name, model, disabled, required, placeholder)
     input.form-control(
-        id='{{ #{name} }}Input'
-        name='{{ #{name} }}'
+        id=`{{ ${name} }}Input`
+        name=`{{ ${name} }}`
         placeholder=placeholder
         type='url'
 
         data-ng-model=model
 
-        data-ng-required=required && '#{required}'
-        data-ng-disabled=disabled && '#{disabled}'
+        data-ng-required=required && `${required}`
+        data-ng-disabled=disabled && `${disabled}`
         data-ng-focus='tableReset()'
 
         data-ignite-form-panel-field=''
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-up.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-up.pug
similarity index 95%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-up.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-up.pug
index c66cd0e6021c4..5b6520166e111 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-up.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-up.pug
@@ -15,4 +15,4 @@
     limitations under the License.
 
 mixin ignite-form-field-up()
-    i.tipField.fa.fa-arrow-up.ng-scope(ignite-form-field-up ng-click="$ctrl.up()")&attributes(attributes)
+    i.tipField.fa.fa-arrow-up.ng-scope(ignite-form-field-up ng-click='$ctrl.up()')&attributes(attributes)
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-group.jade b/modules/web-console/frontend/app/helpers/jade/form/form-group.pug
similarity index 100%
rename from modules/web-console/frontend/app/helpers/jade/form/form-group.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-group.pug
diff --git a/modules/web-console/frontend/app/helpers/jade/mixins.jade b/modules/web-console/frontend/app/helpers/jade/mixins.pug
similarity index 91%
rename from modules/web-console/frontend/app/helpers/jade/mixins.jade
rename to modules/web-console/frontend/app/helpers/jade/mixins.pug
index 6ca41f645c68c..db175a2146ef1 100644
--- a/modules/web-console/frontend/app/helpers/jade/mixins.jade
+++ b/modules/web-console/frontend/app/helpers/jade/mixins.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include ./form.jade
+include ./form
 
 //- Mixin for advanced options toggle.
 mixin advanced-options-toggle(click, cond, showMessage, hideMessage)
     .advanced-options
-        i.fa(ng-click='#{click}' ng-class='#{cond} ? "fa-chevron-circle-down" : "fa-chevron-circle-right"')
-        a(ng-click=click) {{#{cond} ? '#{hideMessage}' : '#{showMessage}'}}
+        i.fa(ng-click=`${click}` ng-class=`${cond} ? 'fa-chevron-circle-down' : 'fa-chevron-circle-right'`)
+        a(ng-click=click) {{ #{cond} ? '#{hideMessage}' : '#{showMessage}' }}
 
 //- Mixin for advanced options toggle with default settings.
 mixin advanced-options-toggle-default
@@ -28,14 +28,14 @@ mixin advanced-options-toggle-default
 
 //- Mixin for main table on screen with list of items.
 mixin main-table(title, rows, focusId, click, rowTemplate, searchField)
-    .padding-bottom-dflt(ng-show='#{rows} && #{rows}.length > 0')
-        table.links(st-table='displayedRows' st-safe-src='#{rows}')
+    .padding-bottom-dflt(ng-show=`${rows} && ${rows}.length > 0`)
+        table.links(st-table='displayedRows' st-safe-src=`${rows}`)
             thead
                 tr
                     th
                         lable.labelHeader.labelFormField #{title}:
                         .col-sm-3.pull-right(style='padding: 0')
-                            input.form-control(type='text' st-search='#{searchField}' placeholder='Filter #{title}...')
+                            input.form-control(type='text' st-search=`${searchField}` placeholder=`Filter ${title}...`)
             tbody
                 tr
                     td
@@ -55,9 +55,9 @@ mixin save-remove-clone-undo-buttons(objectName)
 
     div(ng-show='contentVisible()' style='display: inline-block;')
         .panel-tip-container(ng-hide='!backupItem || backupItem._id')
-            a.btn.btn-primary(ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && saveItem()' bs-tooltip='' data-title='{{saveBtnTipText(ui.inputForm.$dirty, "#{objectName}")}}' data-placement='bottom' data-trigger='hover') Save
+            a.btn.btn-primary(ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && saveItem()' bs-tooltip='' data-title=`{{saveBtnTipText(ui.inputForm.$dirty, '${objectName}')}}` data-placement='bottom' data-trigger='hover') Save
         .panel-tip-container(ng-show='backupItem._id')
-            a.btn.btn-primary(id='save-item' ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && saveItem()' bs-tooltip='' data-title='{{saveBtnTipText(ui.inputForm.$dirty, "#{objectName}")}}' data-placement='bottom' data-trigger='hover') Save
+            a.btn.btn-primary(id='save-item' ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && saveItem()' bs-tooltip='' data-title=`{{saveBtnTipText(ui.inputForm.$dirty, '${objectName}')}}` data-placement='bottom' data-trigger='hover') Save
         .panel-tip-container(ng-show='backupItem._id')
             a.btn.btn-primary(id='clone-item' ng-click='cloneItem()' bs-tooltip=cloneTip data-placement='bottom' data-trigger='hover') Clone
         .btn-group.panel-tip-container(ng-show='backupItem._id')
@@ -71,7 +71,7 @@ mixin save-remove-clone-undo-buttons(objectName)
 mixin error-feedback(visible, error, errorMessage, name)
     i.fa.fa-exclamation-triangle.form-control-feedback(
         ng-if=visible
-        bs-tooltip='"#{errorMessage}"'
+        bs-tooltip=`'${errorMessage}'`
         ignite-error=error
         ignite-error-message=errorMessage
         name=name
@@ -120,7 +120,7 @@ mixin java-class-autofocus-placholder(lbl, model, name, enabled, required, autof
         data-java-keywords='true'
         data-java-built-in-class='true'
         data-ignite-form-field-input-autofocus=autofocus
-        data-validation-active=validationActive ? '{{ #{validationActive} }}' : '"always"'
+        data-validation-active=validationActive ? `{{ ${validationActive} }}` : `'always'`
     )
         if  block
             block
@@ -150,7 +150,7 @@ mixin java-class-typeahead(lbl, model, name, options, enabled, required, placeho
         data-java-identifier='true'
         data-java-package-specified='allow-built-in'
         data-java-keywords='true'
-        data-validation-active=validationActive ? '{{ #{validationActive} }}' : '"always"'
+        data-validation-active=validationActive ? `{{ ${validationActive} }}` : `'always'`
     )
         +form-field-feedback(name, 'javaKeywords', errLbl + ' could not contains reserved Java keyword!')
         +form-field-feedback(name, 'javaPackageSpecified', errLbl + ' does not have package specified!')
@@ -266,12 +266,12 @@ mixin dropdown-multiple(lbl, model, name, enabled, placeholder, placeholderEmpty
 //- Mixin for table text field.
 mixin table-text-field(name, model, items, valid, save, placeholder, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         if block
@@ -289,12 +289,12 @@ mixin table-text-field(name, model, items, valid, save, placeholder, newItem)
 //- Mixin for table java class field.
 mixin table-java-class-field(lbl, name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         +form-field-feedback(name, 'javaBuiltInClass', lbl + ' should not be the Java built-in class!')
@@ -322,12 +322,12 @@ mixin table-java-class-field(lbl, name, model, items, valid, save, newItem)
 //- Mixin for table java package field.
 mixin table-java-package-field(name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         +form-field-feedback(name, 'javaKeywords', 'Package name could not contains reserved Java keyword!')
@@ -351,12 +351,12 @@ mixin table-java-package-field(name, model, items, valid, save, newItem)
 //- Mixin for table java package field.
 mixin table-url-field(name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         if block
@@ -374,12 +374,12 @@ mixin table-url-field(name, model, items, valid, save, newItem)
 //- Mixin for table address field.
 mixin table-address-field(name, model, items, valid, save, newItem, portRange)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         +ipaddress-feedback(name)
@@ -405,12 +405,12 @@ mixin table-address-field(name, model, items, valid, save, newItem, portRange)
 //- Mixin for table UUID field.
 mixin table-uuid-field(name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         if block
@@ -433,7 +433,7 @@ mixin table-save-button(valid, save, newItem)
 
     i.fa.fa-floppy-o.form-field-save(
         ng-show=valid
-        ng-click='!(#{valid}) || (#{save}); !(#{valid}) || (#{reset});'
+        ng-click=`!(${valid}) || (${save}); !(${valid}) || (${reset});`
         bs-tooltip
         data-title='Click icon or press [Enter] to save item'
     )
@@ -441,10 +441,10 @@ mixin table-save-button(valid, save, newItem)
 //- Mixin for table remove button.
 mixin table-remove-conditional-button(items, show, tip, row)
     i.tipField.fa.fa-remove(
-        ng-hide='!#{show} || field.edit'
+        ng-hide=`!${show} || field.edit`
         bs-tooltip
         data-title=tip
-        ng-click='#{items}.splice(#{items}.indexOf(#{row}), 1)'
+        ng-click=`${items}.splice(${items}.indexOf(${row}), 1)`
     )
 
 //- Mixin for table remove button.
@@ -551,11 +551,11 @@ mixin btn-remove-cond(cond, click, tip)
 
 //- LEGACY mixin for LEGACY pair values tables.
 mixin table-pair-edit(tbl, prefix, keyPlaceholder, valPlaceholder, keyJavaBuiltInTypes, valueJavaBuiltInTypes, focusId, index, divider)
-    -var keyModel = tbl + '.' + prefix + 'Key'
-    -var valModel = tbl +'.' + prefix + 'Value'
+    -var keyModel = `${tbl}.${prefix}Key`
+    -var valModel = `${tbl}.${prefix}Value`
 
-    -var keyFocusId = prefix + 'Key' + focusId
-    -var valFocusId = prefix + 'Value' + focusId
+    -var keyFocusId = `${prefix}Key${focusId}`
+    -var valFocusId = `${prefix}Value${focusId}`
 
     .col-xs-6.col-sm-6.col-md-6
         .fieldSep !{divider}
@@ -601,9 +601,9 @@ mixin dialect(lbl, model, name, required, tipTitle, genericDialectName, placehol
 //- Mixin for show/hide links.
 mixin showHideLink(name, text)
     span(ng-init='__ = {};')
-        a.customize(ng-show='__.#{name}' ng-click='__.#{name} = false') Hide #{text}
-        a.customize(ng-hide='__.#{name}' ng-click='__.#{name} = true; ui.loadPanel("#{name}");') Show #{text}
-        div(ng-if='ui.isPanelLoaded("#{name}")')
-            .panel-details(ng-show='__.#{name}')
+        a.customize(ng-show=`__.${name}` ng-click=`__.${name} = false`) Hide #{text}
+        a.customize(ng-hide=`__.${name}` ng-click=`__.${name} = true; ui.loadPanel('${name}');`) Show #{text}
+        div(ng-if=`ui.isPanelLoaded('${name}')`)
+            .panel-details(ng-show=`__.${name}`)
                 if block
                     block
diff --git a/modules/web-console/frontend/app/modules/agent/agent.module.js b/modules/web-console/frontend/app/modules/agent/agent.module.js
index 7ac39d10f3594..c0e92d5d2db99 100644
--- a/modules/web-console/frontend/app/modules/agent/agent.module.js
+++ b/modules/web-console/frontend/app/modules/agent/agent.module.js
@@ -18,6 +18,8 @@
 import angular from 'angular';
 import io from 'socket.io-client'; // eslint-disable-line no-unused-vars
 
+import templateUrl from 'views/templates/agent-download.tpl.pug';
+
 const maskNull = (val) => _.isEmpty(val) ? 'null' : val;
 
 class IgniteAgentMonitor {
@@ -35,7 +37,7 @@ class IgniteAgentMonitor {
         // Pre-fetch modal dialogs.
         this._downloadAgentModal = $modal({
             scope: this._scope,
-            templateUrl: '/templates/agent-download.html',
+            templateUrl,
             show: false,
             backdrop: 'static',
             keyboard: false
diff --git a/modules/web-console/frontend/app/modules/branding/header-logo.directive.js b/modules/web-console/frontend/app/modules/branding/header-logo.directive.js
index 423de9c1c9965..231411b818601 100644
--- a/modules/web-console/frontend/app/modules/branding/header-logo.directive.js
+++ b/modules/web-console/frontend/app/modules/branding/header-logo.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './header-logo.jade';
+import template from './header-logo.pug';
 
 export default ['igniteHeaderLogo', ['IgniteBranding', (branding) => {
     function controller() {
@@ -26,7 +26,7 @@ export default ['igniteHeaderLogo', ['IgniteBranding', (branding) => {
 
     return {
         restrict: 'E',
-        templateUrl,
+        template,
         controller,
         controllerAs: 'logo',
         replace: true
diff --git a/modules/web-console/frontend/app/modules/branding/header-logo.jade b/modules/web-console/frontend/app/modules/branding/header-logo.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/branding/header-logo.jade
rename to modules/web-console/frontend/app/modules/branding/header-logo.pug
diff --git a/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js b/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js
index 2f02446c77ffb..dce7d556353ef 100644
--- a/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js
+++ b/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './powered-by-apache.jade';
+import template from './powered-by-apache.pug';
 
 export default ['ignitePoweredByApache', ['IgniteBranding', (branding) => {
     function controller() {
@@ -26,7 +26,7 @@ export default ['ignitePoweredByApache', ['IgniteBranding', (branding) => {
 
     return {
         restrict: 'E',
-        templateUrl,
+        template,
         controller,
         controllerAs: 'poweredBy',
         replace: true
diff --git a/modules/web-console/frontend/app/modules/branding/powered-by-apache.jade b/modules/web-console/frontend/app/modules/branding/powered-by-apache.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/branding/powered-by-apache.jade
rename to modules/web-console/frontend/app/modules/branding/powered-by-apache.pug
diff --git a/modules/web-console/frontend/app/modules/demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js
index bd759df552c23..740ab30fec4e1 100644
--- a/modules/web-console/frontend/app/modules/demo/Demo.module.js
+++ b/modules/web-console/frontend/app/modules/demo/Demo.module.js
@@ -18,6 +18,7 @@
 import angular from 'angular';
 
 import DEMO_INFO from 'app/data/demo-info.json';
+import templateUrl from 'views/templates/demo-info.tpl.pug';
 
 angular
 .module('ignite-console.demo', [
@@ -129,7 +130,7 @@ angular
     }
 
     const dialog = $modal({
-        templateUrl: '/templates/demo-info.html',
+        templateUrl,
         scope,
         placement: 'center',
         show: false,
diff --git a/modules/web-console/frontend/app/modules/dialog/dialog.factory.js b/modules/web-console/frontend/app/modules/dialog/dialog.factory.js
index e15891f347ed7..2ac891793dc4d 100644
--- a/modules/web-console/frontend/app/modules/dialog/dialog.factory.js
+++ b/modules/web-console/frontend/app/modules/dialog/dialog.factory.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './dialog.jade';
+import templateUrl from './dialog.tpl.pug';
 
 export default ['IgniteDialog', ['$modal', ($modal) => {
     const defaults = {
diff --git a/modules/web-console/frontend/app/modules/dialog/dialog.jade b/modules/web-console/frontend/app/modules/dialog/dialog.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/dialog/dialog.jade
rename to modules/web-console/frontend/app/modules/dialog/dialog.tpl.pug
diff --git a/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js b/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js
index cf9f561bd837c..f4c834cfcc0ab 100644
--- a/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js
+++ b/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js
@@ -19,6 +19,7 @@ import angular from 'angular';
 
 // Getting started pages.
 import PAGES from 'app/data/getting-started.json';
+import templateUrl from 'views/templates/getting-started.tpl.pug';
 
 angular
     .module('ignite-console.getting-started', [])
@@ -77,7 +78,7 @@ angular
             _fillPage();
         };
 
-        const dialog = $modal({templateUrl: '/templates/getting-started.html', scope, placement: 'center', show: false, backdrop: 'static'});
+        const dialog = $modal({ templateUrl, scope, placement: 'center', show: false, backdrop: 'static'});
 
         scope.close = () => {
             try {
diff --git a/modules/web-console/frontend/app/modules/loading/loading.directive.js b/modules/web-console/frontend/app/modules/loading/loading.directive.js
index 064b4c28b2566..8a4ca9f5fab8c 100644
--- a/modules/web-console/frontend/app/modules/loading/loading.directive.js
+++ b/modules/web-console/frontend/app/modules/loading/loading.directive.js
@@ -15,12 +15,12 @@
  * limitations under the License.
  */
 
-import templateUrl from './loading.jade';
-import './loading.css';
+import template from './loading.pug';
+import './loading.scss';
 
-export default ['igniteLoading', ['IgniteLoading', '$templateCache', '$compile', (Loading, $templateCache, $compile) => {
+export default ['igniteLoading', ['IgniteLoading', '$compile', (Loading, $compile) => {
     const link = (scope, element) => {
-        const compiledTemplate = $compile($templateCache.get(templateUrl));
+        const compiledTemplate = $compile(template);
 
         const build = () => {
             scope.position = scope.position || 'middle';
diff --git a/modules/web-console/frontend/app/modules/loading/loading.jade b/modules/web-console/frontend/app/modules/loading/loading.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/loading/loading.jade
rename to modules/web-console/frontend/app/modules/loading/loading.pug
diff --git a/modules/web-console/frontend/app/modules/loading/loading.css b/modules/web-console/frontend/app/modules/loading/loading.scss
similarity index 100%
rename from modules/web-console/frontend/app/modules/loading/loading.css
rename to modules/web-console/frontend/app/modules/loading/loading.scss
diff --git a/modules/web-console/frontend/app/modules/nodes/Nodes.service.js b/modules/web-console/frontend/app/modules/nodes/Nodes.service.js
index b320ae4461a31..4ca1d45f6aacf 100644
--- a/modules/web-console/frontend/app/modules/nodes/Nodes.service.js
+++ b/modules/web-console/frontend/app/modules/nodes/Nodes.service.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import nodesDialogTemplate from './nodes-dialog.jade';
+import nodesDialogTplUrl from './nodes-dialog.tpl.pug';
 
 const DEFAULT_OPTIONS = {
     grid: {
@@ -41,7 +41,7 @@ class Nodes {
         options.target = cacheName;
 
         const modalInstance = $modal({
-            templateUrl: nodesDialogTemplate,
+            templateUrl: nodesDialogTplUrl,
             show: true,
             resolve: {
                 nodes: () => nodes || [],
diff --git a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.jade b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/nodes/nodes-dialog.jade
rename to modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
diff --git a/modules/web-console/frontend/app/modules/sql/notebook.controller.js b/modules/web-console/frontend/app/modules/sql/notebook.controller.js
index f10a4d005da4e..252dee65c844b 100644
--- a/modules/web-console/frontend/app/modules/sql/notebook.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/notebook.controller.js
@@ -15,11 +15,13 @@
  * limitations under the License.
  */
 
+import templateUrl from 'views/sql/notebook-new.tpl.pug';
+
 // Controller that load notebooks in navigation bar .
 export default ['$scope', '$modal', '$state', 'IgniteMessages', 'IgniteNotebook',
     (scope, $modal, $state, Messages, Notebook) => {
         // Pre-fetch modal dialogs.
-        const nameModal = $modal({scope, templateUrl: '/sql/notebook-new.html', show: false});
+        const nameModal = $modal({scope, templateUrl, show: false});
 
         scope.create = (name) => {
             return Notebook.create(name)
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 4e972ef88ab6f..e2ead13ec7c41 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -15,6 +15,12 @@
  * limitations under the License.
  */
 
+import paragraphRateTemplateUrl from 'views/sql/paragraph-rate.tpl.pug';
+import cacheMetadataTemplateUrl from 'views/sql/cache-metadata.tpl.pug';
+import chartSettingsTemplateUrl from 'views/sql/chart-settings.tpl.pug';
+import showQueryTemplateUrl from 'views/templates/message.tpl.pug';
+
+
 // Time line X axis descriptor.
 const TIME_LINE = {value: -1, type: 'java.sql.Date', label: 'TIME_LINE'};
 
@@ -188,6 +194,13 @@ class Paragraph {
 // Controller for SQL notebook screen.
 export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData',
     function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData) {
+        const $ctrl = this;
+
+        // Define template urls.
+        $ctrl.paragraphRateTemplateUrl = paragraphRateTemplateUrl;
+        $ctrl.cacheMetadataTemplateUrl = cacheMetadataTemplateUrl;
+        $ctrl.chartSettingsTemplateUrl = chartSettingsTemplateUrl;
+
         let stopTopology = null;
 
         const _tryStopRefresh = function(paragraph) {
@@ -1737,7 +1750,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 }
 
                 // Show a basic modal from a controller
-                $modal({scope, template: '/templates/message.html', placement: 'center', show: true});
+                $modal({scope, templateUrl: showQueryTemplateUrl, placement: 'center', show: true});
             }
         };
     }
diff --git a/modules/web-console/frontend/app/modules/sql/sql.module.js b/modules/web-console/frontend/app/modules/sql/sql.module.js
index 5875961f0a896..670e4f45090d9 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.module.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.module.js
@@ -20,7 +20,9 @@ import angular from 'angular';
 import NotebookData from './Notebook.data';
 import Notebook from './Notebook.service';
 import notebook from './notebook.controller';
-import sql from './sql.controller';
+import controller from './sql.controller';
+
+import sqlTplUrl from 'app/../views/sql/sql.tpl.pug';
 
 angular.module('ignite-console.sql', [
     'ui.router'
@@ -32,11 +34,13 @@ angular.module('ignite-console.sql', [
                 .state('base.sql', {
                     url: '/queries',
                     abstract: true,
-                    template: '<ui-view></ui-view>'
+                    template: '<ui-view></ui-view>',
+                    controller,
+                    controllerAs: '$ctrl'
                 })
                 .state('base.sql.notebook', {
                     url: '/notebook/{noteId}',
-                    templateUrl: '/sql/sql.html',
+                    templateUrl: sqlTplUrl,
                     onEnter: AclRoute.checkAccess('query'),
                     metaTags: {
                         title: 'Query notebook'
@@ -44,7 +48,7 @@ angular.module('ignite-console.sql', [
                 })
                 .state('base.sql.demo', {
                     url: '/demo',
-                    templateUrl: '/sql/sql.html',
+                    templateUrl: sqlTplUrl,
                     onEnter: AclRoute.checkAccess('query'),
                     metaTags: {
                         title: 'SQL demo'
@@ -54,5 +58,4 @@ angular.module('ignite-console.sql', [
     )
     .service('IgniteNotebookData', NotebookData)
     .service('IgniteNotebook', Notebook)
-    .controller('notebookController', notebook)
-    .controller('sqlController', sql);
+    .controller('notebookController', notebook);
diff --git a/modules/web-console/frontend/app/modules/states/admin.state.js b/modules/web-console/frontend/app/modules/states/admin.state.js
index 35c6fbbfeeab0..93a38ddd8c697 100644
--- a/modules/web-console/frontend/app/modules/states/admin.state.js
+++ b/modules/web-console/frontend/app/modules/states/admin.state.js
@@ -17,6 +17,8 @@
 
 import angular from 'angular';
 
+import templateUrl from 'views/settings/admin.tpl.pug';
+
 angular
 .module('ignite-console.states.admin', [
     'ui.router'
@@ -26,7 +28,7 @@ angular
     $stateProvider
     .state('settings.admin', {
         url: '/admin',
-        templateUrl: '/settings/admin.html',
+        templateUrl,
         onEnter: AclRoute.checkAccess('admin_page'),
         metaTags: {
             title: 'Admin panel'
diff --git a/modules/web-console/frontend/app/modules/states/configuration.state.js b/modules/web-console/frontend/app/modules/states/configuration.state.js
index 61dca13f9633b..a609d3a414409 100644
--- a/modules/web-console/frontend/app/modules/states/configuration.state.js
+++ b/modules/web-console/frontend/app/modules/states/configuration.state.js
@@ -26,6 +26,14 @@ import ConfigurationResource from './configuration/Configuration.resource';
 import summaryTabs from './configuration/summary/summary-tabs.directive';
 import IgniteSummaryZipper from './configuration/summary/summary-zipper.service';
 
+import sidebarTpl from 'views/configuration/sidebar.tpl.pug';
+import clustersTpl from 'views/configuration/clusters.tpl.pug';
+import cachesTpl from 'views/configuration/caches.tpl.pug';
+import domainsTpl from 'views/configuration/domains.tpl.pug';
+import igfsTpl from 'views/configuration/igfs.tpl.pug';
+import summaryTpl from 'views/configuration/summary.tpl.pug';
+import summaryTabsTemplateUrl from 'views/configuration/summary-tabs.pug';
+
 angular.module('ignite-console.states.configuration', ['ui.router'])
     .directive(...previewPanel)
     // Summary screen
@@ -33,18 +41,21 @@ angular.module('ignite-console.states.configuration', ['ui.router'])
     // Services.
     .service('IgniteSummaryZipper', IgniteSummaryZipper)
     .service('IgniteConfigurationResource', ConfigurationResource)
+    .run(['$templateCache', ($templateCache) => {
+        $templateCache.put('summary-tabs.html', summaryTabsTemplateUrl);
+    }])
     // Configure state provider.
     .config(['$stateProvider', 'AclRouteProvider', ($stateProvider, AclRoute) => {
         // Setup the states.
         $stateProvider
             .state('base.configuration', {
                 url: '/configuration',
-                templateUrl: '/configuration/sidebar.html',
+                templateUrl: sidebarTpl,
                 abstract: true
             })
             .state('base.configuration.clusters', {
                 url: '/clusters',
-                templateUrl: '/configuration/clusters.html',
+                templateUrl: clustersTpl,
                 onEnter: AclRoute.checkAccess('configuration'),
                 params: {
                     linkId: null
@@ -55,7 +66,7 @@ angular.module('ignite-console.states.configuration', ['ui.router'])
             })
             .state('base.configuration.caches', {
                 url: '/caches',
-                templateUrl: '/configuration/caches.html',
+                templateUrl: cachesTpl,
                 onEnter: AclRoute.checkAccess('configuration'),
                 params: {
                     linkId: null
@@ -66,7 +77,7 @@ angular.module('ignite-console.states.configuration', ['ui.router'])
             })
             .state('base.configuration.domains', {
                 url: '/domains',
-                templateUrl: '/configuration/domains.html',
+                templateUrl: domainsTpl,
                 onEnter: AclRoute.checkAccess('configuration'),
                 params: {
                     linkId: null
@@ -77,7 +88,7 @@ angular.module('ignite-console.states.configuration', ['ui.router'])
             })
             .state('base.configuration.igfs', {
                 url: '/igfs',
-                templateUrl: '/configuration/igfs.html',
+                templateUrl: igfsTpl,
                 onEnter: AclRoute.checkAccess('configuration'),
                 params: {
                     linkId: null
@@ -88,7 +99,7 @@ angular.module('ignite-console.states.configuration', ['ui.router'])
             })
             .state('base.configuration.summary', {
                 url: '/summary',
-                templateUrl: '/configuration/summary.html',
+                templateUrl: summaryTpl,
                 onEnter: AclRoute.checkAccess('configuration'),
                 controller: ConfigurationSummaryCtrl,
                 controllerAs: 'ctrl',
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug
similarity index 77%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug
index 3c4746bbca794..70b90bda0c4cd 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'affinity'
 -var model = 'backupItem'
@@ -28,18 +28,18 @@ include /app/helpers/jade/mixins.jade
 -var fairPartitionsRequired = fairAff + ' && ' + affModel + '.Fair.affinityBackupFilter'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Affinity Collocation
         ignite-form-field-tooltip.tipLabel
             | Collocate data with data to improve performance and scalability of your application#[br]
             | #[a(href="http://apacheignite.gridgain.org/docs/affinity-collocation" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Function:', affModel + '.kind', '"AffinityKind"', 'true', 'Default',
+                    +dropdown('Function:', `${affModel}.kind`, '"AffinityKind"', 'true', 'Default',
                         '[\
                             {value: "Rendezvous", label: "Rendezvous"},\
                             {value: "Fair", label: "Fair"},\
@@ -55,25 +55,25 @@ include /app/helpers/jade/mixins.jade
                         </ul>')
                 .panel-details(ng-if=rendezvousAff)
                     .details-row
-                        +number-required('Partitions', affModel + '.Rendezvous.partitions', '"RendPartitions"', 'true', rendPartitionsRequired, '1024', '1', 'Number of partitions')
+                        +number-required('Partitions', `${affModel}.Rendezvous.partitions`, '"RendPartitions"', 'true', rendPartitionsRequired, '1024', '1', 'Number of partitions')
                     .details-row
-                        +java-class('Backup filter', affModel + '.Rendezvous.affinityBackupFilter', '"RendAffinityBackupFilter"', 'true', 'false',
+                        +java-class('Backup filter', `${affModel}.Rendezvous.affinityBackupFilter`, '"RendAffinityBackupFilter"', 'true', 'false',
                             'Backups will be selected from all nodes that pass this filter')
                     .details-row
-                        +checkbox('Exclude neighbors', affModel + '.Rendezvous.excludeNeighbors', '"RendExcludeNeighbors"',
+                        +checkbox('Exclude neighbors', `${affModel}.Rendezvous.excludeNeighbors`, '"RendExcludeNeighbors"',
                             'Exclude same - host - neighbors from being backups of each other and specified number of backups')
                 .panel-details(ng-if=fairAff)
                     .details-row
-                        +number-required('Partitions', affModel + '.Fair.partitions', '"FairPartitions"', 'true', fairPartitionsRequired, '256', '1', 'Number of partitions')
+                        +number-required('Partitions', `${affModel}.Fair.partitions`, '"FairPartitions"', 'true', fairPartitionsRequired, '256', '1', 'Number of partitions')
                     .details-row
-                        +java-class('Backup filter', affModel + '.Fair.affinityBackupFilter', '"FairAffinityBackupFilter"', 'true', 'false',
+                        +java-class('Backup filter', `${affModel}.Fair.affinityBackupFilter`, '"FairAffinityBackupFilter"', 'true', 'false',
                             'Backups will be selected from all nodes that pass this filter')
                     .details-row
-                        +checkbox('Exclude neighbors', affModel + '.Fair.excludeNeighbors', '"FairExcludeNeighbors"',
+                        +checkbox('Exclude neighbors', `${affModel}.Fair.excludeNeighbors`, '"FairExcludeNeighbors"',
                             'Exclude same - host - neighbors from being backups of each other and specified number of backups')
                 .panel-details(ng-if=customAff)
                     .details-row
-                        +java-class('Class name:', affModel + '.Custom.className', '"AffCustomClassName"', 'true', customAff,
+                        +java-class('Class name:', `${affModel}.Custom.className`, '"AffCustomClassName"', 'true', customAff,
                             'Custom key affinity function implementation class name')
                 .settings-row
                     +java-class('Mapper:', model + '.affinityMapper', '"AffMapCustomClassName"', 'true', 'false',
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.pug
similarity index 81%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.pug
index e3d1a81589ced..cd5bcc821fca5 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'clientNearCache'
 -var model = 'backupItem.clientNearConfiguration'
 
 .panel.panel-default(ng-form=form novalidate ng-show='backupItem.cacheMode === "PARTITIONED"')
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Client near cache
         ignite-form-field-tooltip.tipLabel
@@ -28,18 +28,18 @@ include /app/helpers/jade/mixins.jade
             | Near cache is a small local cache that stores most recently or most frequently accessed data#[br]
             | Should be used in case when it is impossible to send computations to remote nodes
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var enabled = model + '.clientNearCacheEnabled'
+                -var enabled = `${model}.clientNearCacheEnabled`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"clientNacheEnabled"', 'Flag indicating whether to configure near cache')
                 .settings-row
-                    +number('Start size:', model + '.nearStartSize', '"clientNearStartSize"', enabled, '375000', '0',
+                    +number('Start size:', `${model}.nearStartSize`, '"clientNearStartSize"', enabled, '375000', '0',
                         'Initial cache size for near cache which will be used to pre-create internal hash table after start')
                 .settings-row
-                    +evictionPolicy(model + '.nearEvictionPolicy', '"clientNearCacheEvictionPolicy"', enabled, 'false',
+                    +evictionPolicy(`${model}.nearEvictionPolicy`, '"clientNearCacheEvictionPolicy"', enabled, 'false',
                         'Near cache eviction policy\
                         <ul>\
                             <li>Least Recently Used (LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.pug
similarity index 79%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.pug
index ffcd568852731..6458fbb5d3603 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.pug
@@ -14,31 +14,31 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'concurrency'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Concurrency control
         ignite-form-field-tooltip.tipLabel
             | Cache concurrent asynchronous operations settings
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Max async operations:', model + '.maxConcurrentAsyncOperations', '"maxConcurrentAsyncOperations"', 'true', '500', '0',
+                    +number('Max async operations:', `${model}.maxConcurrentAsyncOperations`, '"maxConcurrentAsyncOperations"', 'true', '500', '0',
                         'Maximum number of allowed concurrent asynchronous operations<br/>\
                         If <b>0</b> then number of concurrent asynchronous operations is unlimited')
                 .settings-row
-                    +number('Default lock timeout:', model + '.defaultLockTimeout', '"defaultLockTimeout"', 'true', '0', '0',
+                    +number('Default lock timeout:', `${model}.defaultLockTimeout`, '"defaultLockTimeout"', 'true', '0', '0',
                         'Default lock acquisition timeout in milliseconds<br/>\
                         If <b>0</b> then lock acquisition will never timeout')
-                .settings-row(ng-hide='#{model}.atomicityMode === "TRANSACTIONAL"')
-                    +dropdown('Entry versioning:', model + '.atomicWriteOrderMode', '"atomicWriteOrderMode"', 'true', 'Choose versioning',
+                .settings-row(ng-hide=`${model}.atomicityMode === 'TRANSACTIONAL'`)
+                    +dropdown('Entry versioning:', `${model}.atomicWriteOrderMode`, '"atomicWriteOrderMode"', 'true', 'Choose versioning',
                         '[\
                             {value: "CLOCK", label: "CLOCK"},\
                             {value: "PRIMARY", label: "PRIMARY"}\
@@ -49,7 +49,7 @@ include /app/helpers/jade/mixins.jade
                             <li>PRIMARY - in this mode version is assigned only on primary node. This means that sender will only send write request to primary node, which in turn will assign write version and forward it to backups</li>\
                         </ul>')
                 .settings-row
-                    +dropdown('Write synchronization mode:', model + '.writeSynchronizationMode', '"writeSynchronizationMode"', 'true', 'PRIMARY_SYNC',
+                    +dropdown('Write synchronization mode:', `${model}.writeSynchronizationMode`, '"writeSynchronizationMode"', 'true', 'PRIMARY_SYNC',
                         '[\
                             {value: "FULL_SYNC", label: "FULL_SYNC"},\
                             {value: "FULL_ASYNC", label: "FULL_ASYNC"},\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/general.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/general.pug
similarity index 77%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/general.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/general.pug
index 14f3ab45e5a63..89676f20e30e1 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/general.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/general.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var model = 'backupItem'
@@ -31,17 +31,17 @@ include /app/helpers/jade/mixins.jade
         .panel-body
             .col-sm-6
                 .settings-row
-                    +text('Name:', model + '.name', '"cacheName"', 'true', 'Input name', 'Cache name')
+                    +text('Name:', `${model}.name`, '"cacheName"', 'true', 'Input name', 'Cache name')
                 .settings-row
                     +clusters(model, 'Associate clusters with the current cache')
                 .settings-row
                     +dropdown-multiple('<span>Domain models:</span><a ui-sref="base.configuration.domains({linkId: linkId()})"> (add)</a>',
-                        model + '.domains', '"domains"', true, 'Choose domain models', 'No valid domain models configured', 'domains',
+                        `${model}.domains`, '"domains"', true, 'Choose domain models', 'No valid domain models configured', 'domains',
                         'Select domain models to describe types in cache')
                 .settings-row
-                    +cacheMode('Mode:', model + '.cacheMode', '"cacheMode"', 'PARTITIONED')
+                    +cacheMode('Mode:', `${model}.cacheMode`, '"cacheMode"', 'PARTITIONED')
                 .settings-row
-                    +dropdown('Atomicity:', model + '.atomicityMode', '"atomicityMode"', 'true', 'ATOMIC',
+                    +dropdown('Atomicity:', `${model}.atomicityMode`, '"atomicityMode"', 'true', 'ATOMIC',
                         '[\
                             {value: "ATOMIC", label: "ATOMIC"},\
                             {value: "TRANSACTIONAL", label: "TRANSACTIONAL"}\
@@ -51,18 +51,18 @@ include /app/helpers/jade/mixins.jade
                             <li>ATOMIC - in this mode distributed transactions and distributed locking are not supported</li>\
                             <li>TRANSACTIONAL - in this mode specified fully ACID-compliant transactional cache behavior</li>\
                         </ul>')
-                .settings-row(data-ng-show='#{model}.cacheMode === "PARTITIONED"')
-                    +number('Backups:', model + '.backups', '"backups"', 'true', '0', '0', 'Number of nodes used to back up single partition for partitioned cache')
-                .settings-row(data-ng-show='#{model}.cacheMode === "PARTITIONED" && #{model}.backups')
-                    +checkbox('Read from backup', model + '.readFromBackup', '"readFromBackup"',
+                .settings-row(data-ng-show=`${model}.cacheMode === 'PARTITIONED'`)
+                    +number('Backups:', `${model}.backups`, '"backups"', 'true', '0', '0', 'Number of nodes used to back up single partition for partitioned cache')
+                .settings-row(data-ng-show=`${model}.cacheMode === 'PARTITIONED' && ${model}.backups`)
+                    +checkbox('Read from backup', `${model}.readFromBackup`, '"readFromBackup"',
                         'Flag indicating whether data can be read from backup<br/>\
                         If not set then always get data from primary node (never from backup)')
                 .settings-row
-                    +checkbox('Copy on read', model + '.copyOnRead', '"copyOnRead"',
+                    +checkbox('Copy on read', `${model}.copyOnRead`, '"copyOnRead"',
                         'Flag indicating whether copy of the value stored in cache should be created for cache operation implying return value<br/>\
                         Also if this flag is set copies are created for values passed to CacheInterceptor and to CacheEntryProcessor')
-                .settings-row(ng-show='#{model}.cacheMode === "PARTITIONED" && #{model}.atomicityMode === "TRANSACTIONAL"')
-                    +checkbox('Invalidate near cache', model + '.invalidate', '"invalidate"',
+                .settings-row(ng-show=`${model}.cacheMode === 'PARTITIONED' && ${model}.atomicityMode === 'TRANSACTIONAL'`)
+                    +checkbox('Invalidate near cache', `${model}.invalidate`, '"invalidate"',
                         'Invalidation flag for near cache entries in transaction<br/>\
                         If set then values will be invalidated (nullified) upon commit in near cache')
             .col-sm-6
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/memory.pug
similarity index 86%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/memory.pug
index e8dfb3a15a1fc..38482a78f8f06 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/memory.pug
@@ -14,24 +14,24 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'memory'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Memory
         ignite-form-field-tooltip.tipLabel
             | Cache memory settings#[br]
             | #[a(href="https://apacheignite.readme.io/docs/off-heap-memory" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Mode:', model + '.memoryMode', '"memoryMode"', 'true', 'ONHEAP_TIERED',
+                    +dropdown('Mode:', `${model}.memoryMode`, '"memoryMode"', 'true', 'ONHEAP_TIERED',
                         '[\
                             {value: "ONHEAP_TIERED", label: "ONHEAP_TIERED"},\
                             {value: "OFFHEAP_TIERED", label: "OFFHEAP_TIERED"},\
@@ -61,9 +61,8 @@ include /app/helpers/jade/mixins.jade
                                 Note that in this mode entries can be evicted only to swap\
                             </li>\
                         </ul>')
-                .settings-row(ng-show=model + '.memoryMode !== "OFFHEAP_VALUES"')
-                    +dropdown-required('Off-heap memory:', model + '.offHeapMode', '"offHeapMode"', 'true',
-                        model + '.memoryMode === "OFFHEAP_TIERED"',
+                .settings-row(ng-show=`${model}.memoryMode !== 'OFFHEAP_VALUES'`)
+                    +dropdown-required('Off-heap memory:', `${model}.offHeapMode`, '"offHeapMode"', 'true', `${model}.memoryMode === 'OFFHEAP_TIERED'`,
                         'Disabled',
                         '[\
                             {value: -1, label: "Disabled"},\
@@ -76,16 +75,16 @@ include /app/helpers/jade/mixins.jade
                             <li>Limited - Off-heap storage has limited size</li>\
                             <li>Unlimited - Off-heap storage grow infinitely (it is up to user to properly add and remove entries from cache to ensure that off-heap storage does not grow infinitely)</li>\
                         </ul>')
-                .settings-row(ng-if=model + '.offHeapMode === 1 && ' + model + '.memoryMode !== "OFFHEAP_VALUES"')
-                    +number-required('Off-heap memory max size:', model + '.offHeapMaxMemory', '"offHeapMaxMemory"', 'true',
-                        model + '.offHeapMode === 1', 'Enter off-heap memory size', '1',
+                .settings-row(ng-if=`${model}.offHeapMode === 1 && ${model}.memoryMode !== 'OFFHEAP_VALUES'`)
+                    +number-required('Off-heap memory max size:', `${model}.offHeapMaxMemory`, '"offHeapMaxMemory"', 'true',
+                        `${model}.offHeapMode === 1`, 'Enter off-heap memory size', '1',
                         'Maximum amount of memory available to off-heap storage in bytes')
                 .settings-row
                     -var onHeapTired = model + '.memoryMode === "ONHEAP_TIERED"'
                     -var swapEnabled = model + '.swapEnabled'
                     -var offHeapMaxMemory = model + '.offHeapMaxMemory'
 
-                    +evictionPolicy(model + '.evictionPolicy', '"evictionPolicy"', 'true',
+                    +evictionPolicy(`${model}.evictionPolicy`, '"evictionPolicy"', 'true',
                         onHeapTired  + ' && (' + swapEnabled + '|| _.isNumber(' + offHeapMaxMemory + ') &&' + offHeapMaxMemory + ' >= 0)',
                         'Optional cache eviction policy<br/>\
                         Must be set for entries to be evicted from on-heap to off-heap or swap\
@@ -95,7 +94,7 @@ include /app/helpers/jade/mixins.jade
                             <li>SORTED - Eviction policy which will select the minimum cache entry for eviction</li>\
                         </ul>')
                 .settings-row
-                    +number('Start size:', model + '.startSize', '"startSize"', 'true', '1500000', '0',
+                    +number('Start size:', `${model}.startSize`, '"startSize"', 'true', '1500000', '0',
                         'In terms of size and capacity, Ignite internal cache map acts exactly like a normal Java HashMap: it has some initial capacity\
                         (which is pretty small by default), which doubles as data arrives. The process of internal cache map resizing is CPU-intensive\
                         and time-consuming, and if you load a huge dataset into cache (which is a normal use case), the map will have to resize a lot of times.\
@@ -104,6 +103,6 @@ include /app/helpers/jade/mixins.jade
                         For example, if you expect to load 10 million entries into cache, you can set this property to 10 000 000.\
                         This will save you from cache internal map resizes.')
                 .settings-row
-                    +checkbox('Swap enabled', model + '.swapEnabled', '"swapEnabled"', 'Flag indicating whether swap storage is enabled or not for this cache')
+                    +checkbox('Swap enabled', `${model}.swapEnabled`, '"swapEnabled"', 'Flag indicating whether swap storage is enabled or not for this cache')
             .col-sm-6
                 +preview-xml-java(model, 'cacheMemory')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.pug
similarity index 76%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.pug
index 56f7e648c8cc3..499dd2de230b2 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'clientNearCache'
 -var model = 'backupItem'
 
-.panel.panel-default(ng-form=form novalidate ng-show='backupItem.cacheMode === "PARTITIONED"')
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+.panel.panel-default(ng-form=form novalidate ng-show=`${model}.cacheMode === 'PARTITIONED'`)
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Near cache on client node
         ignite-form-field-tooltip.tipLabel
@@ -28,19 +28,19 @@ include /app/helpers/jade/mixins.jade
             | Near cache is a small local cache that stores most recently or most frequently accessed data#[br]
             | Should be used in case when it is impossible to send computations to remote nodes
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var nearCfg = model + '.clientNearConfiguration'
-                -var enabled = nearCfg + '.enabled'
+                -var nearCfg = `${model}.clientNearConfiguration`
+                -var enabled = `${nearCfg}.enabled`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"clientNearEnabled"', 'Flag indicating whether to configure near cache')
                 .settings-row
-                    +number('Start size:', nearCfg + '.nearStartSize', '"clientNearStartSize"', enabled, '375000', '0',
+                    +number('Start size:', `${nearCfg}.nearStartSize`, '"clientNearStartSize"', enabled, '375000', '0',
                         'Initial cache size for near cache which will be used to pre-create internal hash table after start')
                 .settings-row
-                    +evictionPolicy(nearCfg + '.nearEvictionPolicy', '"clientNearCacheEvictionPolicy"', enabled, 'false',
+                    +evictionPolicy(`${nearCfg}.nearEvictionPolic`, '"clientNearCacheEvictionPolicy"', enabled, 'false',
                         'Near cache eviction policy\
                         <ul>\
                             <li>Least Recently Used (LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.pug
similarity index 77%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.pug
index 9895281486ef3..2efe43a9e849c 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'serverNearCache'
 -var model = 'backupItem'
 
-.panel.panel-default(ng-form=form novalidate ng-show='#{model}.cacheMode === "PARTITIONED"')
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+.panel.panel-default(ng-form=form novalidate ng-show=`${model}.cacheMode === 'PARTITIONED'`)
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Near cache on server node
         ignite-form-field-tooltip.tipLabel
@@ -29,19 +29,19 @@ include /app/helpers/jade/mixins.jade
             | Should be used in case when it is impossible to send computations to remote nodes#[br]
             | #[a(href="https://apacheignite.readme.io/docs/near-caches" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var nearCfg = model + '.nearConfiguration'
-                -var enabled = nearCfg + '.enabled'
+                -var nearCfg = `${model}.nearConfiguration`
+                -var enabled = `${nearCfg}.enabled`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"nearCacheEnabled"', 'Flag indicating whether to configure near cache')
                 .settings-row
-                    +number('Start size:', nearCfg + '.nearStartSize', '"nearStartSize"', enabled, '375000', '0',
+                    +number('Start size:', `${nearCfg}.nearStartSize`, '"nearStartSize"', enabled, '375000', '0',
                         'Initial cache size for near cache which will be used to pre-create internal hash table after start')
                 .settings-row
-                    +evictionPolicy(model + '.nearConfiguration.nearEvictionPolicy', '"nearCacheEvictionPolicy"', enabled, 'false',
+                    +evictionPolicy(`${model}.nearConfiguration.nearEvictionPolicy`, '"nearCacheEvictionPolicy"', enabled, 'false',
                         'Near cache eviction policy\
                         <ul>\
                             <li>Least Recently Used (LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.pug
similarity index 71%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.pug
index bcac5adcaf995..6715dcd259aa6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.pug
@@ -14,20 +14,20 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'nodeFilter'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label(id='nodeFilter-title') Node filter
         ignite-form-field-tooltip.tipLabel
             | Determines on what nodes the cache should be started
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     -var nodeFilter = model + '.nodeFilter';
@@ -42,18 +42,18 @@ include /app/helpers/jade/mixins.jade
                         'Node filter variant'
                     )
                 .settings-row(ng-show=nodeFilterKind)
-                    div(ng-show='#{nodeFilterKind} === "IGFS"')
-                        -var igfsNodeFilter = nodeFilter + '.IGFS'
-                        -var required = nodeFilterKind + ' === "IGFS"'
+                    div(ng-show=`${nodeFilterKind} === 'IGFS'`)
+                        -var igfsNodeFilter = `${nodeFilter}.IGFS`
+                        -var required = `${nodeFilterKind} === 'IGFS'`
 
                         //(lbl, model, name, enabled, required, placeholder, options, tip)
-                        +dropdown-required-empty('IGFS:', igfsNodeFilter + '.igfs', '"igfsNodeFilter"', 'true', required,
+                        +dropdown-required-empty('IGFS:', `${igfsNodeFilter}.igfs`, '"igfsNodeFilter"', 'true', required,
                             'Choose IGFS', 'No IGFS configured', 'igfss', 'Select IGFS to filter nodes')
-                    div(ng-show='#{nodeFilterKind} === "Custom"')
-                        -var customNodeFilter = nodeFilter + '.Custom'
-                        -var required = nodeFilterKind + ' === "Custom"'
+                    div(ng-show=`${nodeFilterKind} === 'Custom'`)
+                        -var customNodeFilter = `${nodeFilter}.Custom`
+                        -var required = `${nodeFilterKind} === 'Custom'`
 
-                        +java-class('Class name:', customNodeFilter + '.className', '"customNodeFilter"',
+                        +java-class('Class name:', `${customNodeFilter}.className`, '"customNodeFilter"',
                             'true', required, 'Class name of custom node filter implementation', required)
             .col-sm-6
                 +preview-xml-java(model, 'cacheNodeFilter', 'igfss')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/query.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/query.pug
similarity index 76%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/query.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/query.pug
index cfbaf12c57a8a..5bb515aba7dc1 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/query.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/query.pug
@@ -14,24 +14,24 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'query'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Queries & Indexing
         ignite-form-field-tooltip.tipLabel
             | Cache queries settings#[br]
             | #[a(href="https://apacheignite.readme.io/docs/sql-queries" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +text('SQL schema name:', model + '.sqlSchema', '"sqlSchema"', 'false', 'Input schema name',
+                    +text('SQL schema name:', `${model}.sqlSchema`, '"sqlSchema"', 'false', 'Input schema name',
                         'Specify any custom name to be used as SQL schema for current cache. This name will correspond to SQL ANSI-99 standard.\
                         Nonquoted identifiers are not case sensitive. Quoted identifiers are case sensitive.\
                         When SQL schema is not specified, quoted cache name should used instead.<br/>\
@@ -47,19 +47,19 @@ include /app/helpers/jade/mixins.jade
                             </li>\
                         </ul>')
                 .settings-row
-                    +number('On-heap cache for off-heap indexes:', model + '.sqlOnheapRowCacheSize', '"sqlOnheapRowCacheSize"', 'true', '10240', '1',
+                    +number('On-heap cache for off-heap indexes:', `${model}.sqlOnheapRowCacheSize`, '"sqlOnheapRowCacheSize"', 'true', '10240', '1',
                         'Number of SQL rows which will be cached onheap to avoid deserialization on each SQL index access')
                 .settings-row
-                    +number('Long query timeout:', model + '.longQueryWarningTimeout', '"longQueryWarningTimeout"', 'true', '3000', '0',
+                    +number('Long query timeout:', `${model}.longQueryWarningTimeout`, '"longQueryWarningTimeout"', 'true', '3000', '0',
                         'Timeout in milliseconds after which long query warning will be printed')
                 .settings-row
-                    +number('History size:', model + '.queryDetailMetricsSize', '"queryDetailMetricsSize"', 'true', '0', '0',
+                    +number('History size:', `${model}.queryDetailMetricsSize`, '"queryDetailMetricsSize"', 'true', '0', '0',
                         'Size of queries detail metrics that will be stored in memory for monitoring purposes')
                 .settings-row
                     -var form = 'querySqlFunctionClasses';
-                    -var sqlFunctionClasses = model + '.sqlFunctionClasses';
+                    -var sqlFunctionClasses = `${model}.sqlFunctionClasses`;
 
-                    +ignite-form-group(ng-form=form ng-model=sqlFunctionClasses)
+                    +ignite-form-group(ng-form=form ng-model=`${sqlFunctionClasses}`)
                         ignite-form-field-label
                             | SQL functions
                         ignite-form-group-tooltip
@@ -69,20 +69,20 @@ include /app/helpers/jade/mixins.jade
 
                         -var uniqueTip = 'SQL function with such class name already exists!'
 
-                        .group-content(ng-if='#{sqlFunctionClasses}.length')
+                        .group-content(ng-if=`${sqlFunctionClasses}.length`)
                             -var model = 'obj.model';
                             -var name = '"edit" + $index'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = sqlFunctionClasses + '[$index] = ' + model
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${sqlFunctionClasses}[$index] = ${model}`
 
-                            div(ng-repeat='model in #{sqlFunctionClasses} track by $index' ng-init='obj = {}')
+                            div(ng-repeat=`model in ${sqlFunctionClasses} track by $index` ng-init='obj = {}')
                                 label.col-xs-12.col-sm-12.col-md-12
                                     .indexField
                                         | {{ $index+1 }})
                                     +table-remove-button(sqlFunctionClasses, 'Remove user-defined function')
 
                                     span(ng-hide='field.edit')
-                                        a.labelFormField(ng-click='field.edit = true; #{model} = model;') {{ model }}
+                                        a.labelFormField(ng-click=`field.edit = true; ${model} = model;`) {{ model }}
                                     span(ng-if='field.edit')
                                         +table-java-class-field('SQL function', name, model, sqlFunctionClasses, valid, save, false)
                                             +table-save-button(valid, save, false)
@@ -91,8 +91,8 @@ include /app/helpers/jade/mixins.jade
                         .group-content(ng-repeat='field in group.add')
                             -var model = 'new';
                             -var name = '"new"'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = sqlFunctionClasses + '.push(' + model + ')'
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${sqlFunctionClasses}.push(${model})`
 
                             div
                                 label.col-xs-12.col-sm-12.col-md-12
@@ -100,13 +100,13 @@ include /app/helpers/jade/mixins.jade
                                         +table-save-button(valid, save, true)
                                         +unique-feedback(name, uniqueTip)
 
-                        .group-content-empty(ng-if='!(#{sqlFunctionClasses}.length) && !group.add.length')
+                        .group-content-empty(ng-if=`!(${sqlFunctionClasses}.length) && !group.add.length`)
                             | Not defined
                 .settings-row
-                    +checkbox('Snapshotable index', model + '.snapshotableIndex', '"snapshotableIndex"',
+                    +checkbox('Snapshotable index', `${model}.snapshotableIndex`, '"snapshotableIndex"',
                         'Flag indicating whether SQL indexes should support snapshots')
                 .settings-row
-                    +checkbox('Escape table and filed names', model + '.sqlEscapeAll', '"sqlEscapeAll"',
+                    +checkbox('Escape table and filed names', `${model}.sqlEscapeAll`, '"sqlEscapeAll"',
                         'If enabled than all schema, table and field names will be escaped with double quotes (for example: "tableName"."fieldName").<br/>\
                         This enforces case sensitivity for field names and also allows having special characters in table and field names.<br/>\
                         Escaped names will be used for creation internal structures in Ignite SQL engine.')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.pug
similarity index 75%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.pug
index d8ef3adb62239..9850d171c9ce7 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.pug
@@ -14,24 +14,24 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'rebalance'
 -var model = 'backupItem'
 
-.panel.panel-default(ng-form=form novalidate ng-hide='#{model}.cacheMode === "LOCAL"')
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+.panel.panel-default(ng-form=form novalidate ng-hide=`${model}.cacheMode === "LOCAL"`)
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Rebalance
         ignite-form-field-tooltip.tipLabel
             | Cache rebalance settings#[br]
             | #[a(href="https://apacheignite.readme.io/docs/rebalancing" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Mode:', model + '.rebalanceMode', '"rebalanceMode"', 'true', 'ASYNC',
+                    +dropdown('Mode:', `${model}.rebalanceMode`, '"rebalanceMode"', 'true', 'ASYNC',
                         '[\
                             {value: "SYNC", label: "SYNC"},\
                             {value: "ASYNC", label: "ASYNC"},\
@@ -44,23 +44,23 @@ include /app/helpers/jade/mixins.jade
                             <li>None - in this mode no rebalancing will take place which means that caches will be either loaded on demand from persistent store whenever data is accessed, or will be populated explicitly</li>\
                         </ul>')
                     .settings-row
-                        +number('Batch size:', model + '.rebalanceBatchSize', '"rebalanceBatchSize"', 'true', '512 * 1024', '1',
+                        +number('Batch size:', `${model}.rebalanceBatchSize`, '"rebalanceBatchSize"', 'true', '512 * 1024', '1',
                             'Size (in bytes) to be loaded within a single rebalance message<br/>\
                             Rebalancing algorithm will split total data set on every node into multiple batches prior to sending data')
                     .settings-row
-                        +number('Batches prefetch count:', model + '.rebalanceBatchesPrefetchCount', '"rebalanceBatchesPrefetchCount"', 'true', '2', '1',
+                        +number('Batches prefetch count:', `${model}.rebalanceBatchesPrefetchCount`, '"rebalanceBatchesPrefetchCount"', 'true', '2', '1',
                             'Number of batches generated by supply node at rebalancing start')
                     .settings-row
-                        +number('Order:', model + '.rebalanceOrder', '"rebalanceOrder"', 'true', '0', Number.MIN_SAFE_INTEGER,
+                        +number('Order:', `${model}.rebalanceOrder`, '"rebalanceOrder"', 'true', '0', Number.MIN_SAFE_INTEGER,
                             'If cache rebalance order is positive, rebalancing for this cache will be started only when rebalancing for all caches with smaller rebalance order (except caches with rebalance order 0) will be completed')
                     .settings-row
-                        +number('Delay:', model + '.rebalanceDelay', '"rebalanceDelay"', 'true', '0', '0',
+                        +number('Delay:', `${model}.rebalanceDelay`, '"rebalanceDelay"', 'true', '0', '0',
                             'Delay in milliseconds upon a node joining or leaving topology (or crash) after which rebalancing should be started automatically')
                     .settings-row
-                        +number('Timeout:', model + '.rebalanceTimeout', '"rebalanceTimeout"', 'true', '10000', '0',
+                        +number('Timeout:', `${model}.rebalanceTimeout`, '"rebalanceTimeout"', 'true', '10000', '0',
                             'Rebalance timeout in milliseconds')
                     .settings-row
-                        +number('Throttle:', model + '.rebalanceThrottle', '"rebalanceThrottle"', 'true', '0', '0',
+                        +number('Throttle:', `${model}.rebalanceThrottle`, '"rebalanceThrottle"', 'true', '0', '0',
                             'Time in milliseconds to wait between rebalance messages to avoid overloading of CPU or network')
             .col-sm-6
                 +preview-xml-java(model, 'cacheRebalance')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/statistics.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/statistics.pug
similarity index 76%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/statistics.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/statistics.pug
index 9f7a3465b29fb..1eeea84f77656 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/statistics.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/statistics.pug
@@ -14,25 +14,25 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'statistics'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Statistics
         ignite-form-field-tooltip.tipLabel
             | Cache statistics and management settings
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +checkbox('Statistics enabled', model + '.statisticsEnabled', '"statisticsEnabled"', 'Flag indicating whether statistics gathering is enabled on this cache')
+                    +checkbox('Statistics enabled', `${model}.statisticsEnabled`, '"statisticsEnabled"', 'Flag indicating whether statistics gathering is enabled on this cache')
                 .settings-row
-                    +checkbox('Management enabled', model + '.managementEnabled', '"managementEnabled"',
+                    +checkbox('Management enabled', `${model}.managementEnabled`, '"managementEnabled"',
                     'Flag indicating whether management is enabled on this cache<br/>\
                     If enabled the CacheMXBean for each cache is registered in the platform MBean server')
             .col-sm-6
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/store.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/store.pug
similarity index 72%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/store.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/store.pug
index ea350f2394ef7..f09bc0b2cca43 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/store.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/store.pug
@@ -14,19 +14,19 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'store'
 -var model = 'backupItem'
 
 mixin hibernateField(name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         if block
@@ -43,19 +43,19 @@ mixin hibernateField(name, model, items, valid, save, newItem)
             )
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Store
         ignite-form-field-tooltip.tipLabel
             | Cache store settings#[br]
             | #[a(href="https://apacheignite.readme.io/docs/persistent-store" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    -var storeFactory = model + '.cacheStoreFactory';
-                    -var storeFactoryKind = storeFactory + '.kind';
+                    -var storeFactory = `${model}.cacheStoreFactory`;
+                    -var storeFactoryKind = `${storeFactory}.kind`;
 
                     +dropdown('Store factory:', storeFactoryKind, '"cacheStoreFactory"', 'true', 'Not set',
                         '[\
@@ -75,44 +75,44 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                         a.customize(ng-show='__.expanded' ng-click='__.expanded = false') Hide settings
                         a.customize(ng-hide='__.expanded' ng-click='__.expanded = true') Show settings
                         .panel-details(ng-show='__.expanded')
-                            div(ng-show='#{storeFactoryKind} === "CacheJdbcPojoStoreFactory"')
-                                -var pojoStoreFactory = storeFactory + '.CacheJdbcPojoStoreFactory'
-                                -var required = storeFactoryKind + ' === "CacheJdbcPojoStoreFactory"'
+                            div(ng-show=`${storeFactoryKind} === 'CacheJdbcPojoStoreFactory'`)
+                                -var pojoStoreFactory = `${storeFactory}.CacheJdbcPojoStoreFactory`
+                                -var required = `${storeFactoryKind} === 'CacheJdbcPojoStoreFactory'`
 
                                 .details-row
-                                    +text('Data source bean name:', pojoStoreFactory + '.dataSourceBean',
+                                    +text('Data source bean name:', `${pojoStoreFactory}.dataSourceBean`,
                                         '"pojoDataSourceBean"', required, 'Input bean name',
                                         'Name of the data source bean in Spring context')
                                 .details-row
-                                    +dialect('Dialect:', pojoStoreFactory + '.dialect', '"pojoDialect"', required,
+                                    +dialect('Dialect:', `${pojoStoreFactory}.dialect`, '"pojoDialect"', required,
                                         'Dialect of SQL implemented by a particular RDBMS:', 'Generic JDBC dialect',
                                         'Choose JDBC dialect')
                                 .details-row
-                                    +number('Batch size:', pojoStoreFactory + '.batchSize', '"pojoBatchSize"', true, '512', '1',
+                                    +number('Batch size:', `${pojoStoreFactory}.batchSize`, '"pojoBatchSize"', true, '512', '1',
                                         'Maximum batch size for writeAll and deleteAll operations')
                                 .details-row
-                                    +number('Thread count:', pojoStoreFactory + '.maximumPoolSize', '"pojoMaximumPoolSize"', true, 'availableProcessors', '1',
+                                    +number('Thread count:', `${pojoStoreFactory}.maximumPoolSize`, '"pojoMaximumPoolSize"', true, 'availableProcessors', '1',
                                         'Maximum workers thread count.<br/>\
                                         These threads are responsible for load cache.')
                                 .details-row
-                                    +number('Maximum write attempts:', pojoStoreFactory + '.maximumWriteAttempts', '"pojoMaximumWriteAttempts"', true, '2', '0',
+                                    +number('Maximum write attempts:', `${pojoStoreFactory}.maximumWriteAttempts`, '"pojoMaximumWriteAttempts"', true, '2', '0',
                                         'Maximum write attempts in case of database error')
                                 .details-row
-                                    +number('Parallel load threshold:', pojoStoreFactory + '.parallelLoadCacheMinimumThreshold', '"pojoParallelLoadCacheMinimumThreshold"', true, '512', '0',
+                                    +number('Parallel load threshold:', `${pojoStoreFactory}.parallelLoadCacheMinimumThreshold`, '"pojoParallelLoadCacheMinimumThreshold"', true, '512', '0',
                                         'Parallel load cache minimum threshold.<br/>\
                                         If <b>0</b> then load sequentially.')
                                 .details-row
-                                    +java-class('Hasher', pojoStoreFactory + '.hasher', '"pojoHasher"', 'true', 'false', 'Hash calculator', required)
+                                    +java-class('Hasher', `${pojoStoreFactory}.hasher`, '"pojoHasher"', 'true', 'false', 'Hash calculator', required)
                                 .details-row
-                                    +java-class('Transformer', pojoStoreFactory + '.transformer', '"pojoTransformer"', 'true', 'false', 'Types transformer', required)
+                                    +java-class('Transformer', `${pojoStoreFactory}.transformer`, '"pojoTransformer"', 'true', 'false', 'Types transformer', required)
                                 .details-row
-                                    +checkbox('Escape table and filed names', pojoStoreFactory + '.sqlEscapeAll', '"sqlEscapeAll"',
+                                    +checkbox('Escape table and filed names', `${pojoStoreFactory}.sqlEscapeAll`, '"sqlEscapeAll"',
                                         'If enabled than all schema, table and field names will be escaped with double quotes (for example: "tableName"."fieldName").<br/>\
                                         This enforces case sensitivity for field names and also allows having special characters in table and field names.<br/>\
                                         Escaped names will be used for CacheJdbcPojoStore internal SQL queries.')
-                            div(ng-show='#{storeFactoryKind} === "CacheJdbcBlobStoreFactory"')
-                                -var blobStoreFactory = storeFactory + '.CacheJdbcBlobStoreFactory'
-                                -var blobStoreFactoryVia = blobStoreFactory + '.connectVia'
+                            div(ng-show=`${storeFactoryKind} === 'CacheJdbcBlobStoreFactory'`)
+                                -var blobStoreFactory = `${storeFactory}.CacheJdbcBlobStoreFactory`
+                                -var blobStoreFactoryVia = `${blobStoreFactory}.connectVia`
 
                                 .details-row
                                     +dropdown('Connect via:', blobStoreFactoryVia, '"connectVia"', 'true', 'Choose connection method',
@@ -125,51 +125,51 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                                             <li>JDBC URL, for example: jdbc:h2:mem:myDatabase</li>\
                                             <li>Configured data source</li>\
                                         </ul>')
-                                div(ng-show='#{blobStoreFactoryVia} === "URL"')
-                                    -var required = storeFactoryKind + ' === "CacheJdbcBlobStoreFactory" && ' + blobStoreFactoryVia + ' === "URL"'
+                                div(ng-show=`${blobStoreFactoryVia} === 'URL'`)
+                                    -var required = `${storeFactoryKind} === 'CacheJdbcBlobStoreFactory' && ${blobStoreFactoryVia} === 'URL'`
 
                                     .details-row
-                                        +text('Connection URL:', blobStoreFactory + '.connectionUrl', '"connectionUrl"', required, 'Input URL',
+                                        +text('Connection URL:', `${blobStoreFactory}.connectionUrl`, '"connectionUrl"', required, 'Input URL',
                                             'URL for database access, for example: jdbc:h2:mem:myDatabase')
                                     .details-row
-                                        +text('User:', blobStoreFactory + '.user', '"user"', required, 'Input user name', 'User name for database access')
+                                        +text('User:', `${blobStoreFactory}.user`, '"user"', required, 'Input user name', 'User name for database access')
                                     .details-row
                                         label Note, password will be generated as stub
-                                div(ng-show='#{blobStoreFactoryVia} !== "URL"')
-                                    -var required = storeFactoryKind + ' === "CacheJdbcBlobStoreFactory" && ' + blobStoreFactoryVia + '!== "URL"'
+                                div(ng-show=`${blobStoreFactoryVia} !== 'URL'`)
+                                    -var required = `${storeFactoryKind} === 'CacheJdbcBlobStoreFactory' && ${blobStoreFactoryVia} !== 'URL'`
 
                                     .details-row
-                                        +text('Data source bean name:', blobStoreFactory + '.dataSourceBean', '"blobDataSourceBean"', required, 'Input bean name',
+                                        +text('Data source bean name:', `${blobStoreFactory}.dataSourceBean`, '"blobDataSourceBean"', required, 'Input bean name',
                                             'Name of the data source bean in Spring context')
                                     .details-row
-                                        +dialect('Database:', blobStoreFactory + '.dialect', '"blobDialect"', required, 'Supported databases:', 'Generic database', 'Choose database')
+                                        +dialect('Database:', `${blobStoreFactory}.dialect`, '"blobDialect"', required, 'Supported databases:', 'Generic database', 'Choose database')
                                 .details-row
-                                    +checkbox('Init schema', blobStoreFactory + '.initSchema', '"initSchema"',
+                                    +checkbox('Init schema', `${blobStoreFactory}.initSchema`, '"initSchema"',
                                         'Flag indicating whether DB schema should be initialized by Ignite (default behaviour) or was explicitly created by user')
                                 .details-row
-                                    +text('Create query:', blobStoreFactory + '.createTableQuery', '"createTableQuery"', 'false', 'SQL for table creation',
+                                    +text('Create query:', `${blobStoreFactory}.createTableQuery`, '"createTableQuery"', 'false', 'SQL for table creation',
                                         'Query for table creation in underlying database<br/>\
                                         Default value: create table if not exists ENTRIES (key binary primary key, val binary)')
                                 .details-row
-                                    +text('Load query:', blobStoreFactory + '.loadQuery', '"loadQuery"', 'false', 'SQL for load entry',
+                                    +text('Load query:', `${blobStoreFactory}.loadQuery`, '"loadQuery"', 'false', 'SQL for load entry',
                                         'Query for entry load from underlying database<br/>\
                                         Default value: select * from ENTRIES where key=?')
                                 .details-row
-                                    +text('Insert query:', blobStoreFactory + '.insertQuery', '"insertQuery"', 'false', 'SQL for insert entry',
+                                    +text('Insert query:', `${blobStoreFactory}.insertQuery`, '"insertQuery"', 'false', 'SQL for insert entry',
                                         'Query for insert entry into underlying database<br/>\
                                         Default value: insert into ENTRIES (key, val) values (?, ?)')
                                 .details-row
-                                    +text('Update query:', blobStoreFactory + '.updateQuery', '"updateQuery"', 'false', 'SQL for update entry',
+                                    +text('Update query:', `${blobStoreFactory}.updateQuery`, '"updateQuery"', 'false', 'SQL for update entry',
                                         'Query for update entry in underlying database<br/>\
                                         Default value: update ENTRIES set val=? where key=?')
                                 .details-row
-                                    +text('Delete query:', blobStoreFactory + '.deleteQuery', '"deleteQuery"', 'false', 'SQL for delete entry',
+                                    +text('Delete query:', `${blobStoreFactory}.deleteQuery`, '"deleteQuery"', 'false', 'SQL for delete entry',
                                         'Query for delete entry from underlying database<br/>\
                                         Default value: delete from ENTRIES where key=?')
 
-                            div(ng-show='#{storeFactoryKind} === "CacheHibernateBlobStoreFactory"')
-                                -var hibernateStoreFactory = storeFactory + '.CacheHibernateBlobStoreFactory'
-                                -var hibernateProperties = hibernateStoreFactory + '.hibernateProperties'
+                            div(ng-show=`${storeFactoryKind} === 'CacheHibernateBlobStoreFactory'`)
+                                -var hibernateStoreFactory = `${storeFactory}.CacheHibernateBlobStoreFactory`
+                                -var hibernateProperties = `${hibernateStoreFactory}.hibernateProperties`
 
                                 .details-row
                                     +ignite-form-group(ng-form=form ng-model=hibernateProperties)
@@ -184,12 +184,12 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                                         -var tipUnique = 'Property with such key already exists!'
                                         -var tipPropertySpecified = 'Property should be present in format key=value!'
 
-                                        .group-content-empty(ng-if='!((#{hibernateProperties} && #{hibernateProperties}.length > 0) || tableNewItemActive(hibernatePropsTbl))')
+                                        .group-content-empty(ng-if=`!((${hibernateProperties} && ${hibernateProperties}.length > 0) || tableNewItemActive(hibernatePropsTbl))`)
                                             | Not defined
-                                        .group-content(ng-show='(#{hibernateProperties} && #{hibernateProperties}.length > 0) || tableNewItemActive(hibernatePropsTbl)')
+                                        .group-content(ng-show=`(${hibernateProperties} && ${hibernateProperties}.length > 0) || tableNewItemActive(hibernatePropsTbl)`)
                                             table.links-edit(id='hibernateProps' st-table=hibernateProperties)
                                                 tbody
-                                                    tr(ng-repeat='item in #{hibernateProperties}')
+                                                    tr(ng-repeat=`item in ${hibernateProperties}`)
                                                         td.col-sm-12(ng-hide='tableEditing(hibernatePropsTbl, $index)')
                                                             a.labelFormField(ng-click='tableStartEdit(backupItem, hibernatePropsTbl, $index)') {{item.name}} = {{item.value}}
                                                             +btn-remove('tableRemove(backupItem, hibernatePropsTbl, $index)', '"Remove Property"')
@@ -202,10 +202,10 @@ mixin hibernateField(name, model, items, valid, save, newItem)
 
 
                 .settings-row
-                    +checkbox('Keep binary in store', model + '.storeKeepBinary', '"storeKeepBinary"',
+                    +checkbox('Keep binary in store', `${model}.storeKeepBinary`, '"storeKeepBinary"',
                         'Flag indicating that CacheStore implementation is working with binary objects instead of Java objects')
                 .settings-row
-                    +checkbox('Load previous value', model + '.loadPreviousValue', '"loadPreviousValue"',
+                    +checkbox('Load previous value', `${model}.loadPreviousValue`, '"loadPreviousValue"',
                         'Flag indicating whether value should be loaded from store if it is not in the cache for following cache operations: \
                         <ul> \
                             <li>IgniteCache.putIfAbsent()</li> \
@@ -217,9 +217,9 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                             <li> IgniteCache.getAndPutIfAbsent()</li>\
                         </ul>')
                 .settings-row
-                    +checkbox('Read-through', model + '.readThrough', '"readThrough"', 'Flag indicating whether read-through caching should be used')
+                    +checkbox('Read-through', `${model}.readThrough`, '"readThrough"', 'Flag indicating whether read-through caching should be used')
                 .settings-row
-                    +checkbox('Write-through', model + '.writeThrough', '"writeThrough"', 'Flag indicating whether write-through caching should be used')
+                    +checkbox('Write-through', `${model}.writeThrough`, '"writeThrough"', 'Flag indicating whether write-through caching should be used')
                 .settings-row
                     +ignite-form-group
                         ignite-form-field-label
@@ -228,23 +228,23 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                             | Cache write-behind settings#[br]
                             | Write-behind is a special mode when updates to cache accumulated and then asynchronously flushed to persistent store as a bulk operation
                         .group-content
-                            -var enabled = model + '.writeBehindEnabled'
+                            -var enabled = `${model}.writeBehindEnabled`
 
                             .details-row
                                 +checkbox('Enabled', enabled, '"writeBehindEnabled"', 'Flag indicating whether Ignite should use write-behind behaviour for the cache store')
                             .details-row
-                                +number('Batch size:', model + '.writeBehindBatchSize', '"writeBehindBatchSize"', enabled, '512', '1',
+                                +number('Batch size:', `${model}.writeBehindBatchSize`, '"writeBehindBatchSize"', enabled, '512', '1',
                                     'Maximum batch size for write-behind cache store operations<br/>\
                                      Store operations(get or remove) are combined in a batch of this size to be passed to cache store')
                             .details-row
-                                +number('Flush size:', model + '.writeBehindFlushSize', '"writeBehindFlushSize"', enabled, '10240', '0',
+                                +number('Flush size:', `${model}.writeBehindFlushSize`, '"writeBehindFlushSize"', enabled, '10240', '0',
                                     'Maximum size of the write-behind cache<br/>\
                                      If cache size exceeds this value, all cached items are flushed to the cache store and write cache is cleared')
                             .details-row
-                                +number('Flush frequency:', model + '.writeBehindFlushFrequency', '"writeBehindFlushFrequency"', enabled, '5000', '0',
+                                +number('Flush frequency:', `${model}.writeBehindFlushFrequency`, '"writeBehindFlushFrequency"', enabled, '5000', '0',
                                     'Frequency with which write-behind cache is flushed to the cache store in milliseconds')
                             .details-row
-                                +number('Flush threads count:', model + '.writeBehindFlushThreadCount', '"writeBehindFlushThreadCount"', enabled, '1', '1',
+                                +number('Flush threads count:', `${model}.writeBehindFlushThreadCount`, '"writeBehindFlushThreadCount"', enabled, '1', '1',
                                     'Number of threads that will perform cache flushing')
             .col-sm-6
                 +preview-xml-java(model, 'cacheStore', 'domains')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.pug
similarity index 83%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.pug
index 412ca3ad538b8..61532ab3f54c6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'atomics'
 -var model = 'backupItem.atomicConfiguration'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Atomic configuration
         ignite-form-field-tooltip.tipLabel
@@ -28,11 +28,11 @@ include /app/helpers/jade/mixins.jade
             | Atomics are distributed across the cluster, essentially enabling performing atomic operations (such as increment-and-get or compare-and-set) with the same globally-visible value#[br]
             | #[a(href="https://apacheignite.readme.io/docs/atomic-types" target="_blank") More info]
         ignite-form-revert 
-    .panel-collapse(role='tabpanel' bs-collapse-target='' id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target='' id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Cache mode:', model + '.cacheMode', '"cacheMode"', 'true', 'PARTITIONED',
+                    +dropdown('Cache mode:', `${model}.cacheMode`, '"cacheMode"', 'true', 'PARTITIONED',
                         '[\
                             {value: "LOCAL", label: "LOCAL"},\
                             {value: "REPLICATED", label: "REPLICATED"},\
@@ -45,10 +45,10 @@ include /app/helpers/jade/mixins.jade
                             <li>Local - in this mode caches residing on different grid nodes will not know about each other</li>\
                         </ul>')
                 .settings-row
-                    +number('Sequence reserve:', model + '.atomicSequenceReserveSize', '"atomicSequenceReserveSize"', 'true', '1000', '0',
+                    +number('Sequence reserve:', `${model}.atomicSequenceReserveSize`, '"atomicSequenceReserveSize"', 'true', '1000', '0',
                         'Default number of sequence values reserved for IgniteAtomicSequence instances<br/>\
                         After a certain number has been reserved, consequent increments of sequence will happen locally, without communication with other nodes, until the next reservation has to be made')
-                .settings-row(ng-show='!(#{model}.cacheMode && #{model}.cacheMode != "PARTITIONED")')
+                .settings-row(ng-show=`!(${model}.cacheMode && ${model}.cacheMode != "PARTITIONED")`)
                     +number('Backups:', model + '.backups', '"backups"', 'true', '0', '0', 'Number of backup nodes')
             .col-sm-6
                 +preview-xml-java(model, 'clusterAtomics')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.pug
similarity index 80%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.pug
index 0366ed87deba4..cac122b6b00ef 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.pug
@@ -14,36 +14,36 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'attributes'
 -var model = 'backupItem'
 -var userAttributes = model + '.attributes'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label User attributes
         ignite-form-field-tooltip.tipLabel
             | Configuration for Ignite user attributes
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +ignite-form-group(ng-model='#{userAttributes}' ng-form='#{form}')
+                    +ignite-form-group(ng-model=`${userAttributes}` ng-form=form)
                         ignite-form-field-label
                             | User attributes
                         ignite-form-group-tooltip
                             | User-defined attributes to add to node
                         ignite-form-group-add(ng-click='tableNewItem(attributesTbl)')
                             | Add user attribute
-                        .group-content-empty(ng-if='!((#{userAttributes} && #{userAttributes}.length > 0) || tableNewItemActive(attributesTbl))')
+                        .group-content-empty(ng-if=`!((${userAttributes} && ${userAttributes}.length > 0) || tableNewItemActive(attributesTbl))`)
                             | Not defined
-                        .group-content(ng-show='(#{userAttributes} && #{userAttributes}.length > 0) || tableNewItemActive(attributesTbl)')
+                        .group-content(ng-show=`(${userAttributes} && ${userAttributes}.length > 0) || tableNewItemActive(attributesTbl)`)
                             table.links-edit(id='attributes' st-table=userAttributes)
                                 tbody
-                                    tr(ng-repeat='item in #{userAttributes} track by $index')
+                                    tr(ng-repeat=`item in ${userAttributes} track by $index`)
                                         td.col-sm-12(ng-hide='tableEditing(attributesTbl, $index)')
                                             a.labelFormField(ng-click='tableStartEdit(backupItem, attributesTbl, $index)') {{item.name}} = {{item.value}}
                                             +btn-remove('tableRemove(backupItem, attributesTbl, $index)', '"Remove attribute"')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
similarity index 92%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/binary.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
index f8ec8f9375b76..ee9f37bd1645b 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
@@ -14,22 +14,22 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'binary'
 -var model = 'backupItem.binaryConfiguration'
 -var types = model + '.typeConfigurations'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Binary configuration
         ignite-form-field-tooltip.tipLabel
             | Configuration of specific binary types#[br]
             | #[a(href="https://apacheignite.readme.io/docs/binary-marshaller" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +java-class('ID mapper:', model + '.idMapper', '"idMapper"', 'true', 'false',
@@ -45,11 +45,11 @@ include /app/helpers/jade/mixins.jade
                             | Type configurations
                         ignite-form-group-tooltip
                             | Configuration properties for binary types
-                        ignite-form-group-add(ng-click='#{types}.push({})')
+                        ignite-form-group-add(ng-click=`${types}.push({})`)
                             | Add new type configuration.
-                        .group-content-empty(ng-if='!#{types}.length')
+                        .group-content-empty(ng-if=`!${types}.length`)
                             | Not defined
-                        .group-content(ng-repeat='model in #{types} track by $index')
+                        .group-content(ng-repeat=`model in ${types} track by $index`)
                             hr(ng-if='$index !== 0')
                             .settings-row
                                 +java-class-autofocus('Type name:', 'model.typeName', '"typeName" + $index', 'true', 'true', 'true', 'Type name')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.pug
similarity index 83%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.pug
index 41a72c1c5a1b2..c2926f57f2836 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.pug
@@ -14,20 +14,20 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'cacheKeyCfg'
 -var model = 'backupItem.cacheKeyConfiguration'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Cache key configuration
         ignite-form-field-tooltip.tipLabel
             | Cache key configuration allows to collocate objects in a partitioned cache based on field in cache key without explicit usage of annotations on user classes
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +ignite-form-group()
@@ -35,11 +35,11 @@ include /app/helpers/jade/mixins.jade
                             | Cache key configuration
                         ignite-form-group-tooltip
                             | Cache key configuration
-                        ignite-form-group-add(ng-click='#{model}.push({})')
+                        ignite-form-group-add(ng-click=`${model}.push({})`)
                             | Add new cache key configuration
-                        .group-content-empty(ng-if='!#{model}.length')
+                        .group-content-empty(ng-if=`!${model}.length`)
                             | Not defined
-                        .group-content(ng-repeat='model in #{model} track by $index')
+                        .group-content(ng-repeat=`model in ${model} track by $index`)
                             hr(ng-if='$index !== 0')
                             .settings-row
                                 +java-class-autofocus('Type name:', 'model.typeName', '"cacheKeyTypeName" + $index', 'true', 'true', 'true', 'Type name')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.pug
similarity index 86%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.pug
index 259909eac9ed9..716e082c68f48 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'checkpoint'
 -var model = 'backupItem.checkpointSpi'
@@ -22,15 +22,15 @@ include /app/helpers/jade/mixins.jade
 -var CacheCheckpoint = 'model.kind === "Cache"'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Checkpointing
         ignite-form-field-tooltip.tipLabel
             | Checkpointing provides an ability to save an intermediate job state#[br]
             | #[a(href="http://apacheignite.readme.io/docs/checkpointing" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row(ng-init='checkpointSpiTbl={type: "checkpointSpi", model: "checkpointSpi", focusId: "kind", ui: "checkpoint-table"}')
                     +ignite-form-group()
@@ -40,9 +40,9 @@ include /app/helpers/jade/mixins.jade
                             | Checkpoint SPI configurations
                         ignite-form-group-add(ng-click='tableNewItem(checkpointSpiTbl)')
                             | Add checkpoint SPI
-                        .group-content-empty(ng-if='!(#{model} && #{model}.length > 0)')
+                        .group-content-empty(ng-if=`!(${model} && ${model}.length > 0)`)
                             | Not defined
-                        .group-content(ng-show='#{model} && #{model}.length > 0' ng-repeat='model in #{model} track by $index')
+                        .group-content(ng-show=`${model} && ${model}.length > 0` ng-repeat=`model in ${model} track by $index`)
                             hr(ng-if='$index != 0')
                             .settings-row
                                 +dropdown-required-autofocus('Checkpoint SPI:', 'model.kind', '"checkpointKind" + $index', 'true', 'true', 'Choose checkpoint configuration variant', '[\
@@ -63,21 +63,21 @@ include /app/helpers/jade/mixins.jade
                                     +table-remove-button(model, 'Remove Checkpoint SPI')
 
                             div(ng-show='model.kind === "FS"')
-                                include ./checkpoint/fs.jade
+                                include ./checkpoint/fs
 
                             div(ng-show=CacheCheckpoint)
                                 .settings-row
-                                    +dropdown-required-empty('Cache:', 'model.Cache.cache', '"checkpointCacheCache"+ $index', 'true', CacheCheckpoint,
+                                    +dropdown-required-empty('Cache:', 'model.Cache.cache', '"checkpointCacheCache" + $index', 'true', CacheCheckpoint,
                                         'Choose cache', 'No caches configured for current cluster', 'clusterCaches', 'Cache to use for storing checkpoints')
                                 .settings-row
                                     +java-class('Listener:', 'model.Cache.checkpointListener', '"checkpointCacheListener" + $index', 'true', 'false',
                                         'Checkpoint listener implementation class name', CacheCheckpoint)
 
                             div(ng-show='model.kind === "S3"')
-                                include ./checkpoint/s3.jade
+                                include ./checkpoint/s3
 
                             div(ng-show='model.kind === "JDBC"')
-                                include ./checkpoint/jdbc.jade
+                                include ./checkpoint/jdbc
 
                             .settings-row(ng-show=CustomCheckpoint)
                                 +java-class('Class name:', 'model.Custom.className', '"checkpointCustomClassName" + $index', 'true', CustomCheckpoint,
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.pug
similarity index 85%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.pug
index 6ec4535dbdcf5..04cc7fb27f48a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'checkpointFsPaths'
 -var dirPaths = 'model.FS.directoryPaths'
@@ -30,19 +30,19 @@ include /app/helpers/jade/mixins.jade
         ignite-form-group-add(ng-click='(group.add = [{}])')
             | Add new path
 
-        .group-content(ng-if='#{dirPaths}.length')
+        .group-content(ng-if=`${dirPaths}.length`)
             -var model = 'obj.model';
             -var name = '"edit" + $index'
-            -var valid = form + '[' + name + '].$valid'
+            -var valid = `${form}[${name}].$valid`
             -var save = dirPaths + '[$index] = ' + model
 
-            div(ng-repeat='item in #{dirPaths} track by $index' ng-init='obj = {}')
+            div(ng-repeat=`item in ${dirPaths} track by $index` ng-init='obj = {}')
                 label.col-xs-12.col-sm-12.col-md-12
                     .indexField
                         | {{ $index+1 }})
                     +table-remove-conditional-button(dirPaths, 'true', 'Remove path', 'item')
                     span(ng-hide='field.edit')
-                        a.labelFormField(ng-click='(field.edit = true) && (#{model} = item)') {{ item }}
+                        a.labelFormField(ng-click=`(field.edit = true) && (${model} = item)`) {{ item }}
                     span(ng-if='field.edit')
                         +table-text-field(name, model, dirPaths, valid, save, 'Input directory path', false)
                             +table-save-button(valid, save, false)
@@ -50,7 +50,7 @@ include /app/helpers/jade/mixins.jade
         .group-content(ng-repeat='field in group.add')
             -var model = 'new';
             -var name = '"new"'
-            -var valid = form + '[' + name + '].$valid'
+            -var valid = `${form}[${name}].$valid`
             -var save = dirPaths + '.push(' + model + ')'
 
             div
@@ -58,7 +58,7 @@ include /app/helpers/jade/mixins.jade
                     +table-text-field(name, model, dirPaths, valid, save, 'Input directory path', true)
                         +table-save-button(valid, save, true)
                         +unique-feedback(name, uniqueTip)
-        .group-content-empty(ng-if='!(#{dirPaths}.length) && !group.add.length')
+        .group-content-empty(ng-if=`!(${dirPaths}.length) && !group.add.length`)
             | Not defined
 
 .settings-row
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.pug
similarity index 98%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.pug
index 5a13337f564f1..ea67977e3629a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var jdbcCheckpoint = 'model.kind === "JDBC"'
 
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.pug
similarity index 99%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.pug
index 6531897130051..16be6c0a818a6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var credentialsModel = 'model.S3.awsCredentials'
 -var clientCfgModel = 'model.S3.clientConfiguration'
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision.pug
similarity index 75%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/collision.pug
index 91676df7cc1be..2f58e0a6d2aab 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision.pug
@@ -14,22 +14,22 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'collision'
 -var model = 'backupItem.collision'
 -var modelCollisionKind = model + '.kind';
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Collision configuration
         ignite-form-field-tooltip.tipLabel
             | Configuration Collision SPI allows to regulate how grid jobs get executed when they arrive on a destination node for execution#[br]
             | #[a(href="https://apacheignite.readme.io/docs/job-scheduling" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +dropdown('CollisionSpi:', modelCollisionKind, '"collisionKind"', 'true', '',
@@ -48,16 +48,16 @@ include /app/helpers/jade/mixins.jade
                             <li>Custom - custom CollisionSpi implementation</li>\
                             <li>Default - jobs are activated immediately on arrival to mapped node</li>\
                         </ul>')
-                .settings-row(ng-show='#{modelCollisionKind} !== "Noop"')
+                .settings-row(ng-show=`${modelCollisionKind} !== 'Noop'`)
                     .panel-details
-                        div(ng-show='#{modelCollisionKind} === "JobStealing"')
-                            include ./collision/job-stealing.jade
-                        div(ng-show='#{modelCollisionKind} === "FifoQueue"')
-                            include ./collision/fifo-queue.jade
-                        div(ng-show='#{modelCollisionKind} === "PriorityQueue"')
-                            include ./collision/priority-queue.jade
-                        div(ng-show='#{modelCollisionKind} === "Custom"')
-                            include ./collision/custom.jade
+                        div(ng-show=`${modelCollisionKind} === 'JobStealing'`)
+                            include ./collision/job-stealing
+                        div(ng-show=`${modelCollisionKind} === 'FifoQueue'`)
+                            include ./collision/fifo-queue
+                        div(ng-show=`${modelCollisionKind} === 'PriorityQueue'`)
+                            include ./collision/priority-queue
+                        div(ng-show=`${modelCollisionKind} === 'Custom'`)
+                            include ./collision/custom
             .col-sm-6
                 -var model = 'backupItem.collision'
                 +preview-xml-java(model, 'clusterCollision')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.pug
index 8e77ac42922c6..dc5dee09f7f6f 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.pug
@@ -14,11 +14,11 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.collision.Custom'
 -var required = 'backupItem.collision.kind === "Custom"'
 
 div
     .details-row
-        +java-class('Class:', model + '.class', '"collisionCustom"', 'true', required, 'CollisionSpi implementation class', required)
+        +java-class('Class:', `${model}.class`, '"collisionCustom"', 'true', required, 'CollisionSpi implementation class', required)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.pug
similarity index 78%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.pug
index cd8b6a3c92e5b..159b463ab9a89 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.collision.FifoQueue'
 
 div
     .details-row
-        +number('Parallel jobs number:', model + '.parallelJobsNumber', '"fifoParallelJobsNumber"', 'true', 'availableProcessors * 2', '1',
+        +number('Parallel jobs number:', `${model}.parallelJobsNumber`, '"fifoParallelJobsNumber"', 'true', 'availableProcessors * 2', '1',
             'Number of jobs that can be executed in parallel')
     .details-row
-        +number('Wait jobs number:', model + '.waitingJobsNumber', '"fifoWaitingJobsNumber"', 'true', 'Integer.MAX_VALUE', '0',
+        +number('Wait jobs number:', `${model}.waitingJobsNumber`, '"fifoWaitingJobsNumber"', 'true', 'Integer.MAX_VALUE', '0',
             'Maximum number of jobs that are allowed to wait in waiting queue')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.pug
similarity index 73%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.pug
index dbe0478344492..d10a02eb56e23 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.pug
@@ -14,29 +14,29 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.collision.JobStealing'
--var stealingAttributes = model + '.stealingAttributes'
+-var stealingAttributes = `${model}.stealingAttributes`
 
 div
     .details-row
-        +number('Active jobs threshold:', model + '.activeJobsThreshold', '"jsActiveJobsThreshold"', 'true', '95', '0',
+        +number('Active jobs threshold:', `${model}.activeJobsThreshold`, '"jsActiveJobsThreshold"', 'true', '95', '0',
             'Number of jobs that can be executed in parallel')
     .details-row
-        +number('Wait jobs threshold:', model + '.waitJobsThreshold', '"jsWaitJobsThreshold"', 'true', '0', '0',
+        +number('Wait jobs threshold:', `${model}.waitJobsThreshold`, '"jsWaitJobsThreshold"', 'true', '0', '0',
             'Job count threshold at which this node will start stealing jobs from other nodes')
     .details-row
-        +number('Message expire time:', model + '.messageExpireTime', '"jsMessageExpireTime"', 'true', '1000', '1',
+        +number('Message expire time:', `${model}.messageExpireTime`, '"jsMessageExpireTime"', 'true', '1000', '1',
             'Message expire time in ms')
     .details-row
-        +number('Maximum stealing attempts:', model + '.maximumStealingAttempts', '"jsMaximumStealingAttempts"', 'true', '5', '1',
+        +number('Maximum stealing attempts:', `${model}.maximumStealingAttempts`, '"jsMaximumStealingAttempts"', 'true', '5', '1',
             'Maximum number of attempts to steal job by another node')
     .details-row
-        +checkbox('Stealing enabled', model + '.stealingEnabled', '"jsStealingEnabled"',
+        +checkbox('Stealing enabled', `${model}.stealingEnabled`, '"jsStealingEnabled"',
             'Node should attempt to steal jobs from other nodes')
     .details-row
-        +java-class('External listener:', model + '.externalCollisionListener', '"jsExternalCollisionListener"', 'true', 'false',
+        +java-class('External listener:', `${model}.externalCollisionListener`, '"jsExternalCollisionListener"', 'true', 'false',
             'Listener to be set for notification of external collision events', 'backupItem.collision.kind === "JobStealing"')
     .details-row
         +ignite-form-group
@@ -46,12 +46,12 @@ div
                 | Configuration parameter to enable stealing to/from only nodes that have these attributes set
             ignite-form-group-add(ng-click='tableNewItem(stealingAttributesTbl)')
                 | Add stealing attribute
-            .group-content-empty(ng-if='!((#{stealingAttributes} && #{stealingAttributes}.length > 0) || tableNewItemActive(stealingAttributesTbl))')
+            .group-content-empty(ng-if=`!((${stealingAttributes} && ${stealingAttributes}.length > 0) || tableNewItemActive(stealingAttributesTbl))`)
                 | Not defined
-            .group-content(ng-show='(#{stealingAttributes} && #{stealingAttributes}.length > 0) || tableNewItemActive(stealingAttributesTbl)')
+            .group-content(ng-show=`(${stealingAttributes} && ${stealingAttributes}.length > 0) || tableNewItemActive(stealingAttributesTbl)`)
                 table.links-edit(id='attributes' st-table=stealingAttributes)
                     tbody
-                        tr(ng-repeat='item in #{stealingAttributes} track by $index')
+                        tr(ng-repeat=`item in ${stealingAttributes} track by $index`)
                             td.col-sm-12(ng-show='!tableEditing(stealingAttributesTbl, $index)')
                                 a.labelFormField(ng-click='tableStartEdit(backupItem, stealingAttributesTbl, $index)') {{item.name}} = {{item.value}}
                                 +btn-remove('tableRemove(backupItem, stealingAttributesTbl, $index)', '"Remove attribute"')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.pug
similarity index 64%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.pug
index 6f52ee0f07f19..04056df2d9c77 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.pug
@@ -14,29 +14,29 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.collision.PriorityQueue'
 
 div
     .details-row
-        +number('Parallel jobs number:', model + '.parallelJobsNumber', '"priorityParallelJobsNumber"', 'true', 'availableProcessors * 2', '1',
+        +number('Parallel jobs number:', `${model}.parallelJobsNumber`, '"priorityParallelJobsNumber"', 'true', 'availableProcessors * 2', '1',
             'Number of jobs that can be executed in parallel')
     .details-row
-        +number('Waiting jobs number:', model + '.waitingJobsNumber', '"priorityWaitingJobsNumber"', 'true', 'Integer.MAX_VALUE', '0',
+        +number('Waiting jobs number:', `${model}.waitingJobsNumber`, '"priorityWaitingJobsNumber"', 'true', 'Integer.MAX_VALUE', '0',
             'Maximum number of jobs that are allowed to wait in waiting queue')
     .details-row
-        +text('Priority attribute key:', model + '.priorityAttributeKey', '"priorityPriorityAttributeKey"', 'false', 'grid.task.priority',
+        +text('Priority attribute key:', `${model}.priorityAttributeKey`, '"priorityPriorityAttributeKey"', 'false', 'grid.task.priority',
             'Task priority attribute key')
     .details-row
-        +text('Job priority attribute key:', model + '.jobPriorityAttributeKey', '"priorityJobPriorityAttributeKey"', 'false', 'grid.job.priority',
+        +text('Job priority attribute key:', `${model}.jobPriorityAttributeKey`, '"priorityJobPriorityAttributeKey"', 'false', 'grid.job.priority',
             'Job priority attribute key')
     .details-row
-        +number('Default priority:', model + '.defaultPriority', '"priorityDefaultPriority"', 'true', '0', '0',
+        +number('Default priority:', `${model}.defaultPriority`, '"priorityDefaultPriority"', 'true', '0', '0',
             'Default priority to use if a job does not have priority attribute set')
     .details-row
-        +number('Starvation increment:', model + '.starvationIncrement', '"priorityStarvationIncrement"', 'true', '1', '0',
+        +number('Starvation increment:', `${model}.starvationIncrement`, '"priorityStarvationIncrement"', 'true', '1', '0',
             'Value to increment job priority by every time a lower priority job gets behind a higher priority job')
     .details-row
-        +checkbox('Starvation prevention enabled', model + '.starvationPreventionEnabled', '"priorityStarvationPreventionEnabled"',
+        +checkbox('Starvation prevention enabled', `${model}.starvationPreventionEnabled`, '"priorityStarvationPreventionEnabled"',
             'Job starvation prevention is enabled')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
similarity index 56%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/communication.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
index 047c9a28f5309..cd93565720e2b 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'communication'
 -var model = 'backupItem'
 -var communication = model + '.communication'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Communication
         ignite-form-field-tooltip.tipLabel
@@ -29,53 +29,53 @@ include /app/helpers/jade/mixins.jade
             | Provide basic plumbing to send and receive grid messages and is utilized for all distributed grid operations#[br]
             | #[a(href="https://apacheignite.readme.io/docs/network-config" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Timeout:', model + '.networkTimeout', '"commNetworkTimeout"', 'true', '5000', '1', 'Maximum timeout in milliseconds for network requests')
+                    +number('Timeout:', `${model}.networkTimeout`, '"commNetworkTimeout"', 'true', '5000', '1', 'Maximum timeout in milliseconds for network requests')
                 .settings-row
-                    +number('Send retry delay:', model + '.networkSendRetryDelay', '"networkSendRetryDelay"', 'true', '1000', '1', 'Interval in milliseconds between message send retries')
+                    +number('Send retry delay:', `${model}.networkSendRetryDelay`, '"networkSendRetryDelay"', 'true', '1000', '1', 'Interval in milliseconds between message send retries')
                 .settings-row
-                    +number('Send retry count:', model + '.networkSendRetryCount', '"networkSendRetryCount"', 'true', '3', '1', 'Message send retries count')
+                    +number('Send retry count:', `${model}.networkSendRetryCount`, '"networkSendRetryCount"', 'true', '3', '1', 'Message send retries count')
                 .settings-row
-                    +number('Discovery startup delay:', model + '.discoveryStartupDelay', '"discoveryStartupDelay"', 'true', '60000', '1', 'This value is used to expire messages from waiting list whenever node discovery discrepancies happen')
+                    +number('Discovery startup delay:', `${model}.discoveryStartupDelay`, '"discoveryStartupDelay"', 'true', '60000', '1', 'This value is used to expire messages from waiting list whenever node discovery discrepancies happen')
                 .settings-row
-                    +java-class('Communication listener:', communication + '.listener', '"comListener"', 'true', 'false', 'Listener of communication events')
+                    +java-class('Communication listener:', `${communication}.listener`, '"comListener"', 'true', 'false', 'Listener of communication events')
                 .settings-row
-                    +text-ip-address('Local IP address:', communication + '.localAddress', '"comLocalAddress"', 'true', '0.0.0.0',
+                    +text-ip-address('Local IP address:', `${communication}.localAddress`, '"comLocalAddress"', 'true', '0.0.0.0',
                         'Local host address for socket binding<br/>\
                         If not specified use all available addres on local host')
                 .settings-row
-                    +number-min-max('Local port:', communication + '.localPort', '"comLocalPort"', 'true', '47100', '1024', '65535', 'Local port for socket binding')
+                    +number-min-max('Local port:', `${communication}.localPort`, '"comLocalPort"', 'true', '47100', '1024', '65535', 'Local port for socket binding')
                 .settings-row
-                    +number('Local port range:', communication + '.localPortRange', '"comLocalPortRange"', 'true', '100', '1', 'Local port range for local host ports')
+                    +number('Local port range:', `${communication}.localPortRange`, '"comLocalPortRange"', 'true', '100', '1', 'Local port range for local host ports')
                 .settings-row
-                    +number-min-max('Shared memory port:', communication + '.sharedMemoryPort', '"sharedMemoryPort"', 'true', '48100', '-1', '65535',
+                    +number-min-max('Shared memory port:', `${communication}.sharedMemoryPort`, '"sharedMemoryPort"', 'true', '48100', '-1', '65535',
                         'Local port to accept shared memory connections<br/>\
                         If set to #[b -1] shared memory communication will be disabled')
                 .settings-row
-                    +number('Idle connection timeout:', communication + '.idleConnectionTimeout', '"idleConnectionTimeout"', 'true', '30000', '1',
+                    +number('Idle connection timeout:', `${communication}.idleConnectionTimeout`, '"idleConnectionTimeout"', 'true', '30000', '1',
                         'Maximum idle connection timeout upon which a connection to client will be closed')
                 .settings-row
-                    +number('Connect timeout:', communication + '.connectTimeout', '"connectTimeout"', 'true', '5000', '0', 'Connect timeout used when establishing connection with remote nodes')
+                    +number('Connect timeout:', `${communication}.connectTimeout`, '"connectTimeout"', 'true', '5000', '0', 'Connect timeout used when establishing connection with remote nodes')
                 .settings-row
-                    +number('Maximum connect timeout:', communication + '.maxConnectTimeout', '"maxConnectTimeout"', 'true', '600000', '0', 'Maximum connect timeout')
+                    +number('Maximum connect timeout:', `${communication}.maxConnectTimeout`, '"maxConnectTimeout"', 'true', '600000', '0', 'Maximum connect timeout')
                 .settings-row
-                    +number('Reconnect count:', communication + '.reconnectCount', '"comReconnectCount"', 'true', '10', '1',
+                    +number('Reconnect count:', `${communication}.reconnectCount`, '"comReconnectCount"', 'true', '10', '1',
                         'Maximum number of reconnect attempts used when establishing connection with remote nodes')
                 .settings-row
-                    +number('Socket send buffer:', communication + '.socketSendBuffer', '"socketSendBuffer"', 'true', '32768', '0', 'Send buffer size for sockets created or accepted by this SPI')
+                    +number('Socket send buffer:', `${communication}.socketSendBuffer`, '"socketSendBuffer"', 'true', '32768', '0', 'Send buffer size for sockets created or accepted by this SPI')
                 .settings-row
-                    +number('Socket receive buffer:', communication + '.socketReceiveBuffer', '"socketReceiveBuffer"', 'true', '32768', '0', 'Receive buffer size for sockets created or accepted by this SPI')
+                    +number('Socket receive buffer:', `${communication}.socketReceiveBuffer`, '"socketReceiveBuffer"', 'true', '32768', '0', 'Receive buffer size for sockets created or accepted by this SPI')
                 .settings-row
-                    +number('Slow client queue limit:', communication + '.slowClientQueueLimit', '"slowClientQueueLimit"', 'true', '0', '0', 'Slow client queue limit')
+                    +number('Slow client queue limit:', `${communication}.slowClientQueueLimit`, '"slowClientQueueLimit"', 'true', '0', '0', 'Slow client queue limit')
                 .settings-row
-                    +number('Ack send threshold:', communication + '.ackSendThreshold', '"ackSendThreshold"', 'true', '16', '1', 'Number of received messages per connection to node after which acknowledgment message is sent')
+                    +number('Ack send threshold:', `${communication}.ackSendThreshold`, '"ackSendThreshold"', 'true', '16', '1', 'Number of received messages per connection to node after which acknowledgment message is sent')
                 .settings-row
-                    +number('Message queue limit:', communication + '.messageQueueLimit', '"messageQueueLimit"', 'true', '1024', '0', 'Message queue limit for incoming and outgoing messages')
+                    +number('Message queue limit:', `${communication}.messageQueueLimit`, '"messageQueueLimit"', 'true', '1024', '0', 'Message queue limit for incoming and outgoing messages')
                 .settings-row
-                    +number('Unacknowledged messages:', communication + '.unacknowledgedMessagesBufferSize', '"unacknowledgedMessagesBufferSize"', 'true', '0', '0',
+                    +number('Unacknowledged messages:', `${communication}.unacknowledgedMessagesBufferSize`, '"unacknowledgedMessagesBufferSize"', 'true', '0', '0',
                         'Maximum number of stored unacknowledged messages per connection to node<br/>\
                         If specified non zero value it should be\
                         <ul>\
@@ -83,18 +83,18 @@ include /app/helpers/jade/mixins.jade
                             <li>At least message queue limit * 5</li>\
                         </ul>')
                 .settings-row
-                    +number('Socket write timeout:', communication + '.socketWriteTimeout', '"socketWriteTimeout"', 'true', '2000', '0', 'Socket write timeout')
+                    +number('Socket write timeout:', `${communication}.socketWriteTimeout`, '"socketWriteTimeout"', 'true', '2000', '0', 'Socket write timeout')
                 .settings-row
-                    +number('Selectors count:', communication + '.selectorsCount', '"selectorsCount"', 'true', 'min(4, availableProcessors)', '1', 'Count of selectors te be used in TCP server')
+                    +number('Selectors count:', `${communication}.selectorsCount`, '"selectorsCount"', 'true', 'min(4, availableProcessors)', '1', 'Count of selectors te be used in TCP server')
                 .settings-row
-                    +java-class('Address resolver:', communication + '.addressResolver', '"comAddressResolver"', 'true', 'false', 'Provides resolution between external and internal addresses')
+                    +java-class('Address resolver:', `${communication}.addressResolver`, '"comAddressResolver"', 'true', 'false', 'Provides resolution between external and internal addresses')
                 .settings-row
-                    +checkbox('Direct buffer', communication + '.directBuffer', '"directBuffer"',
+                    +checkbox('Direct buffer', `${communication}.directBuffer`, '"directBuffer"',
                     'If value is true, then SPI will use ByteBuffer.allocateDirect(int) call<br/>\
                     Otherwise, SPI will use ByteBuffer.allocate(int) call')
                 .settings-row
-                    +checkbox('Direct send buffer', communication + '.directSendBuffer', '"directSendBuffer"', 'Flag defining whether direct send buffer should be used')
+                    +checkbox('Direct send buffer', `${communication}.directSendBuffer`, '"directSendBuffer"', 'Flag defining whether direct send buffer should be used')
                 .settings-row
-                    +checkbox('TCP_NODELAY option', communication + '.tcpNoDelay', '"tcpNoDelay"', 'Value for TCP_NODELAY socket option')
+                    +checkbox('TCP_NODELAY option', `${communication}.tcpNoDelay`, '"tcpNoDelay"', 'Value for TCP_NODELAY socket option')
             .col-sm-6
                 +preview-xml-java(model, 'clusterCommunication')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/connector.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/connector.pug
similarity index 71%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/connector.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/connector.pug
index 40d65af250fce..b145ce28cfe26 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/connector.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/connector.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'connector'
 -var model = 'backupItem.connector'
@@ -22,83 +22,83 @@ include /app/helpers/jade/mixins.jade
 -var sslEnabled = enabled + ' && ' + model + '.sslEnabled'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Connector configuration
         ignite-form-field-tooltip.tipLabel
             | Configure HTTP REST configuration to enable HTTP server features#[br]
             | #[a(href="https://apacheignite.readme.io/docs/configuration" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +checkbox('Enabled', enabled, '"restEnabled"', 'Flag indicating whether to configure connector configuration')
                 .settings-row
-                    +text-enabled('Jetty configuration path:', model + '.jettyPath', '"connectorJettyPath"', enabled, 'false', 'Input path to Jetty configuration',
+                    +text-enabled('Jetty configuration path:', `${model}.jettyPath`, '"connectorJettyPath"', enabled, 'false', 'Input path to Jetty configuration',
                         'Path, either absolute or relative to IGNITE_HOME, to Jetty XML configuration file<br/>\
                         Jetty is used to support REST over HTTP protocol for accessing Ignite APIs remotely<br/>\
                         If not provided, Jetty instance with default configuration will be started picking IgniteSystemProperties.IGNITE_JETTY_HOST and IgniteSystemProperties.IGNITE_JETTY_PORT as host and port respectively')
                 .settings-row
-                    +text-ip-address('TCP host:', model + '.host', '"connectorHost"', enabled, 'IgniteConfiguration#getLocalHost()',
+                    +text-ip-address('TCP host:', `${model}.host`, '"connectorHost"', enabled, 'IgniteConfiguration#getLocalHost()',
                         'Host for TCP binary protocol server<br/>\
                         This can be either an IP address or a domain name<br/>\
                         If not defined, system - wide local address will be used IgniteConfiguration#getLocalHost()<br/>\
                         You can also use "0.0.0.0" value to bind to all locally - available IP addresses')
                 .settings-row
-                    +number-min-max('TCP port:', model + '.port', '"connectorPort"', enabled, '11211', '1024', '65535', 'Port for TCP binary protocol server')
+                    +number-min-max('TCP port:', `${model}.port`, '"connectorPort"', enabled, '11211', '1024', '65535', 'Port for TCP binary protocol server')
                 .settings-row
-                    +number('TCP port range:', model + '.portRange', '"connectorPortRange"', enabled, '100', '1', 'Number of ports for TCP binary protocol server to try if configured port is already in use')
+                    +number('TCP port range:', `${model}.portRange`, '"connectorPortRange"', enabled, '100', '1', 'Number of ports for TCP binary protocol server to try if configured port is already in use')
                 .settings-row
-                    +number('Idle query cursor timeout:', model + '.idleQueryCursorTimeout', '"connectorIdleQueryCursorTimeout"', enabled, '600000', '0',
+                    +number('Idle query cursor timeout:', `${model}.idleQueryCursorTimeout`, '"connectorIdleQueryCursorTimeout"', enabled, '600000', '0',
                         'Reject open query cursors that is not used timeout<br/>\
                         If no fetch query request come within idle timeout, it will be removed on next check for old query cursors')
                 .settings-row
-                    +number('Idle query cursor check frequency:', model + '.idleQueryCursorCheckFrequency', '"connectorIdleQueryCursorCheckFrequency"', enabled, '60000', '0',
+                    +number('Idle query cursor check frequency:', `${model}.idleQueryCursorCheckFrequency`, '"connectorIdleQueryCursorCheckFrequency"', enabled, '60000', '0',
                         'Idle query cursors check frequency<br/>\
                         This setting is used to reject open query cursors that is not used')
                 .settings-row
-                    +number('Idle timeout:', model + '.idleTimeout', '"connectorIdleTimeout"', enabled, '7000', '0',
+                    +number('Idle timeout:', `${model}.idleTimeout`, '"connectorIdleTimeout"', enabled, '7000', '0',
                         'Idle timeout for REST server<br/>\
                         This setting is used to reject half - opened sockets<br/>\
                         If no packets come within idle timeout, the connection is closed')
                 .settings-row
-                    +number('Receive buffer size:', model + '.receiveBufferSize', '"connectorReceiveBufferSize"', enabled, '32768', '0', 'REST TCP server receive buffer size')
+                    +number('Receive buffer size:', `${model}.receiveBufferSize`, '"connectorReceiveBufferSize"', enabled, '32768', '0', 'REST TCP server receive buffer size')
                 .settings-row
-                    +number('Send buffer size:', model + '.sendBufferSize', '"connectorSendBufferSize"', enabled, '32768', '0', 'REST TCP server send buffer size')
+                    +number('Send buffer size:', `${model}.sendBufferSize`, '"connectorSendBufferSize"', enabled, '32768', '0', 'REST TCP server send buffer size')
                 .settings-row
-                    +number('Send queue limit:', model + '.sendQueueLimit', '"connectorSendQueueLimit"', enabled, 'unlimited', '0',
+                    +number('Send queue limit:', `${model}.sendQueueLimit`, '"connectorSendQueueLimit"', enabled, 'unlimited', '0',
                         'REST TCP server send queue limit<br/>\
                         If the limit exceeds, all successive writes will block until the queue has enough capacity')
                 .settings-row
-                    +checkbox-enabled('Direct buffer', model + '.directBuffer', '"connectorDirectBuffer"', enabled,
+                    +checkbox-enabled('Direct buffer', `${model}.directBuffer`, '"connectorDirectBuffer"', enabled,
                         'Flag indicating whether REST TCP server should use direct buffers<br/>\
                         A direct buffer is a buffer that is allocated and accessed using native system calls, without using JVM heap<br/>\
                         Enabling direct buffer may improve performance and avoid memory issues(long GC pauses due to huge buffer size)')
                 .settings-row
-                    +checkbox-enabled('TCP_NODELAY option', model + '.noDelay', '"connectorNoDelay"', enabled,
+                    +checkbox-enabled('TCP_NODELAY option', `${model}.noDelay`, '"connectorNoDelay"', enabled,
                         'Flag indicating whether TCP_NODELAY option should be set for accepted client connections<br/>\
                         Setting this option reduces network latency and should be enabled in majority of cases<br/>\
                         For more information, see Socket#setTcpNoDelay(boolean)')
                 .settings-row
-                    +number('Selector count:', model + '.selectorCount', '"connectorSelectorCount"', enabled, 'min(4, availableProcessors)', '1',
+                    +number('Selector count:', `${model}.selectorCount`, '"connectorSelectorCount"', enabled, 'min(4, availableProcessors)', '1',
                         'Number of selector threads in REST TCP server<br/>\
                         Higher value for this parameter may increase throughput, but also increases context switching')
                 .settings-row
-                    +number('Thread pool size:', model + '.threadPoolSize', '"connectorThreadPoolSize"', enabled, 'max(8, availableProcessors) * 2', '1',
+                    +number('Thread pool size:', `${model}.threadPoolSize`, '"connectorThreadPoolSize"', enabled, 'max(8, availableProcessors) * 2', '1',
                         'Thread pool size to use for processing of client messages (REST requests)')
                 .settings-row
-                    +java-class('Message interceptor:', model + '.messageInterceptor', '"connectorMessageInterceptor"', enabled, 'false',
+                    +java-class('Message interceptor:', `${model}.messageInterceptor`, '"connectorMessageInterceptor"', enabled, 'false',
                         'Interceptor allows to transform all objects exchanged via REST protocol<br/>\
                         For example if you use custom serialisation on client you can write interceptor to transform binary representations received from client to Java objects and later access them from java code directly')
                 .settings-row
-                    +text-enabled('Secret key:', model + '.secretKey', '"connectorSecretKey"', enabled, 'false', 'Specify to enable authentication', 'Secret key to authenticate REST requests')
+                    +text-enabled('Secret key:', `${model}.secretKey`, '"connectorSecretKey"', enabled, 'false', 'Specify to enable authentication', 'Secret key to authenticate REST requests')
                 .settings-row
-                    +checkbox-enabled('Enable SSL', model + '.sslEnabled', '"connectorSslEnabled"', enabled, 'Enables/disables SSL for REST TCP binary protocol')
+                    +checkbox-enabled('Enable SSL', `${model}.sslEnabled`, '"connectorSslEnabled"', enabled, 'Enables/disables SSL for REST TCP binary protocol')
                 .settings-row
-                    +checkbox-enabled('Enable SSL client auth', model + '.sslClientAuth', '"connectorSslClientAuth"', sslEnabled, 'Flag indicating whether or not SSL client authentication is required')
+                    +checkbox-enabled('Enable SSL client auth', `${model}.sslClientAuth`, '"connectorSslClientAuth"', sslEnabled, 'Flag indicating whether or not SSL client authentication is required')
                 .settings-row
-                    +java-class('SSL factory:', model + '.sslFactory', '"connectorSslFactory"', sslEnabled, sslEnabled,
+                    +java-class('SSL factory:', `${model}.sslFactory`, '"connectorSslFactory"', sslEnabled, sslEnabled,
                         'Instance of Factory that will be used to create an instance of SSLContext for Secure Socket Layer on TCP binary protocol')
             .col-sm-6
                 +preview-xml-java(model, 'clusterConnector')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.pug
similarity index 78%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.pug
index aa99b491c0f1c..2021c1c93f4a6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'deployment'
 -var model = 'backupItem'
@@ -28,7 +28,7 @@ include /app/helpers/jade/mixins.jade
 -var customDeployment = modelDeployment + '.kind === "Custom"'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Class deployment
         ignite-form-field-tooltip.tipLabel
@@ -36,10 +36,10 @@ include /app/helpers/jade/mixins.jade
             | #[a(href="https://apacheignite.readme.io/docs/deployment-modes" target="_blank") More info]
         ignite-form-revert
     .panel-collapse(role='tabpanel' bs-collapse-target id='deployment')
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Deployment mode:', model + '.deploymentMode', '"deploymentMode"', 'true', 'SHARED',
+                    +dropdown('Deployment mode:', `${model}.deploymentMode`, '"deploymentMode"', 'true', 'SHARED',
                         '[\
                             {value: "PRIVATE", label: "PRIVATE"},\
                             {value: "ISOLATED", label: "ISOLATED"}, \
@@ -55,13 +55,13 @@ include /app/helpers/jade/mixins.jade
                             <li>CONTINUOUS - same as SHARED deployment mode, but resources will not be undeployed even after all master nodes left grid</li>\
                         </ul>')
                 .settings-row
-                    +checkbox('Enable peer class loading', model + '.peerClassLoadingEnabled', '"peerClassLoadingEnabled"', 'Enables/disables peer class loading')
+                    +checkbox('Enable peer class loading', `${model}.peerClassLoadingEnabled`, '"peerClassLoadingEnabled"', 'Enables/disables peer class loading')
                 .settings-row
-                    +number('Missed resources cache size:', model + '.peerClassLoadingMissedResourcesCacheSize', '"peerClassLoadingMissedResourcesCacheSize"', enabled, '100', '0',
+                    +number('Missed resources cache size:', `${model}.peerClassLoadingMissedResourcesCacheSize`, '"peerClassLoadingMissedResourcesCacheSize"', enabled, '100', '0',
                         'If size greater than 0, missed resources will be cached and next resource request ignored<br/>\
                         If size is 0, then request for the resource will be sent to the remote node every time this resource is requested')
                 .settings-row
-                    +number('Pool size:', model + '.peerClassLoadingThreadPoolSize', '"peerClassLoadingThreadPoolSize"', enabled, '2', '1', 'Thread pool size to use for peer class loading')
+                    +number('Pool size:', `${model}.peerClassLoadingThreadPoolSize`, '"peerClassLoadingThreadPoolSize"', enabled, '2', '1', 'Thread pool size to use for peer class loading')
                 .settings-row
                     +ignite-form-group
                         -var uniqueTip = 'Such package already exists'
@@ -71,30 +71,30 @@ include /app/helpers/jade/mixins.jade
                         ignite-form-group-tooltip
                             | List of packages from the system classpath that need to be peer-to-peer loaded from task originating node<br/>
                             | '*' is supported at the end of the package name which means that all sub-packages and their classes are included like in Java package import clause
-                        ignite-form-group-add(ng-show='#{enabled}' ng-click='(group.add = [{}])')
+                        ignite-form-group-add(ng-show=`${enabled}` ng-click='(group.add = [{}])')
                             | Add package name.
 
-                        .group-content(ng-if=exclude + '.length')
+                        .group-content(ng-if=`${exclude}.length`)
                             -var model = 'obj.model';
                             -var name = '"edit" + $index'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = exclude + '[$index] = ' + model
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${exclude}[$index] = ${model}`
 
                             div(ng-show=enabled)
-                                div(ng-repeat='model in #{exclude} track by $index' ng-init='obj = {}')
+                                div(ng-repeat=`model in ${exclude} track by $index` ng-init='obj = {}')
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .indexField
                                             | {{ $index+1 }})
                                         +table-remove-button(exclude, 'Remove package name')
                                         span(ng-hide='field.edit')
-                                            a.labelFormField(ng-click='(field.edit = true) && (#{model} = model)') {{ model }}
+                                            a.labelFormField(ng-click=`(field.edit = true) && (${model} = model)`) {{ model }}
                                         span(ng-if='field.edit')
                                             +table-java-package-field(name, model, exclude, valid, save, false)
                                                 +table-save-button(valid, save, false)
                                                 +unique-feedback(name, uniqueTip)
 
                             div(ng-hide=enabled)
-                                div(ng-repeat='model in #{exclude} track by $index')
+                                div(ng-repeat=`model in ${exclude} track by $index`)
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .labelFormField.labelField
                                             | {{ $index+1 }})
@@ -104,8 +104,8 @@ include /app/helpers/jade/mixins.jade
                         .group-content(ng-repeat='field in group.add')
                             -var model = 'new';
                             -var name = '"new"'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = exclude + '.push(' + model + ')'
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${exclude}.push(${model})`
 
                             div(type='internal' name='Package name')
                                 label.col-xs-12.col-sm-12.col-md-12
@@ -113,7 +113,7 @@ include /app/helpers/jade/mixins.jade
                                         +table-save-button(valid, save, true)
                                         +unique-feedback(name, uniqueTip)
 
-                        .group-content-empty(ng-if='!(#{exclude}.length) && !group.add.length')
+                        .group-content-empty(ng-if=`!(${exclude}.length) && !group.add.length`)
                             | Not defined
                 .settings-row
                     +dropdown('Deployment variant:', modelDeployment + '.kind', '"deploymentKind"', 'true', 'Default',
@@ -145,16 +145,16 @@ include /app/helpers/jade/mixins.jade
                             .group-content(ng-if=uriListModel + '.length')
                                 -var model = 'obj.model';
                                 -var name = '"edit" + $index'
-                                -var valid = form + '[' + name + '].$valid'
-                                -var save = uriListModel + '[$index] = ' + model
+                                -var valid = `${form}[${name}].$valid`
+                                -var save = `${uriListModel}[$index] = ${model}`
 
-                                div(ng-repeat='model in #{uriListModel} track by $index' ng-init='obj = {}')
+                                div(ng-repeat=`model in ${uriListModel} track by $index` ng-init='obj = {}')
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .indexField
                                             | {{ $index+1 }})
                                         +table-remove-button(uriListModel, 'Remove URI')
                                         span(ng-hide='field.edit')
-                                            a.labelFormField(ng-click='(field.edit = true) && (#{model} = model)') {{ model }}
+                                            a.labelFormField(ng-click=`(field.edit = true) && (${model} = model)`) {{ model }}
                                         span(ng-if='field.edit')
                                             +table-url-field(name, model, uriListModel, valid, save, false)
                                                 +table-save-button(valid, save, false)
@@ -163,8 +163,8 @@ include /app/helpers/jade/mixins.jade
                             .group-content(ng-repeat='field in group.add')
                                 -var model = 'new';
                                 -var name = '"new"'
-                                -var valid = form + '[' + name + '].$valid'
-                                -var save = uriListModel + '.push(' + model + ')'
+                                -var valid = `${form}[${name}].$valid`
+                                -var save = `${uriListModel}.push(${model})`
 
                                 div(type='internal' name='URI')
                                     label.col-xs-12.col-sm-12.col-md-12
@@ -172,7 +172,7 @@ include /app/helpers/jade/mixins.jade
                                             +table-save-button(valid, save, true)
                                             +unique-feedback(name, uniqueTip)
 
-                            .group-content-empty(ng-if='!(#{uriListModel}.length) && !group.add.length')
+                            .group-content-empty(ng-if=`!(${uriListModel}.length) && !group.add.length`)
                                 | Not defined
                     .details-row
                         +text('Temporary directory path:', modelDeployment + '.URI.temporaryDirectoryPath', '"DeploymentURITemporaryDirectoryPath"', 'false', 'Temporary directory path',
@@ -191,16 +191,16 @@ include /app/helpers/jade/mixins.jade
                             .group-content(ng-if=scannerModel + '.length')
                                 -var model = 'obj.model';
                                 -var name = '"edit" + $index'
-                                -var valid = form + '[' + name + '].$valid'
-                                -var save = scannerModel + '[$index] = ' + model
+                                -var valid = `${form}[${name}].$valid`
+                                -var save = `${scannerModel}[$index] = ${model}`
 
-                                div(ng-repeat='model in #{scannerModel} track by $index' ng-init='obj = {}')
+                                div(ng-repeat=`model in ${scannerModel} track by $index` ng-init='obj = {}')
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .indexField
                                             | {{ $index+1 }})
                                         +table-remove-button(scannerModel, 'Remove scanner')
                                         span(ng-hide='field.edit')
-                                            a.labelFormField(ng-click='(field.edit = true) && (#{model} = model)') {{ model }}
+                                            a.labelFormField(ng-click=`(field.edit = true) && (${model} = model)`) {{ model }}
                                         span(ng-if='field.edit')
                                             +table-java-class-field('Scanner:', name, model, scannerModel, valid, save, false)
                                                 +table-save-button(valid, save, false)
@@ -209,8 +209,8 @@ include /app/helpers/jade/mixins.jade
                             .group-content(ng-repeat='field in group.add')
                                 -var model = 'new';
                                 -var name = '"new"'
-                                -var valid = form + '[' + name + '].$valid'
-                                -var save = scannerModel + '.push(' + model + ')'
+                                -var valid = `${form}[${name}].$valid`
+                                -var save = `${scannerModel}.push(${model})`
 
                                 div(type='internal' name='Scanner')
                                     label.col-xs-12.col-sm-12.col-md-12
@@ -219,19 +219,19 @@ include /app/helpers/jade/mixins.jade
                                             +table-save-button(valid, save, true)
                                             +unique-feedback(name, uniqueTip)
 
-                            .group-content-empty(ng-if='!(#{scannerModel}.length) && !group.add.length')
+                            .group-content-empty(ng-if=`!(${scannerModel}.length) && !group.add.length`)
                                 | Not defined
                     .details-row
-                        +java-class('Listener:', modelDeployment + '.URI.listener', '"DeploymentURIListener"', 'true', 'false', 'Deployment event listener', uriDeployment)
+                        +java-class('Listener:', `${modelDeployment}.URI.listener`, '"DeploymentURIListener"', 'true', 'false', 'Deployment event listener', uriDeployment)
                     .details-row
-                        +checkbox('Check MD5', modelDeployment + '.URI.checkMd5', '"DeploymentURICheckMd5"', 'Exclude files with same md5s from deployment')
+                        +checkbox('Check MD5', `${modelDeployment}.URI.checkMd5`, '"DeploymentURICheckMd5"', 'Exclude files with same md5s from deployment')
                     .details-row
-                        +checkbox('Encode URI', modelDeployment + '.URI.encodeUri', '"DeploymentURIEncodeUri"', 'URI must be encoded before usage')
+                        +checkbox('Encode URI', `${modelDeployment}.URI.encodeUri`, '"DeploymentURIEncodeUri"', 'URI must be encoded before usage')
                 .panel-details(ng-show=localDeployment)
                     .details-row
-                        +java-class('Listener:', modelDeployment + '.Local.listener', '"DeploymentLocalListener"', 'true', 'false', 'Deployment event listener', localDeployment)
+                        +java-class('Listener:', `${modelDeployment}.Local.listener`, '"DeploymentLocalListener"', 'true', 'false', 'Deployment event listener', localDeployment)
                 .panel-details(ng-show=customDeployment)
                     .details-row
-                        +java-class('Class:', modelDeployment + '.Custom.className', '"DeploymentCustom"', 'true', customDeployment, 'DeploymentSpi implementation class', customDeployment)
+                        +java-class('Class:', `${modelDeployment}.Custom.className`, '"DeploymentCustom"', 'true', customDeployment, 'DeploymentSpi implementation class', customDeployment)
             .col-sm-6
                 +preview-xml-java(model, 'clusterDeployment')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.pug
similarity index 50%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.pug
index 3bf0e29440050..af9d875651fac 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.pug
@@ -14,75 +14,75 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'discovery'
 -var model = 'backupItem.discovery'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Discovery
         ignite-form-field-tooltip.tipLabel
             | TCP/IP discovery configuration#[br]
             | #[a(href="https://apacheignite.readme.io/docs/cluster-config" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +text-ip-address('Local address:', model + '.localAddress', '"discoLocalAddress"', 'true', '228.1.2.4',
+                    +text-ip-address('Local address:', `${model}.localAddress`, '"discoLocalAddress"', 'true', '228.1.2.4',
                         'Local host IP address that discovery SPI uses<br/>\
                         If not provided a first found non-loopback address will be used')
                 .settings-row
-                    +number-min-max('Local port:', model + '.localPort', '"discoLocalPort"', 'true', '47500', '1024', '65535', 'Local port which node uses')
+                    +number-min-max('Local port:', `${model}.localPort`, '"discoLocalPort"', 'true', '47500', '1024', '65535', 'Local port which node uses')
                 .settings-row
-                    +number('Local port range:', model + '.localPortRange', '"discoLocalPortRange"', 'true', '100', '1', 'Local port range')
+                    +number('Local port range:', `${model}.localPortRange`, '"discoLocalPortRange"', 'true', '100', '1', 'Local port range')
                 .settings-row
-                    +java-class('Address resolver:', model + '.addressResolver', '"discoAddressResolver"', 'true', 'false',
+                    +java-class('Address resolver:', `${model}.addressResolver`, '"discoAddressResolver"', 'true', 'false',
                         'Provides resolution between external and internal addresses')
                 .settings-row
-                    +number('Socket timeout:', model + '.socketTimeout', '"socketTimeout"', 'true', '5000', '0', 'Socket operations timeout')
+                    +number('Socket timeout:', `${model}.socketTimeout`, '"socketTimeout"', 'true', '5000', '0', 'Socket operations timeout')
                 .settings-row
-                    +number('Acknowledgement timeout:', model + '.ackTimeout', '"ackTimeout"', 'true', '5000', '0', 'Message acknowledgement timeout')
+                    +number('Acknowledgement timeout:', `${model}.ackTimeout`, '"ackTimeout"', 'true', '5000', '0', 'Message acknowledgement timeout')
                 .settings-row
-                    +number('Max acknowledgement timeout:', model + '.maxAckTimeout', '"maxAckTimeout"', 'true', '600000', '0', 'Maximum message acknowledgement timeout')
+                    +number('Max acknowledgement timeout:', `${model}.maxAckTimeout`, '"maxAckTimeout"', 'true', '600000', '0', 'Maximum message acknowledgement timeout')
                 .settings-row
-                    +number('Network timeout:', model + '.networkTimeout', '"discoNetworkTimeout"', 'true', '5000', '1', 'Timeout to use for network operations')
+                    +number('Network timeout:', `${model}.networkTimeout`, '"discoNetworkTimeout"', 'true', '5000', '1', 'Timeout to use for network operations')
                 .settings-row
-                    +number('Join timeout:', model + '.joinTimeout', '"joinTimeout"', 'true', '0', '0',
+                    +number('Join timeout:', `${model}.joinTimeout`, '"joinTimeout"', 'true', '0', '0',
                         'Join timeout<br/>' +
                         '0 means wait forever')
                 .settings-row
-                    +number('Thread priority:', model + '.threadPriority', '"threadPriority"', 'true', '10', '1', 'Thread priority for all threads started by SPI')
+                    +number('Thread priority:', `${model}.threadPriority`, '"threadPriority"', 'true', '10', '1', 'Thread priority for all threads started by SPI')
                 .settings-row
-                    +number('Heartbeat frequency:', model + '.heartbeatFrequency', '"heartbeatFrequency"', 'true', '2000', '1', 'Heartbeat messages issuing frequency')
+                    +number('Heartbeat frequency:', `${model}.heartbeatFrequency`, '"heartbeatFrequency"', 'true', '2000', '1', 'Heartbeat messages issuing frequency')
                 .settings-row
-                    +number('Max heartbeats miss w/o init:', model + '.maxMissedHeartbeats', '"maxMissedHeartbeats"', 'true', '1', '1',
+                    +number('Max heartbeats miss w/o init:', `${model}.maxMissedHeartbeats`, '"maxMissedHeartbeats"', 'true', '1', '1',
                         'Max heartbeats count node can miss without initiating status check')
                 .settings-row
-                    +number('Max missed client heartbeats:', model + '.maxMissedClientHeartbeats', '"maxMissedClientHeartbeats"', 'true', '5', '1',
+                    +number('Max missed client heartbeats:', `${model}.maxMissedClientHeartbeats`, '"maxMissedClientHeartbeats"', 'true', '5', '1',
                         'Max heartbeats count node can miss without failing client node')
                 .settings-row
-                    +number('Topology history:', model + '.topHistorySize', '"topHistorySize"', 'true', '1000', '0', 'Size of topology snapshots history')
+                    +number('Topology history:', `${model}.topHistorySize`, '"topHistorySize"', 'true', '1000', '0', 'Size of topology snapshots history')
                 .settings-row
-                    +java-class('Discovery listener:', model + '.listener', '"discoListener"', 'true', 'false', 'Listener for grid node discovery events')
+                    +java-class('Discovery listener:', `${model}.listener`, '"discoListener"', 'true', 'false', 'Listener for grid node discovery events')
                 .settings-row
-                    +java-class('Data exchange:', model + '.dataExchange', '"dataExchange"', 'true', 'false', 'Class name of handler for initial data exchange between Ignite nodes')
+                    +java-class('Data exchange:', `${model}.dataExchange`, '"dataExchange"', 'true', 'false', 'Class name of handler for initial data exchange between Ignite nodes')
                 .settings-row
-                    +java-class('Metrics provider:', model + '.metricsProvider', '"metricsProvider"', 'true', 'false', 'Class name of metric provider to discovery SPI')
+                    +java-class('Metrics provider:', `${model}.metricsProvider`, '"metricsProvider"', 'true', 'false', 'Class name of metric provider to discovery SPI')
                 .settings-row
-                    +number('Reconnect count:', model + '.reconnectCount', '"discoReconnectCount"', 'true', '10', '1', 'Reconnect attempts count')
+                    +number('Reconnect count:', `${model}.reconnectCount`, '"discoReconnectCount"', 'true', '10', '1', 'Reconnect attempts count')
                 .settings-row
-                    +number('Statistics frequency:', model + '.statisticsPrintFrequency', '"statisticsPrintFrequency"', 'true', '0', '1', 'Statistics print frequency')
+                    +number('Statistics frequency:', `${model}.statisticsPrintFrequency`, '"statisticsPrintFrequency"', 'true', '0', '1', 'Statistics print frequency')
                 .settings-row
-                    +number('IP finder clean frequency:', model + '.ipFinderCleanFrequency', '"ipFinderCleanFrequency"', 'true', '60000', '1', 'IP finder clean frequency')
+                    +number('IP finder clean frequency:', `${model}.ipFinderCleanFrequency`, '"ipFinderCleanFrequency"', 'true', '60000', '1', 'IP finder clean frequency')
                 .settings-row
-                    +java-class('Node authenticator:', model + '.authenticator', '"authenticator"', 'true', 'false', 'Class name of node authenticator implementation')
+                    +java-class('Node authenticator:', `${model}.authenticator`, '"authenticator"', 'true', 'false', 'Class name of node authenticator implementation')
                 .settings-row
-                    +checkbox('Force server mode', model + '.forceServerMode', '"forceServerMode"', 'Force start TCP/IP discovery in server mode')
+                    +checkbox('Force server mode', `${model}.forceServerMode`, '"forceServerMode"', 'Force start TCP/IP discovery in server mode')
                 .settings-row
-                    +checkbox('Client reconnect disabled', model + '.clientReconnectDisabled', '"clientReconnectDisabled"',
+                    +checkbox('Client reconnect disabled', `${model}.clientReconnectDisabled`, '"clientReconnectDisabled"',
                         'Disable try of client to reconnect after server detected client node failure')
             .col-sm-6
                 +preview-xml-java(model, 'clusterDiscovery')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/events.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/events.pug
similarity index 78%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/events.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/events.pug
index 643ea9703d230..70fb71494a0db 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/events.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/events.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'events'
 -var model = 'backupItem'
@@ -24,18 +24,18 @@ include /app/helpers/jade/mixins.jade
 -var eventStorageCustom = modelEventStorageKind + ' === "Custom"'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Events
         ignite-form-field-tooltip.tipLabel
             | Grid events are used for notification about what happens within the grid#[br]
             | #[a(href="https://apacheignite.readme.io/docs/events" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown-multiple('Include type:', model + '.includeEventTypes', '"includeEventTypes"', true, 'Choose recorded event types', '', 'eventGroups',
+                    +dropdown-multiple('Include type:', `${model}.includeEventTypes`, '"includeEventTypes"', true, 'Choose recorded event types', '', 'eventGroups',
                     'Array of event types, which will be recorded by GridEventStorageManager#record(Event)<br/>\
                     Note, that either the include event types or the exclude event types can be established')
 
@@ -53,16 +53,16 @@ include /app/helpers/jade/mixins.jade
 
                 div(ng-show=eventStorageMemory)
                     .settings-row
-                        +number('Events expiration time:', modelEventStorage + '.Memory.expireAgeMs', '"EventStorageExpireAgeMs"', 'true', 'Long.MAX_VALUE', '1', 'All events that exceed this value will be removed from the queue when next event comes')
+                        +number('Events expiration time:', `${modelEventStorage}.Memory.expireAgeMs`, '"EventStorageExpireAgeMs"', 'true', 'Long.MAX_VALUE', '1', 'All events that exceed this value will be removed from the queue when next event comes')
                     .settings-row
-                        +number('Events queue size:', modelEventStorage + '.Memory.expireCount', '"EventStorageExpireCount"', 'true', '10000', '1', 'Events will be filtered out when new request comes')
+                        +number('Events queue size:', `${modelEventStorage}.Memory.expireCount`, '"EventStorageExpireCount"', 'true', '10000', '1', 'Events will be filtered out when new request comes')
                     .settings-row
-                        +java-class('Filter:', modelEventStorage + '.Memory.filter', '"EventStorageFilter"', 'true', 'false',
+                        +java-class('Filter:', `${modelEventStorage}.Memory.filter`, '"EventStorageFilter"', 'true', 'false',
                         'Filter for events to be recorded<br/>\
                         Should be implementation of o.a.i.lang.IgnitePredicate&lt;o.a.i.events.Event&gt;', eventStorageMemory)
 
                 .settings-row(ng-show=eventStorageCustom)
-                    +java-class('Class:', modelEventStorage + '.Custom.className', '"EventStorageCustom"', 'true', eventStorageCustom, 'Event storage implementation class name', eventStorageCustom)
+                    +java-class('Class:', `${modelEventStorage}.Custom.className`, '"EventStorageCustom"', 'true', eventStorageCustom, 'Event storage implementation class name', eventStorageCustom)
 
             .col-sm-6
                 +preview-xml-java(model, 'clusterEvents')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/failover.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/failover.pug
index 16656598c2f99..e0c2357fedb65 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem'
 -var form = 'failoverSpi'
@@ -22,15 +22,15 @@ include /app/helpers/jade/mixins.jade
 -var failoverCustom = 'model.kind === "Custom"'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Failover configuration
         ignite-form-field-tooltip.tipLabel
             | Failover SPI provides ability to supply custom logic for handling failed execution of a grid job#[br]
             | #[a(href="https://apacheignite.readme.io/docs/fault-tolerance" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row(ng-init='failoverSpiTbl={type: "failoverSpi", model: "failoverSpi", focusId: "kind", ui: "failover-table"}')
                     +ignite-form-group()
@@ -40,9 +40,9 @@ include /app/helpers/jade/mixins.jade
                             | Failover SPI configurations
                         ignite-form-group-add(ng-click='tableNewItem(failoverSpiTbl)')
                             | Add failover SPI
-                        .group-content-empty(ng-if='!(#{failoverSpi} && #{failoverSpi}.length > 0)')
+                        .group-content-empty(ng-if=`!(${failoverSpi} && ${failoverSpi}.length > 0)`)
                             | Not defined
-                        .group-content(ng-show='#{failoverSpi} && #{failoverSpi}.length > 0' ng-repeat='model in #{failoverSpi} track by $index')
+                        .group-content(ng-show=`${failoverSpi} && ${failoverSpi}.length > 0` ng-repeat=`model in ${failoverSpi} track by $index`)
                             hr(ng-if='$index != 0')
                             .settings-row
                                 +dropdown-required-autofocus('Failover SPI:', 'model.kind', '"failoverKind" + $index', 'true', 'true', 'Choose Failover SPI', '[\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
similarity index 71%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
index b2ce71fa14e53..be56a6c44d08f 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var model = 'backupItem'
@@ -28,20 +28,20 @@ include /app/helpers/jade/mixins.jade
             | Common cluster configuration#[br]
             | #[a(href="https://apacheignite.readme.io/docs/clustering" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
         .panel-body
             .col-sm-6
                 .settings-row
-                    +text('Name:', model + '.name', '"clusterName"', 'true', 'Input name', 'Grid name allows to indicate to what grid this particular grid instance belongs to')
+                    +text('Name:', `${model}.name`, '"clusterName"', 'true', 'Input name', 'Grid name allows to indicate to what grid this particular grid instance belongs to')
                 .settings-row
                     +caches(model, 'Select caches to start in cluster or add a new cache')
                 .settings-row
-                    +text-ip-address('Local host:', model + '.localHost', '"localHost"', 'true', '0.0.0.0',
+                    +text-ip-address('Local host:', `${model}.localHost`, '"localHost"', 'true', '0.0.0.0',
                         'System-wide local address or host for all Ignite components to bind to<br/>\
                         If not defined then Ignite tries to use local wildcard address<br/>\
                         That means that all services will be available on all network interfaces of the host machine')
                 .settings-row
-                    +dropdown('Discovery:', model + '.discovery.kind', '"discovery"', 'true', 'Choose discovery', 'discoveries',
+                    +dropdown('Discovery:', `${model}.discovery.kind`, '"discovery"', 'true', 'Choose discovery', 'discoveries',
                         'Discovery allows to discover remote nodes in grid\
                         <ul>\
                             <li>Static IPs - IP Finder which works only with pre configured list of IP addresses specified</li>\
@@ -55,22 +55,22 @@ include /app/helpers/jade/mixins.jade
                         </ul>')
                 .settings-row
                     .panel-details
-                        div(ng-if='#{modelDiscoveryKind} === "Cloud"')
-                            include ./general/discovery/cloud.jade
-                        div(ng-if='#{modelDiscoveryKind} === "GoogleStorage"')
-                            include ./general/discovery/google.jade
-                        div(ng-if='#{modelDiscoveryKind} === "Jdbc"')
-                            include ./general/discovery/jdbc.jade
-                        div(ng-if='#{modelDiscoveryKind} === "Multicast"')
-                            include ./general/discovery/multicast.jade
-                        div(ng-if='#{modelDiscoveryKind} === "S3"')
-                            include ./general/discovery/s3.jade
-                        div(ng-if='#{modelDiscoveryKind} === "SharedFs"')
-                            include ./general/discovery/shared.jade
-                        div(ng-if='#{modelDiscoveryKind} === "Vm"')
-                            include ./general/discovery/vm.jade
-                        div(ng-if='#{modelDiscoveryKind} === "ZooKeeper"')
-                            include ./general/discovery/zookeeper.jade
+                        div(ng-if=`${modelDiscoveryKind} === 'Cloud'`)
+                            include ./general/discovery/cloud
+                        div(ng-if=`${modelDiscoveryKind} === 'GoogleStorage'`)
+                            include ./general/discovery/google
+                        div(ng-if=`${modelDiscoveryKind} === 'Jdbc'`)
+                            include ./general/discovery/jdbc
+                        div(ng-if=`${modelDiscoveryKind} === 'Multicast'`)
+                            include ./general/discovery/multicast
+                        div(ng-if=`${modelDiscoveryKind} === 'S3'`)
+                            include ./general/discovery/s3
+                        div(ng-if=`${modelDiscoveryKind} === 'SharedFs'`)
+                            include ./general/discovery/shared
+                        div(ng-if=`${modelDiscoveryKind} === 'Vm'`)
+                            include ./general/discovery/vm
+                        div(ng-if=`${modelDiscoveryKind} === 'ZooKeeper'`)
+                            include ./general/discovery/zookeeper
             .col-sm-6
                 -var model = 'backupItem'
                 +preview-xml-java(model, 'clusterCaches', 'caches')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.pug
similarity index 78%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.pug
index 52fb21b619ae7..b127a4c7a13c3 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.pug
@@ -14,34 +14,34 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var discoveryKind = 'Cloud'
 -var required = 'backupItem.discovery.kind == "' + discoveryKind + '"'
 -var model = 'backupItem.discovery.Cloud'
--var regions = model + '.regions'
--var zones = model + '.zones'
+-var regions = `${model}.regions`
+-var zones = `${model}.zones`
 -var formRegions = 'discoveryCloudRegions'
 -var formZones = 'discoveryCloudZones'
 
 div
     .details-row
-        +text('Credential:', model + '.credential', '"credential"', 'false', 'Input cloud credential',
+        +text('Credential:', `${model}.credential`, '"credential"', 'false', 'Input cloud credential',
             'Credential that is used during authentication on the cloud<br/>\
             Depending on a cloud platform it can be a password or access key')
     .details-row
-        +text('Path to credential:', model + '.credentialPath', '"credentialPath"', 'false', 'Input pathto credential',
+        +text('Path to credential:', `${model}.credentialPath`, '"credentialPath"', 'false', 'Input pathto credential',
             'Path to a credential that is used during authentication on the cloud<br/>\
             Access key or private key should be stored in a plain or PEM file without a passphrase')
     .details-row
-        +text('Identity:', model + '.identity', '"' + discoveryKind + 'Identity"', required, 'Input identity',
+        +text('Identity:', `${model}.identity`, '"' + discoveryKind + 'Identity"', required, 'Input identity',
             'Identity that is used as a user name during a connection to the cloud<br/>\
             Depending on a cloud platform it can be an email address, user name, etc')
     .details-row
-        +text('Provider:', model + '.provider', '"' + discoveryKind + 'Provider"', required, 'Input provider', 'Cloud provider to use')
+        +text('Provider:', `${model}.provider`, '"' + discoveryKind + 'Provider"', required, 'Input provider', 'Cloud provider to use')
     .details-row
         -var form = formRegions;
-        +ignite-form-group(ng-model=regions ng-form=form)
+        +ignite-form-group(ng-model=`${regions}` ng-form=form)
             -var uniqueTip = 'Such region already exists!'
 
             ignite-form-field-label
@@ -53,19 +53,19 @@ div
             ignite-form-group-add(ng-click='group.add = [{}]')
                 | Add new region
 
-            .group-content(ng-if='#{regions}.length')
+            .group-content(ng-if=`${regions}.length`)
                 -var model = 'field.model';
                 -var name = '"edit" + $index'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = regions + '[$index] = ' + model
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${regions}[$index] = ${model}`
 
-                div(ng-repeat='model in #{regions} track by $index')
+                div(ng-repeat=`model in ${regions} track by $index`)
                     label.col-xs-12.col-sm-12.col-md-12(ng-init='field = {}')
                         .indexField
                             | {{ $index+1 }})
                         +table-remove-button(regions, 'Remove region')
                         span(ng-hide='field.edit')
-                            a.labelFormField(ng-click='field.edit = true; #{model} = model;') {{ model }}
+                            a.labelFormField(ng-click=`field.edit = true; ${model} = model;`) {{ model }}
                         span(ng-if='field.edit')
                             +table-text-field(name, model, regions, valid, save, 'Region name', false)
                                 +table-save-button(valid, save, false)
@@ -74,8 +74,8 @@ div
             .group-content(ng-repeat='field in group.add')
                 -var model = 'field.new';
                 -var name = '"new"'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = regions + '.push(' + model + ')'
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${regions}.push(${model})`
 
                 div
                     label.col-xs-12.col-sm-12.col-md-12
@@ -83,7 +83,7 @@ div
                             +table-save-button(valid, save, false)
                             +unique-feedback(name, uniqueTip)
 
-            .group-content-empty(ng-if='!(#{regions}.length) && !group.add.length')
+            .group-content-empty(ng-if=`!(${regions}.length) && !group.add.length`)
                 | Not defined
     .details-row
         -var form = formZones;
@@ -100,19 +100,19 @@ div
                 | Add new zone
 
             -var form = formZones;
-            .group-content(ng-if='#{zones}.length')
+            .group-content(ng-if=`${zones}.length`)
                 -var model = 'field.model';
                 -var name = '"edit" + $index'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = zones + '[$index] = ' + model
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${zones}[$index] = ${model}`
 
-                div(ng-repeat='model in #{zones} track by $index')
+                div(ng-repeat=`model in ${zones} track by $index`)
                     label.col-xs-12.col-sm-12.col-md-12(ng-init='field = {}')
                         .indexField
                             | {{ $index+1 }})
                         +table-remove-button(zones, 'Remove zone')
                         span(ng-hide='field.edit')
-                            a.labelFormField(ng-click='field.edit = true; #{model} = model;') {{ model }}
+                            a.labelFormField(ng-click=`field.edit = true; ${model} = model;`) {{ model }}
                         span(ng-if='field.edit')
                             +table-text-field(name, model, zones, valid, save, 'Zone name', false)
                                 +table-save-button(valid, save, false)
@@ -121,8 +121,8 @@ div
             .group-content(ng-repeat='field in group.add')
                 -var model = 'field.new';
                 -var name = '"new"'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = zones + '.push(' + model + ')'
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${zones}.push(${model})`
 
                 div
                     label.col-xs-12.col-sm-12.col-md-12
@@ -130,5 +130,5 @@ div
                             +table-save-button(valid, save, true)
                             +unique-feedback(name, uniqueTip)
 
-            .group-content-empty(ng-if='!(#{zones}.length) && !group.add.length')
+            .group-content-empty(ng-if=`!(${zones}.length) && !group.add.length`)
                 | Not defined
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.pug
similarity index 68%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.pug
index c12bd04c8f55c..929350e5f7ecd 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.pug
@@ -14,25 +14,25 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 
 -var discoveryKind = 'GoogleStorage'
--var required = 'backupItem.discovery.kind == "' + discoveryKind + '"'
+-var required = `backupItem.discovery.kind == '${discoveryKind}'`
 -var model = 'backupItem.discovery.GoogleStorage'
 
 div
     .details-row
-        +text('Project name:', model + '.projectName', '"' + discoveryKind + 'ProjectName"', required, 'Input project name', '' +
+        +text('Project name:', `${model}.projectName`, `'${discoveryKind}ProjectName'`, required, 'Input project name', '' +
             'Google Cloud Platforms project name<br/>\
             Usually this is an auto generated project number(ex. 208709979073) that can be found in "Overview" section of Google Developer Console')
     .details-row
-        +text('Bucket name:', model + '.bucketName', '"' + discoveryKind + 'BucketName"', required, 'Input bucket name',
+        +text('Bucket name:', `${model}.bucketName`, `'${discoveryKind}BucketName'`, required, 'Input bucket name',
             'Google Cloud Storage bucket name<br/>\
             If the bucket does not exist Ignite will automatically create it<br/>\
             However the name must be unique across whole Google Cloud Storage and Service Account Id must be authorized to perform this operation')
     .details-row
-        +text('Private key path:', model + '.serviceAccountP12FilePath', '"' + discoveryKind + 'ServiceAccountP12FilePath"', required, 'Input private key path',
+        +text('Private key path:', `${model}.serviceAccountP12FilePath`, `'${discoveryKind}ServiceAccountP12FilePath'`, required, 'Input private key path',
             'Full path to the private key in PKCS12 format of the Service Account')
     .details-row
-        +text('Account id:', model + '.serviceAccountId', '"' + discoveryKind + 'ServiceAccountId"', required, 'Input account id', 'Service account ID (typically an e-mail address)')
+        +text('Account id:', `${model}.serviceAccountId`, `'${discoveryKind}ServiceAccountId'`, required, 'Input account id', 'Service account ID (typically an e-mail address)')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.pug
similarity index 82%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.pug
index 4e4246db22f61..4e9a430308af1 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.pug
@@ -14,18 +14,18 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.Jdbc'
 -var required = 'backupItem.discovery.kind === "Jdbc"'
 
 div
     .details-row
-        +checkbox('DB schema should be initialized by Ignite', model + '.initSchema', '"initSchema"',
+        +checkbox('DB schema should be initialized by Ignite', `${model}.initSchema`, '"initSchema"',
             'Flag indicating whether DB schema should be initialized by Ignite or was explicitly created by user')
     .details-row
-        +text('Data source bean name:', model + '.dataSourceBean',
+        +text('Data source bean name:', `${model}.dataSourceBean`,
             '"dataSourceBean"', required, 'Input bean name', 'Name of the data source bean in Spring context')
     .details-row
-        +dialect('Dialect:', model + '.dialect', '"dialect"', required,
+        +dialect('Dialect:', `${model}.dialect`, '"dialect"', required,
             'Dialect of SQL implemented by a particular RDBMS:', 'Generic JDBC dialect', 'Choose JDBC dialect')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.pug
similarity index 79%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.pug
index 6a7e9faf80a51..f865fd37829c7 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var model = 'backupItem.discovery.Multicast'
@@ -22,25 +22,25 @@ include /app/helpers/jade/mixins.jade
 
 div
     .details-row
-        +text-ip-address('IP address:', model + '.multicastGroup', '"multicastGroup"', 'true', '228.1.2.4', 'IP address of multicast group')
+        +text-ip-address('IP address:', `${model}.multicastGroup`, '"multicastGroup"', 'true', '228.1.2.4', 'IP address of multicast group')
     .details-row
-        +number-min-max('Port number:', model + '.multicastPort', '"multicastPort"', 'true', '47400', '0', '65535', 'Port number which multicast messages are sent to')
+        +number-min-max('Port number:', `${model}.multicastPort`, '"multicastPort"', 'true', '47400', '0', '65535', 'Port number which multicast messages are sent to')
     .details-row
-        +number('Waits for reply:', model + '.responseWaitTime', '"responseWaitTime"', 'true', '500', '0',
+        +number('Waits for reply:', `${model}.responseWaitTime`, '"responseWaitTime"', 'true', '500', '0',
             'Time in milliseconds IP finder waits for reply to multicast address request')
     .details-row
-        +number('Attempts count:', model + '.addressRequestAttempts', '"addressRequestAttempts"', 'true', '2', '0',
+        +number('Attempts count:', `${model}.addressRequestAttempts`, '"addressRequestAttempts"', 'true', '2', '0',
             'Number of attempts to send multicast address request<br/>\
             IP finder re - sends request only in case if no reply for previous request is received')
     .details-row
-        +text-ip-address('Local address:', model + '.localAddress', '"localAddress"', 'true', '0.0.0.0',
+        +text-ip-address('Local address:', `${model}.localAddress`, '"localAddress"', 'true', '0.0.0.0',
             'Local host address used by this IP finder<br/>\
             If provided address is non - loopback then multicast socket is bound to this interface<br/>\
             If local address is not set or is any local address then IP finder creates multicast sockets for all found non - loopback addresses')
     .details-row
         -var form = 'discoveryMulticastAddresses';
 
-        +ignite-form-group(ng-model=addresses ng-form=form)
+        +ignite-form-group(ng-model=`${addresses}` ng-form=form)
             -var uniqueTip = 'Such IP address already exists!'
             -var ipAddressTip = 'Invalid IP address!'
 
@@ -61,13 +61,13 @@ div
             ignite-form-group-add(ng-click='group.add = [{}]')
                 | Add new address
 
-            .group-content(ng-if='#{addresses}.length')
+            .group-content(ng-if=`${addresses}.length`)
                 -var model = 'obj.model';
                 -var name = '"edit" + $index'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = addresses + '[$index] = ' + model
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${addresses}[$index] = ${model}`
 
-                div(ng-repeat='model in #{addresses} track by $index' ng-init='obj = {}')
+                div(ng-repeat=`model in ${addresses} track by $index` ng-init='obj = {}')
                     label.col-xs-12.col-sm-12.col-md-12
                         .indexField
                             | {{ $index+1 }})
@@ -77,7 +77,7 @@ div
                         +ignite-form-field-up(ng-if='!$first' ng-hide='field.edit' data-ng-model='model' data-models=addresses)
 
                         span(ng-hide='field.edit')
-                            a.labelFormField(ng-click='field.edit = true; #{model} = model;') {{ model }}
+                            a.labelFormField(ng-click=`field.edit = true; ${model} = model;`) {{ model }}
                         span(ng-if='field.edit')
                             +table-address-field(name, model, addresses, valid, save, false, true)
                                 +table-save-button(valid, save, false)
@@ -86,8 +86,8 @@ div
             .group-content(ng-repeat='field in group.add')
                 -var model = 'new';
                 -var name = '"new"'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = addresses + '.push(' + model + ')'
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${addresses}.push(${model})`
 
                 div
                     label.col-xs-12.col-sm-12.col-md-12
@@ -95,5 +95,5 @@ div
                             +table-save-button(valid, save, true)
                             +unique-feedback(name, uniqueTip)
 
-            .group-content-empty(ng-if='!(#{addresses}.length) && !group.add.length')
+            .group-content-empty(ng-if=`!(${addresses}.length) && !group.add.length`)
                 | Not defined
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
similarity index 79%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
index e5eaff342adbd..ed6e20e4c3c1a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var discoveryKind = 'S3'
--var required = 'backupItem.discovery.kind == "' + discoveryKind + '"'
+-var required = `backupItem.discovery.kind == '${discoveryKind}'`
 -var model = 'backupItem.discovery.S3'
 
 div
     .details-row
-        +text('Bucket name:', model + '.bucketName', '"' + discoveryKind + 'BucketName"', required, 'Input bucket name', 'Bucket name for IP finder')
+        +text('Bucket name:', `${model}.bucketName`, `${discoveryKind}BucketName'`, required, 'Input bucket name', 'Bucket name for IP finder')
     .details-row
         label Note, AWS credentials will be generated as stub
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.pug
similarity index 89%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.pug
index ddd9bfa9b1f05..8eb97368b0389 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.pug
@@ -14,10 +14,10 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.SharedFs'
 
 div
     .details-row
-        +text('File path:', model + '.path', '"path"', 'false', 'disco/tcp', 'Shared path')
+        +text('File path:', `${model}.path`, '"path"', 'false', 'disco/tcp', 'Shared path')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.pug
similarity index 83%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.pug
index 46ebae005833d..9205c7248b3fa 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.Vm'
 -var addresses = model + '.addresses'
 -var form = 'discoveryVmAddresses'
 
 .details-row
-    +ignite-form-group(ng-form=form ng-model=addresses)
+    +ignite-form-group(ng-form=form ng-model=`${addresses}`)
         -var uniqueTip = 'Such IP address already exists!'
 
         ignite-form-field-label
@@ -41,13 +41,13 @@ include /app/helpers/jade/mixins.jade
         ignite-form-group-add(ng-click='group.add = [{}]')
             | Add new address
 
-        .group-content(ng-if='#{addresses}.length')
+        .group-content(ng-if=`${addresses}.length`)
             -var model = 'obj.model';
             -var name = '"edit" + $index'
-            -var valid = form + '[' + name + '].$valid'
-            -var save = addresses + '[$index] = ' + model
+            -var valid = `${form}[${name}].$valid`
+            -var save = `${addresses}[$index] = ${model}`
 
-            div(ng-repeat='model in #{addresses} track by $index' ng-init='obj = {}')
+            div(ng-repeat=`model in ${addresses} track by $index` ng-init='obj = {}')
                 label.col-xs-12.col-sm-12.col-md-12
                     .indexField
                         | {{ $index+1 }})
@@ -57,7 +57,7 @@ include /app/helpers/jade/mixins.jade
                     +ignite-form-field-up(ng-if='!$first' ng-hide='field.edit' data-ng-model='model' data-models=addresses)
 
                     span(ng-hide='field.edit')
-                        a.labelFormField(ng-click='field.edit = true; #{model} = model;') {{ model }}
+                        a.labelFormField(ng-click=`field.edit = true; ${model} = model;`) {{ model }}
                     span(ng-if='field.edit')
                         +table-address-field(name, model, addresses, valid, save, false, true)
                             +table-save-button(valid, save, false)
@@ -66,8 +66,8 @@ include /app/helpers/jade/mixins.jade
         .group-content(ng-repeat='field in group.add')
             -var model = 'new';
             -var name = '"new"'
-            -var valid = form + '[' + name + '].$valid'
-            -var save = addresses + '.push(' + model + ')'
+            -var valid = `${form}[${name}].$valid`
+            -var save = `${addresses}.push(${model})`
 
             div
                 label.col-xs-12.col-sm-12.col-md-12
@@ -75,5 +75,5 @@ include /app/helpers/jade/mixins.jade
                         +table-save-button(valid, save, true)
                         +unique-feedback(name, uniqueTip)
 
-        .group-content-empty(id='addresses' ng-if='!(#{addresses}.length) && !group.add.length')
+        .group-content-empty(id='addresses' ng-if=`!(${addresses}.length) && !group.add.length`)
                 | Not defined
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.pug
similarity index 70%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.pug
index 48b1776db2c35..91548e4de99d6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.pug
@@ -14,25 +14,25 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var discoveryKind = 'ZooKeeper'
--var required = 'backupItem.discovery.kind == "' + discoveryKind + '"'
+-var required = `backupItem.discovery.kind == '${discoveryKind}'`
 -var model = 'backupItem.discovery.ZooKeeper'
--var modelRetryPolicyKind = model + '.retryPolicy.kind'
+-var modelRetryPolicyKind = `${model}.retryPolicy.kind`
 
 div
     .details-row
-        +java-class('Curator:', model + '.curator', '"curator"', 'true', 'false',
+        +java-class('Curator:', `${model}.curator`, '"curator"', 'true', 'false',
             'The Curator framework in use<br/>\
             By default generates curator of org.apache.curator. framework.imps.CuratorFrameworkImpl\
             class with configured connect string, retry policy, and default session and connection timeouts', required)
     .details-row
-        +text('Connect string:', model + '.zkConnectionString', '"' + discoveryKind + 'ConnectionString"', required, 'host:port[chroot][,host:port[chroot]]',
+        +text('Connect string:', `${model}.zkConnectionString`, `'${discoveryKind}ConnectionString'`, required, 'host:port[chroot][,host:port[chroot]]',
             'When "IGNITE_ZK_CONNECTION_STRING" system property is not configured this property will be used')
     .details-row
-        +dropdown('Retry policy:', model + '.retryPolicy.kind', '"retryPolicy"', 'true', 'Default',
+        +dropdown('Retry policy:', `${model}.retryPolicy.kind`, '"retryPolicy"', 'true', 'Default',
             '[\
                 {value: "ExponentialBackoff", label: "Exponential backoff"},\
                 {value: "BoundedExponentialBackoff", label: "Bounded exponential backoff"},\
@@ -54,32 +54,32 @@ div
                 <li>Custom - custom retry policy implementation</li>\
                 <li>Default - exponential backoff retry policy with configured base sleep time equal to 1000ms and max retry count equal to 10</li>\
             </ul>')
-    .details-row(ng-show='#{model}.retryPolicy.kind')
+    .details-row(ng-show=`${model}.retryPolicy.kind`)
         .panel-details
-                div(ng-show='#{modelRetryPolicyKind} === "ExponentialBackoff"')
-                    include ./zookeeper/retrypolicy/exponential-backoff.jade
-                div(ng-show='#{modelRetryPolicyKind} === "BoundedExponentialBackoff"')
-                    include ./zookeeper/retrypolicy/bounded-exponential-backoff.jade
-                div(ng-show='#{modelRetryPolicyKind} === "UntilElapsed"')
-                    include ./zookeeper/retrypolicy/until-elapsed.jade
-                div(ng-show='#{modelRetryPolicyKind} === "NTimes"')
-                    include ./zookeeper/retrypolicy/n-times.jade
-                div(ng-show='#{modelRetryPolicyKind} === "OneTime"')
-                    include ./zookeeper/retrypolicy/one-time.jade
-                div(ng-show='#{modelRetryPolicyKind} === "Forever"')
-                    include ./zookeeper/retrypolicy/forever.jade
-                div(ng-show='#{modelRetryPolicyKind} === "Custom"')
-                    include ./zookeeper/retrypolicy/custom.jade
+                div(ng-show=`${modelRetryPolicyKind} === 'ExponentialBackoff'`)
+                    include ./zookeeper/retrypolicy/exponential-backoff
+                div(ng-show=`${modelRetryPolicyKind} === 'BoundedExponentialBackoff'`)
+                    include ./zookeeper/retrypolicy/bounded-exponential-backoff
+                div(ng-show=`${modelRetryPolicyKind} === 'UntilElapsed'`)
+                    include ./zookeeper/retrypolicy/until-elapsed
+                div(ng-show=`${modelRetryPolicyKind} === 'NTimes'`)
+                    include ./zookeeper/retrypolicy/n-times
+                div(ng-show=`${modelRetryPolicyKind} === 'OneTime'`)
+                    include ./zookeeper/retrypolicy/one-time
+                div(ng-show=`${modelRetryPolicyKind} === 'Forever'`)
+                    include ./zookeeper/retrypolicy/forever
+                div(ng-show=`${modelRetryPolicyKind} === 'Custom'`)
+                    include ./zookeeper/retrypolicy/custom
     .details-row
         -var model = 'backupItem.discovery.ZooKeeper'
 
-        +text('Base path:', model + '.basePath', '"basePath"', 'false', '/services', 'Base path for service registration')
+        +text('Base path:', `${model}.basePath`, '"basePath"', 'false', '/services', 'Base path for service registration')
     .details-row
-        +text('Service name:', model + '.serviceName', '"serviceName"', 'false', 'ignite',
+        +text('Service name:', `${model}.serviceName`, '"serviceName"', 'false', 'ignite',
             'Service name to use, as defined by Curator&#39;s ServiceDiscovery recipe<br/>\
             In physical ZooKeeper terms, it represents the node under basePath, under which services will be registered')
     .details-row
-        +checkbox('Allow duplicate registrations', model + '.allowDuplicateRegistrations', '"allowDuplicateRegistrations"',
+        +checkbox('Allow duplicate registrations', `${model}.allowDuplicateRegistrations`, '"allowDuplicateRegistrations"',
             'Whether to register each node only once, or if duplicate registrations are allowed<br/>\
             Nodes will attempt to register themselves, plus those they know about<br/>\
             By default, duplicate registrations are not allowed, but you might want to set this property to <b>true</b> if you have multiple network interfaces or if you are facing troubles')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
similarity index 84%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
index d3c1f9fc88fbc..c71be9a1915cf 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.BoundedExponentialBackoff'
 
 div
     .details-row
-        +number('Base interval:', model + '.baseSleepTimeMs', '"beBaseSleepTimeMs"', 'true', '1000', '0', 'Initial amount of time in ms to wait between retries')
+        +number('Base interval:', `${model}.baseSleepTimeMs`, '"beBaseSleepTimeMs"', 'true', '1000', '0', 'Initial amount of time in ms to wait between retries')
     .details-row
-        +number('Max interval:', model + '.maxSleepTimeMs', '"beMaxSleepTimeMs"', 'true', 'Integer.MAX_VALUE', '0', 'Max time in ms to sleep on each retry')
+        +number('Max interval:', `${model}.maxSleepTimeMs`, '"beMaxSleepTimeMs"', 'true', 'Integer.MAX_VALUE', '0', 'Max time in ms to sleep on each retry')
     .details-row
-        +number-min-max('Max retries:', model + '.maxRetries', '"beMaxRetries"', 'true', '10', '0', '29', 'Max number of times to retry')
+        +number-min-max('Max retries:', `${model}.maxRetries`, '"beMaxRetries"', 'true', '10', '0', '29', 'Max number of times to retry')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.pug
similarity index 89%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.pug
index 5db89f5162e55..884a7d61bfd81 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.pug
@@ -14,11 +14,11 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy'
--var retry = model + '.Custom'
+-var retry = `${model}.Custom`
 -var required = 'backupItem.discovery.kind === "ZooKeeper" && backupItem.discovery.ZooKeeper.retryPolicy.kind === "Custom"'
 
 .details-row
-    +java-class('Class name:', retry + '.className', '"customClassName"', 'true', required, 'Custom retry policy implementation class name', required)
+    +java-class('Class name:', `${retry}.className`, '"customClassName"', 'true', required, 'Custom retry policy implementation class name', required)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
similarity index 83%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
index 0a072f73301f3..93b2709136c7b 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.ExponentialBackoff'
 
 div
     .details-row
-        +number('Base interval:', model + '.baseSleepTimeMs', '"expBaseSleepTimeMs"', 'true', '1000', '0', 'Initial amount of time in ms to wait between retries')
+        +number('Base interval:', `${model}.baseSleepTimeMs`, '"expBaseSleepTimeMs"', 'true', '1000', '0', 'Initial amount of time in ms to wait between retries')
     .details-row
-        +number-min-max('Max retries:', model + '.maxRetries', '"expMaxRetries"', 'true', '10', '0', '29', 'Max number of times to retry')
+        +number-min-max('Max retries:', `${model}.maxRetries`, '"expMaxRetries"', 'true', '10', '0', '29', 'Max number of times to retry')
     .details-row
-        +number('Max interval:', model + '.maxSleepMs', '"expMaxSleepMs"', 'true', 'Integer.MAX_VALUE', '0', 'Max time in ms to sleep on each retry')
+        +number('Max interval:', `${model}.maxSleepMs`, '"expMaxSleepMs"', 'true', 'Integer.MAX_VALUE', '0', 'Max time in ms to sleep on each retry')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.pug
index a7bfd20b42171..2d7d2e8397f2f 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.pug
@@ -14,9 +14,9 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.Forever'
 
 .details-row
-    +number('Interval:', model + '.retryIntervalMs', '"feRetryIntervalMs"', 'true', '1000', '0', 'Time in ms between retry attempts')
+    +number('Interval:', `${model}.retryIntervalMs`, '"feRetryIntervalMs"', 'true', '1000', '0', 'Time in ms between retry attempts')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.pug
similarity index 86%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.pug
index 1746232124ee8..fdaead1b4201d 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.pug
@@ -14,12 +14,12 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.NTimes'
 
 div
     .details-row
-        +number('Retries:', model + '.n', '"n"', 'true', '10', '0', 'Number of times to retry')
+        +number('Retries:', `${model}.n`, '"n"', 'true', '10', '0', 'Number of times to retry')
     .details-row
-        +number('Interval:', model + '.sleepMsBetweenRetries', '"ntSleepMsBetweenRetries"', 'true', '1000', '0', 'Time in ms between retry attempts')
+        +number('Interval:', `${model}.sleepMsBetweenRetries`, '"ntSleepMsBetweenRetries"', 'true', '1000', '0', 'Time in ms between retry attempts')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.pug
index 5a7e3fe28ccec..53fc9be265d88 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.pug
@@ -14,10 +14,10 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.OneTime'
 
 div
     .details-row
-        +number('Interval:', model + '.sleepMsBetweenRetry', '"oneSleepMsBetweenRetry"', 'true', '1000', '0', 'Time in ms to retry attempt')
+        +number('Interval:', `${model}.sleepMsBetweenRetry`, '"oneSleepMsBetweenRetry"', 'true', '1000', '0', 'Time in ms to retry attempt')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
similarity index 86%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
index 548211c4ed2a9..16ddfc5277d94 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
@@ -14,12 +14,12 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.UntilElapsed'
 
 div
     .details-row
-        +number('Total time:', model + '.maxElapsedTimeMs', '"ueMaxElapsedTimeMs"', 'true', '60000', '0', 'Total time in ms for execution of retry attempt')
+        +number('Total time:', `${model}.maxElapsedTimeMs`, '"ueMaxElapsedTimeMs"', 'true', '60000', '0', 'Total time in ms for execution of retry attempt')
     .details-row
-        +number('Interval:', model + '.sleepMsBetweenRetries', '"ueSleepMsBetweenRetries"', 'true', '1000', '0', 'Time in ms between retry attempts')
+        +number('Interval:', `${model}.sleepMsBetweenRetries`, '"ueSleepMsBetweenRetries"', 'true', '1000', '0', 'Time in ms between retry attempts')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.pug
similarity index 84%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.pug
index 25bd5b0193168..8c20a45ea1fbe 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.pug
@@ -14,25 +14,25 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'igfs'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label IGFS
         ignite-form-field-tooltip.tipLabel
             | IGFS (Ignite In-Memory File System) configurations assigned to cluster#[br]
             | #[a(href="https://apacheignite-fs.readme.io/docs/in-memory-file-system" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +dropdown-multiple('<span>IGFS:</span><a ui-sref="base.configuration.igfs({linkId: linkId()})"> (add)</a>',
-                        model + '.igfss', '"igfss"', true, 'Choose IGFS', 'No IGFS configured', 'igfss',
+                        `${model}.igfss`, '"igfss"', true, 'Choose IGFS', 'No IGFS configured', 'igfss',
                         'Select IGFS to start in cluster or add a new IGFS')
             .col-sm-6
                 +preview-xml-java(model, 'igfss', 'igfss')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.pug
similarity index 94%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.pug
index 9fa9fc9300cf5..b80f43065351d 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem'
 -var form = 'loadBalancing'
@@ -23,15 +23,15 @@ include /app/helpers/jade/mixins.jade
 -var loadProbeCustom = 'model.kind === "Adaptive" && model.Adaptive.loadProbe.kind === "Custom"'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Load balancing configuration
         ignite-form-field-tooltip.tipLabel
             | Load balancing component balances job distribution among cluster nodes#[br]
             | #[a(href="https://apacheignite.readme.io/docs/load-balancing" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row(ng-init='loadBalancingSpiTbl={type: "loadBalancingSpi", model: "loadBalancingSpi", focusId: "kind", ui: "load-balancing-table"}')
                     +ignite-form-group()
@@ -41,9 +41,9 @@ include /app/helpers/jade/mixins.jade
                             | Load balancing component balances job distribution among cluster nodes
                         ignite-form-group-add(ng-click='tableNewItem(loadBalancingSpiTbl)')
                             | Add load balancing configuration
-                        .group-content-empty(ng-if='!(#{loadBalancingSpi} && #{loadBalancingSpi}.length > 0)')
+                        .group-content-empty(ng-if=`!(${loadBalancingSpi} && ${loadBalancingSpi}.length > 0)`)
                             | Not defined
-                        .group-content(ng-show='#{loadBalancingSpi} && #{loadBalancingSpi}.length > 0' ng-repeat='model in #{loadBalancingSpi} track by $index')
+                        .group-content(ng-show=`${loadBalancingSpi} && ${loadBalancingSpi}.length > 0` ng-repeat=`model in ${loadBalancingSpi} track by $index`)
                             hr(ng-if='$index != 0')
                             .settings-row
                                 +dropdown-required-autofocus('Load balancing:', 'model.kind', '"loadBalancingKind" + $index', 'true', 'true', 'Choose load balancing SPI', '[\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger.pug
similarity index 80%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/logger.pug
index 983d81855cba6..f1351141210b8 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger.pug
@@ -14,21 +14,21 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'logger'
 -var model = 'backupItem.logger'
 -var kind = model + '.kind'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Logger configuration
         ignite-form-field-tooltip.tipLabel
             | Logging functionality used throughout the system
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +dropdown('Logger:', kind, '"logger"', 'true', 'Default',
@@ -53,14 +53,14 @@ include /app/helpers/jade/mixins.jade
                             <li>Custom - custom logger implementation</li>\
                             <li>Default - Apache Log4j if awailable on classpath or Java logger otherwise</li>\
                         </ul>')
-                .settings-row(ng-show='#{kind} && (#{kind} === "Log4j2" || #{kind} === "Log4j" || #{kind} === "Custom")')
+                .settings-row(ng-show=`${kind} && (${kind} === 'Log4j2' || ${kind} === 'Log4j' || ${kind} === 'Custom')`)
                     .panel-details
-                        div(ng-show='#{kind} === "Log4j2"')
-                            include ./logger/log4j2.jade
-                        div(ng-show='#{kind} === "Log4j"')
-                            include ./logger/log4j.jade
-                        div(ng-show='#{kind} === "Custom"')
-                            include ./logger/custom.jade
+                        div(ng-show=`${kind} === 'Log4j2'`)
+                            include ./logger/log4j2
+                        div(ng-show=`${kind} === 'Log4j'`)
+                            include ./logger/log4j
+                        div(ng-show=`${kind} === 'Custom'`)
+                            include ./logger/custom
             .col-sm-6
                 -var model = 'backupItem.logger'
                 +preview-xml-java(model, 'clusterLogger')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.pug
index 87d2b7d7e56b9..9852e94fbd0ac 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'logger'
 -var model = 'backupItem.logger.Custom'
@@ -22,4 +22,4 @@ include /app/helpers/jade/mixins.jade
 
 div
     .details-row
-        +java-class('Class:', model + '.class', '"customLogger"', 'true', required, 'Logger implementation class name', required)
+        +java-class('Class:', `${model}.class`, '"customLogger"', 'true', required, 'Logger implementation class name', required)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.pug
similarity index 87%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.pug
index 1eaa52988955b..fadaea48a4f6a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'logger'
 -var model = 'backupItem.logger.Log4j'
@@ -22,7 +22,7 @@ include /app/helpers/jade/mixins.jade
 
 div
     .details-row
-        +dropdown('Level:', model + '.level', '"log4jLevel"', 'true', 'Default',
+        +dropdown('Level:', `${model}.level`, '"log4jLevel"', 'true', 'Default',
             '[\
                 {value: "OFF", label: "OFF"},\
                 {value: "FATAL", label: "FATAL"},\
@@ -36,7 +36,7 @@ div
             ]',
             'Level for internal log4j implementation')
     .details-row
-        +dropdown-required('Logger configuration:', model + '.mode', '"log4jMode"', 'true', 'true', 'Choose logger mode',
+        +dropdown-required('Logger configuration:', `${model}.mode`, '"log4jMode"', 'true', 'true', 'Choose logger mode',
             '[\
                 {value: "Default", label: "Default"},\
                 {value: "Path", label: "Path"}\
@@ -47,4 +47,4 @@ div
                 <li>Path - path or URI to XML configuration</li>\
             </ul>')
     .details-row(ng-show=pathRequired)
-        +text('Path:', model + '.path', '"log4jPath"', pathRequired, 'Input path', 'Path or URI to XML configuration')
+        +text('Path:', `${model}.path`, '"log4jPath"', pathRequired, 'Input path', 'Path or URI to XML configuration')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.pug
index 17df4811ff019..d90ca09dba2c6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'logger'
 -var model = 'backupItem.logger.Log4j2'
@@ -22,7 +22,7 @@ include /app/helpers/jade/mixins.jade
 
 div
     .details-row
-        +dropdown('Level:', model + '.level', '"log4j2Level"', 'true', 'Default',
+        +dropdown('Level:', `${model}.level`, '"log4j2Level"', 'true', 'Default',
             '[\
                 {value: "OFF", label: "OFF"},\
                 {value: "FATAL", label: "FATAL"},\
@@ -36,4 +36,4 @@ div
             ]',
             'Level for internal log4j2 implementation')
     .details-row
-        +text('Path:', model + '.path', '"log4j2Path"', log4j2Required, 'Input path', 'Path or URI to XML configuration')
+        +text('Path:', `${model}.path`, '"log4j2Path"', log4j2Required, 'Input path', 'Path or URI to XML configuration')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.pug
similarity index 76%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.pug
index 9316e7962fbed..f9d29b5ac7fcd 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'marshaller'
 -var model = 'backupItem'
@@ -22,7 +22,7 @@ include /app/helpers/jade/mixins.jade
 -var optMarshaller = marshaller + '.OptimizedMarshaller'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Marshaller
         ignite-form-field-tooltip.tipLabel
@@ -31,8 +31,8 @@ include /app/helpers/jade/mixins.jade
             | By default BinaryMarshaller will be used#[br]
             | #[a(href="https://apacheignite.readme.io/docs/binary-marshaller" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +dropdown('Marshaller:', marshaller + '.kind', '"kind"', 'true', 'Choose marshaller',
@@ -48,13 +48,13 @@ include /app/helpers/jade/mixins.jade
                             <li>Not set - BinaryMarshaller serialize and deserialize all objects in the binary format</li>\
                         </ul>')
                     a.customize(
-                        ng-if='#{marshaller}.kind && #{marshaller}.kind === "OptimizedMarshaller"'
-                        ng-click='#{marshaller}.expanded = !#{marshaller}.expanded'
-                    ) {{ #{marshaller}.expanded ? "Hide settings" : "Show settings"}}
+                        ng-if=`${marshaller}.kind && ${marshaller}.kind === 'OptimizedMarshaller'`
+                        ng-click=`${marshaller}.expanded = !${marshaller}.expanded`
+                    ) {{ #{marshaller}.expanded ? 'Hide settings' : 'Show settings'}}
                 .settings-row
-                    .panel-details(ng-show='#{marshaller}.expanded && #{marshaller}.kind === "OptimizedMarshaller"')
+                    .panel-details(ng-show=`${marshaller}.expanded && ${marshaller}.kind === 'OptimizedMarshaller'`)
                         .details-row
-                            +number('Streams pool size:', optMarshaller + '.poolSize', '"poolSize"', 'true', '0', '0',
+                            +number('Streams pool size:', `${optMarshaller}.poolSize`, '"poolSize"', 'true', '0', '0',
                                 'Specifies size of cached object streams used by marshaller<br/>\
                                 Object streams are cached for performance reason to avoid costly recreation for every serialization routine<br/>\
                                 If 0 (default), pool is not used and each thread has its own cached object stream which it keeps reusing<br/>\
@@ -62,15 +62,15 @@ include /app/helpers/jade/mixins.jade
                                 Consider using pool in this case. This will limit number of streams that can be created and, therefore, decrease memory consumption<br/>\
                                 NOTE: Using streams pool can decrease performance since streams will be shared between different threads which will lead to more frequent context switching')
                         .details-row
-                            +checkbox('Require serializable', optMarshaller + '.requireSerializable', '"requireSerializable"',
+                            +checkbox('Require serializable', `${optMarshaller}.requireSerializable`, '"requireSerializable"',
                                 'Whether marshaller should require Serializable interface or not')
                 .settings-row
-                    +checkbox('Marshal local jobs', model + '.marshalLocalJobs', '"marshalLocalJobs"', 'If this flag is enabled, jobs mapped to local node will be marshalled as if it was remote node')
+                    +checkbox('Marshal local jobs', `${model}.marshalLocalJobs`, '"marshalLocalJobs"', 'If this flag is enabled, jobs mapped to local node will be marshalled as if it was remote node')
                 .settings-row
-                    +number('Keep alive time:', model + '.marshallerCacheKeepAliveTime', '"marshallerCacheKeepAliveTime"', 'true', '10000', '0',
+                    +number('Keep alive time:', `${model}.marshallerCacheKeepAliveTime`, '"marshallerCacheKeepAliveTime"', 'true', '10000', '0',
                         'Keep alive time of thread pool that is in charge of processing marshaller messages')
                 .settings-row
-                    +number('Pool size:', model + '.marshallerCacheThreadPoolSize', '"marshallerCacheThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
+                    +number('Pool size:', `${model}.marshallerCacheThreadPoolSize`, '"marshallerCacheThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
                         'Default size of thread pool that is in charge of processing marshaller messages')
             .col-sm-6
                 +preview-xml-java(model, 'clusterMarshaller')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.pug
similarity index 73%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.pug
index d5212a94402fe..6810e6b298704 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.pug
@@ -14,33 +14,33 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'metrics'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Metrics
         ignite-form-field-tooltip.tipLabel
             | Cluster runtime metrics settings
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Elapsed time:', model + '.metricsExpireTime', '"metricsExpireTime"', 'true', 'Long.MAX_VALUE', '1',
+                    +number('Elapsed time:', `${model}.metricsExpireTime`, '"metricsExpireTime"', 'true', 'Long.MAX_VALUE', '1',
                         'Time in milliseconds after which a certain metric value is considered expired')
                 .settings-row
-                    +number('History size:', model + '.metricsHistorySize', '"metricsHistorySize"', 'true', '10000', '1',
+                    +number('History size:', `${model}.metricsHistorySize`, '"metricsHistorySize"', 'true', '10000', '1',
                         'Number of metrics kept in history to compute totals and averages')
                 .settings-row
-                    +number('Log frequency:', model + '.metricsLogFrequency', '"metricsLogFrequency"', 'true', '60000', '0',
+                    +number('Log frequency:', `${model}.metricsLogFrequency`, '"metricsLogFrequency"', 'true', '60000', '0',
                         'Frequency of metrics log print out<br/>\ ' +
                         'When <b>0</b> log print of metrics is disabled')
                 .settings-row
-                    +number('Update frequency:', model + '.metricsUpdateFrequency', '"metricsUpdateFrequency"', 'true', '2000', '0',
+                    +number('Update frequency:', `${model}.metricsUpdateFrequency`, '"metricsUpdateFrequency"', 'true', '2000', '0',
                         'Job metrics update frequency in milliseconds\
                         <ul>\
                             <li>If set to -1 job metrics are never updated</li>\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.pug
similarity index 83%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.pug
index 101a11c11caac..a69c7ad6844ba 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.pug
@@ -14,27 +14,27 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'odbcConfiguration'
 -var model = 'backupItem.odbc'
 -var enabled = model + '.odbcEnabled'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label ODBC configuration
         ignite-form-field-tooltip.tipLabel
             | ODBC server configuration#[br]
             | #[a(href="https://apacheignite.readme.io/docs/odbc-driver" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +checkbox('Enabled', enabled, '"odbcEnabled"', 'Flag indicating whether to configure ODBC configuration')
                 .settings-row
-                    +text-ip-address-with-port-range('ODBC endpoint address:', model + '.endpointAddress', '"endpointAddress"', enabled, '0.0.0.0:10800..10810',
+                    +text-ip-address-with-port-range('ODBC endpoint address:', `${model}.endpointAddress`, '"endpointAddress"', enabled, '0.0.0.0:10800..10810',
                         'ODBC endpoint address. <br/>\
                         The following address formats are permitted:\
                         <ul>\
@@ -43,6 +43,6 @@ include /app/helpers/jade/mixins.jade
                             <li>hostname:port_from..port_to - will use provided hostname and port range</li>\
                         </ul>')
                 .settings-row
-                    +number('Maximum open cursors', model + '.maxOpenCursors', '"maxOpenCursors"', enabled, '128', '1', 'Maximum number of opened cursors per connection')
+                    +number('Maximum open cursors', `${model}.maxOpenCursors`, '"maxOpenCursors"', enabled, '128', '1', 'Maximum number of opened cursors per connection')
             .col-sm-6
                 +preview-xml-java(model, 'clusterODBC')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.pug
similarity index 73%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.pug
index fbd979c954c05..dcb3b21c300a0 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'sslConfiguration'
 -var cluster = 'backupItem'
@@ -23,31 +23,31 @@ include /app/helpers/jade/mixins.jade
 -var trust = model + '.trustManagers'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label(id='sslConfiguration-title') SSL configuration
         ignite-form-field-tooltip.tipLabel
             | Settings for SSL configuration for creating a secure socket layer#[br]
             | #[a(href="https://apacheignite.readme.io/docs/ssltls" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +checkbox('Enabled', enabled, '"sslEnabled"', 'Flag indicating whether to configure SSL configuration')
                 .settings-row
-                    +text-options('Algorithm to create a key manager:', model + '.keyAlgorithm', '"keyAlgorithm"', '["SumX509", "X509"]', enabled, 'false', 'SumX509',
+                    +text-options('Algorithm to create a key manager:', `${model}.keyAlgorithm`, '"keyAlgorithm"', '["SumX509", "X509"]', enabled, 'false', 'SumX509',
                         'Sets key manager algorithm that will be used to create a key manager<br/>\
                         Notice that in most cased default value suites well, however, on Android platform this value need to be set to X509')
                 .settings-row
-                    +text-enabled('Key store file:', model + '.keyStoreFilePath', '"keyStoreFilePath"', enabled, enabled, 'Path to the key store file',
+                    +text-enabled('Key store file:', `${model}.keyStoreFilePath`, '"keyStoreFilePath"', enabled, enabled, 'Path to the key store file',
                         'Path to the key store file<br/>\
                         This is a mandatory parameter since ssl context could not be initialized without key manager')
                 .settings-row
-                    +text-options('Key store type:', model + '.keyStoreType', '"keyStoreType"', '["JKS", "PCKS11", "PCKS12"]', enabled, 'false', 'JKS',
+                    +text-options('Key store type:', `${model}.keyStoreType`, '"keyStoreType"', '["JKS", "PCKS11", "PCKS12"]', enabled, 'false', 'JKS',
                         'Key store type used in context initialization')
                 .settings-row
-                    +text-options('Protocol:', model + '.protocol', '"protocol"', '["TSL", "SSL"]', enabled, 'false', 'TSL', 'Protocol for secure transport')
+                    +text-options('Protocol:', `${model}.protocol`, '"protocol"', '["TSL", "SSL"]', enabled, 'false', 'TSL', 'Protocol for secure transport')
                 .settings-row
                     -var form = 'trustManagers'
 
@@ -58,29 +58,29 @@ include /app/helpers/jade/mixins.jade
                             | Trust managers
                         ignite-form-group-tooltip
                             | Pre-configured trust managers
-                        ignite-form-group-add(ng-show='#{enabled}' ng-click='(group.add = [{}])')
+                        ignite-form-group-add(ng-show=`${enabled}` ng-click='(group.add = [{}])')
                             | Add new trust manager
 
-                        .group-content(ng-if='#{trust}.length')
+                        .group-content(ng-if=`${trust}.length`)
                             -var model = 'obj.model';
                             -var name = '"edit" + $index'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = trust + '[$index] = ' + model
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${trust}[$index] = ${model}`
                 
                             div(ng-show=enabled)
-                                div(ng-repeat='model in #{trust} track by $index' ng-init='obj = {}')
+                                div(ng-repeat=`model in ${trust} track by $index` ng-init='obj = {}')
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .indexField
                                             | {{ $index+1 }})
                                         +table-remove-conditional-button(trust, enabled, 'Remove trust manager', 'model')
                                         span(ng-hide='field.edit')
-                                            a.labelFormField(ng-click='#{enabled} && (field.edit = true) && (#{model} = model)') {{ model }}
+                                            a.labelFormField(ng-click=`${enabled} && (field.edit = true) && (${model} = model)`) {{ model }}
                                         span(ng-if='field.edit')
                                             +table-java-class-field('Trust manager', name, model, trust, valid, save, false)
                                                 +table-save-button(valid, save, false)
                                                 +unique-feedback(name, uniqueTip)
                             div(ng-hide=enabled)
-                                div(ng-repeat='model in #{trust} track by $index')
+                                div(ng-repeat=`model in ${trust} track by $index`)
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .labelFormField.labelField
                                             | {{ $index+1 }})
@@ -90,8 +90,8 @@ include /app/helpers/jade/mixins.jade
                         .group-content(ng-repeat='field in group.add')
                             -var model = 'new';
                             -var name = '"new"'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = trust + '.push(' + model + ')'
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${trust}.push(${model})`
  
                             div
                                 label.col-xs-12.col-sm-12.col-md-12
@@ -99,12 +99,12 @@ include /app/helpers/jade/mixins.jade
                                         +table-save-button(valid, save, true)
                                         +unique-feedback(name, uniqueTip)
 
-                        .group-content-empty(ng-if='!(#{trust}.length) && !group.add.length')
+                        .group-content-empty(ng-if=`!(${trust}.length) && !group.add.length`)
                             | Not defined
 
-                .settings-row(ng-show='!#{trust}.length')
-                    +text-enabled('Trust store file:', model + '.trustStoreFilePath', '"trustStoreFilePath"', enabled, 'false', 'Path to the trust store file', 'Path to the trust store file')
-                .settings-row(ng-show='!#{trust}.length')
-                    +text-options('Trust store type:', model + '.trustStoreType', '"trustStoreType"', '["JKS", "PCKS11", "PCKS12"]', enabled, 'false', 'JKS', 'Trust store type used in context initialization')
+                .settings-row(ng-show=`!${trust}.length`)
+                    +text-enabled('Trust store file:', `${model}.trustStoreFilePath`, '"trustStoreFilePath"', enabled, 'false', 'Path to the trust store file', 'Path to the trust store file')
+                .settings-row(ng-show=`!${trust}.length`)
+                    +text-options('Trust store type:', `${model}.trustStoreType`, '"trustStoreType"', '["JKS", "PCKS11", "PCKS12"]', enabled, 'false', 'JKS', 'Trust store type used in context initialization')
             .col-sm-6
                 +preview-xml-java(cluster, 'clusterSsl')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/swap.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/swap.pug
similarity index 73%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/swap.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/swap.pug
index 8071f3c282a6b..54b6db07a708b 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/swap.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/swap.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'swap'
 -var model = 'backupItem'
@@ -22,18 +22,18 @@ include /app/helpers/jade/mixins.jade
 -var fileSwapModel = swapModel + '.FileSwapSpaceSpi'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Swap
         ignite-form-field-tooltip.tipLabel
             | Settings for overflow data to disk if it cannot fit in memory#[br]
             | #[a(href="https://apacheignite.readme.io/docs/off-heap-memory#swap-space" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Swap space SPI:', swapModel + '.kind', '"swapSpaceSpi"', 'true', 'Choose swap SPI',
+                    +dropdown('Swap space SPI:', `${swapModel}.kind`, '"swapSpaceSpi"', 'true', 'Choose swap SPI',
                         '[\
                             {value: "FileSwapSpaceSpi", label: "File-based swap"},\
                             {value: undefined, label: "Not set"}\
@@ -45,27 +45,27 @@ include /app/helpers/jade/mixins.jade
                             <li>Not set - File-based swap space SPI with default configuration when it needed</li>\
                         </ul>')
                     a.customize(
-                        ng-if='#{swapModel}.kind'
-                        ng-click='#{swapModel}.expanded = !#{swapModel}.expanded'
-                    ) {{ #{swapModel}.expanded ? "Hide settings" : "Show settings"}}
+                        ng-if=`${swapModel}.kind`
+                        ng-click=`${swapModel}.expanded = !${swapModel}.expanded`
+                    ) {{ #{swapModel}.expanded ? 'Hide settings' : 'Show settings'}}
                 .settings-row
-                    .panel-details(ng-show='#{swapModel}.expanded && #{swapModel}.kind')
+                    .panel-details(ng-show=`${swapModel}.expanded && ${swapModel}.kind`)
                         .details-row
-                            +text('Base directory:', fileSwapModel + '.baseDirectory', '"baseDirectory"', 'false', 'swapspace',
+                            +text('Base directory:', `${fileSwapModel}.baseDirectory`, '"baseDirectory"', 'false', 'swapspace',
                                 'Base directory where to write files')
                         .details-row
-                            +number('Read stripe size:', fileSwapModel + '.readStripesNumber', '"readStripesNumber"', 'true', 'availableProcessors', '0',
+                            +number('Read stripe size:', `${fileSwapModel}.readStripesNumber`, '"readStripesNumber"', 'true', 'availableProcessors', '0',
                                 'Read stripe size defines number of file channels to be used concurrently')
                         .details-row
-                            +number-min-max-step('Maximum sparsity:', fileSwapModel + '.maximumSparsity', '"maximumSparsity"', 'true', '0.5', '0', '0.999', '0.05',
+                            +number-min-max-step('Maximum sparsity:', `${fileSwapModel}.maximumSparsity`, '"maximumSparsity"', 'true', '0.5', '0', '0.999', '0.05',
                                 'This property defines maximum acceptable wasted file space to whole file size ratio<br/>\
                                 When this ratio becomes higher than specified number compacting thread starts working')
                         .details-row
-                            +number('Max write queue size:', fileSwapModel + '.maxWriteQueueSize', '"maxWriteQueueSize"', 'true', '1024 * 1024', '0',
+                            +number('Max write queue size:', `${fileSwapModel}.maxWriteQueueSize`, '"maxWriteQueueSize"', 'true', '1024 * 1024', '0',
                                 'Max write queue size in bytes<br/>\
                                 If there are more values are waiting for being written to disk then specified size, SPI will block on store operation')
                         .details-row
-                            +number('Write buffer size:', fileSwapModel + '.writeBufferSize', '"writeBufferSize"', 'true', '64 * 1024', '0',
+                            +number('Write buffer size:', `${fileSwapModel}.writeBufferSize`, '"writeBufferSize"', 'true', '64 * 1024', '0',
                                 'Write buffer size in bytes<br/>\
                                 Write to disk occurs only when this buffer is full')
             .col-sm-6
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/thread.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/thread.pug
similarity index 68%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/thread.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/thread.pug
index d2bea86587db1..31dd333ebdf6d 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/thread.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/thread.pug
@@ -14,35 +14,35 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'pools'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Thread pools size
         ignite-form-field-tooltip.tipLabel
             | Settings for node thread pools
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Public:', model + '.publicThreadPoolSize', '"publicThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
+                    +number('Public:', `${model}.publicThreadPoolSize`, '"publicThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
                         'Thread pool that is in charge of processing ComputeJob, GridJobs and user messages sent to node')
                 .settings-row
-                    +number('System:', model + '.systemThreadPoolSize', '"systemThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
+                    +number('System:', `${model}.systemThreadPoolSize`, '"systemThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
                         'Thread pool that is in charge of processing internal system messages')
                 .settings-row
-                    +number('Management:', model + '.managementThreadPoolSize', '"managementThreadPoolSize"', 'true', '4', '1',
+                    +number('Management:', `${model}.managementThreadPoolSize`, '"managementThreadPoolSize"', 'true', '4', '1',
                         'Thread pool that is in charge of processing internal and Visor ComputeJob, GridJobs')
                 .settings-row
-                    +number('IGFS:', model + '.igfsThreadPoolSize', '"igfsThreadPoolSize"', 'true', 'availableProcessors', '1',
+                    +number('IGFS:', `${model}.igfsThreadPoolSize`, '"igfsThreadPoolSize"', 'true', 'availableProcessors', '1',
                         'Thread pool that is in charge of processing outgoing IGFS messages')
                 .settings-row
-                    +number('Rebalance:', model + '.rebalanceThreadPoolSize', '"rebalanceThreadPoolSize"', 'true', '1', '1',
+                    +number('Rebalance:', `${model}.rebalanceThreadPoolSize`, '"rebalanceThreadPoolSize"', 'true', '1', '1',
                         'Max count of threads can be used at rebalancing')
             .col-sm-6
                 +preview-xml-java(model, 'clusterPools')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/time.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/time.pug
similarity index 73%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/time.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/time.pug
index bcb1cb71f5b53..da781baef08ed 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/time.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/time.pug
@@ -14,34 +14,34 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'time'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Time configuration
         ignite-form-field-tooltip.tipLabel
             | Time settings for CLOCK write ordering mode
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Samples size:', model + '.clockSyncSamples', '"clockSyncSamples"', 'true', '8', '0',
+                    +number('Samples size:', `${model}.clockSyncSamples`, '"clockSyncSamples"', 'true', '8', '0',
                         'Number of samples used to synchronize clocks between different nodes<br/>\
                         Clock synchronization is used for cache version assignment in CLOCK order mode')
                 .settings-row
-                    +number('Frequency:', model + '.clockSyncFrequency', '"clockSyncFrequency"', 'true', '120000', '0',
+                    +number('Frequency:', `${model}.clockSyncFrequency`, '"clockSyncFrequency"', 'true', '120000', '0',
                         'Frequency at which clock is synchronized between nodes, in milliseconds<br/>\
                         Clock synchronization is used for cache version assignment in CLOCK order mode')
                 .settings-row
-                    +number-min-max('Port base:', model + '.timeServerPortBase', '"timeServerPortBase"', 'true', '31100', '0', '65535',
+                    +number-min-max('Port base:', `${model}.timeServerPortBase`, '"timeServerPortBase"', 'true', '31100', '0', '65535',
                         'Time server provides clock synchronization between nodes<br/>\
                         Base UPD port number for grid time server. Time server will be started on one of free ports in range')
                 .settings-row
-                    +number('Port range:', model + '.timeServerPortRange', '"timeServerPortRange"', 'true', '100', '1', 'Time server port range')
+                    +number('Port range:', `${model}.timeServerPortRange`, '"timeServerPortRange"', 'true', '100', '1', 'Time server port range')
             .col-sm-6
                 +preview-xml-java(model, 'clusterTime')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.pug
similarity index 78%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.pug
index 6a18bb0e6a6f2..f60589f29ef0d 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.pug
@@ -14,24 +14,24 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'transactions'
 -var model = 'backupItem.transactionConfiguration'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Transactions
         ignite-form-field-tooltip.tipLabel
             | Settings for transactions#[br]
             | #[a(href="https://apacheignite.readme.io/docs/transactions" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Concurrency:', model + '.defaultTxConcurrency', '"defaultTxConcurrency"', 'true', 'PESSIMISTIC',
+                    +dropdown('Concurrency:', `${model}.defaultTxConcurrency`, '"defaultTxConcurrency"', 'true', 'PESSIMISTIC',
                         '[\
                             {value: "OPTIMISTIC", label: "OPTIMISTIC"},\
                             {value: "PESSIMISTIC", label: "PESSIMISTIC"}\
@@ -42,7 +42,7 @@ include /app/helpers/jade/mixins.jade
                             <li>PESSIMISTIC - A lock is acquired on all cache operations with exception of read operations in READ_COMMITTED mode</li>\
                         </ul>')
                 .settings-row
-                    +dropdown('Isolation:', model + '.defaultTxIsolation', '"defaultTxIsolation"', 'true', 'REPEATABLE_READ',
+                    +dropdown('Isolation:', `${model}.defaultTxIsolation`, '"defaultTxIsolation"', 'true', 'REPEATABLE_READ',
                         '[\
                             {value: "READ_COMMITTED", label: "READ_COMMITTED"},\
                             {value: "REPEATABLE_READ", label: "REPEATABLE_READ"},\
@@ -55,15 +55,15 @@ include /app/helpers/jade/mixins.jade
                             <li>SERIALIZABLE - All transactions occur in a completely isolated fashion, as if all transactions in the system had executed serially, one after the other.</li>\
                         </ul>')
                 .settings-row
-                    +number('Default timeout:', model + '.defaultTxTimeout', '"defaultTxTimeout"', 'true', '0', '0', 'Default transaction timeout')
+                    +number('Default timeout:', `${model}.defaultTxTimeout`, '"defaultTxTimeout"', 'true', '0', '0', 'Default transaction timeout')
                 .settings-row
-                    +number('Pessimistic log cleanup delay:', model + '.pessimisticTxLogLinger', '"pessimisticTxLogLinger"', 'true', '10000', '0',
+                    +number('Pessimistic log cleanup delay:', `${model}.pessimisticTxLogLinger`, '"pessimisticTxLogLinger"', 'true', '10000', '0',
                         'Delay, in milliseconds, after which pessimistic recovery entries will be cleaned up for failed node')
                 .settings-row
-                    +number('Pessimistic log size:', model + '.pessimisticTxLogSize', '"pessimisticTxLogSize"', 'true', '0', '0',
+                    +number('Pessimistic log size:', `${model}.pessimisticTxLogSize`, '"pessimisticTxLogSize"', 'true', '0', '0',
                         'Size of pessimistic transactions log stored on node in order to recover transaction commit if originating node has left grid before it has sent all messages to transaction nodes')
                 .settings-row
-                    +java-class('Manager factory:', model + '.txManagerFactory', '"txManagerFactory"', 'true', 'false',
+                    +java-class('Manager factory:', `${model}.txManagerFactory`, '"txManagerFactory"', 'true', 'false',
                         'Class name of transaction manager factory for integration with JEE app servers')
             .col-sm-6
                 +preview-xml-java(model, 'clusterTransactions')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/domains/general.jade b/modules/web-console/frontend/app/modules/states/configuration/domains/general.pug
similarity index 79%
rename from modules/web-console/frontend/app/modules/states/configuration/domains/general.jade
rename to modules/web-console/frontend/app/modules/states/configuration/domains/general.pug
index 40759e30c0748..6432adff97dcb 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/domains/general.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/domains/general.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var model = 'backupItem'
@@ -28,25 +28,25 @@ include /app/helpers/jade/mixins.jade
             | #[a(href="https://apacheignite.readme.io/docs/cache-queries" target="_blank") More info about query configuration]#[br]
             | #[a(href="https://apacheignite.readme.io/docs/persistent-store" target="_blank") More info about store]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
         .panel-body
             .col-sm-6
                 .settings-row
-                    +checkbox('Generate POJO classes', model + '.generatePojo', '"generatePojo"', 'If selected then POJO classes will be generated from database tables')
+                    +checkbox('Generate POJO classes', `${model}.generatePojo`, '"generatePojo"', 'If selected then POJO classes will be generated from database tables')
                 .settings-row
                     +caches(model, 'Select caches to associate domain model with cache')
                 .settings-row
-                    +dropdown-required('Query metadata:', model + '.queryMetadata', '"queryMetadata"', 'true', 'true', '', 'queryMetadataVariants',
+                    +dropdown-required('Query metadata:', `${model}.queryMetadata`, '"queryMetadata"', 'true', 'true', '', 'queryMetadataVariants',
                         'Query metadata configured with:\
                         <ul>\
                             <li>Java annotations like @QuerySqlField</li>\
                             <li>Configuration via QueryEntity class</li>\
                         </ul>')
-                -var generatePojo = model + '.generatePojo'
+                -var generatePojo = `${model}.generatePojo`
                 .settings-row
-                    +java-class-typeahead('Key type:', model + '.keyType', '"keyType"', 'javaBuiltInClasses', 'true', 'true', '{{ ' + generatePojo + ' ? "Full class name for Key" : "Key type name" }}', 'Key class used to store key in cache', generatePojo)
+                    +java-class-typeahead('Key type:', `${model}.keyType`, '"keyType"', 'javaBuiltInClasses', 'true', 'true', '{{ ' + generatePojo + ' ? "Full class name for Key" : "Key type name" }}', 'Key class used to store key in cache', generatePojo)
                 .settings-row
-                    +java-class-autofocus-placholder('Value type:', model + '.valueType', '"valueType"', 'true', 'true', 'false', '{{ ' + generatePojo +' ? "Enter fully qualified class name" : "Value type name" }}', 'Value class used to store value in cache', generatePojo)
+                    +java-class-autofocus-placholder('Value type:', `${model}.valueType`, '"valueType"', 'true', 'true', 'false', '{{ ' + generatePojo +' ? "Enter fully qualified class name" : "Value type name" }}', 'Value class used to store value in cache', generatePojo)
 
             .col-sm-6
                 +preview-xml-java(model, 'domainModelGeneral')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/domains/query.jade b/modules/web-console/frontend/app/modules/states/configuration/domains/query.pug
similarity index 77%
rename from modules/web-console/frontend/app/modules/states/configuration/domains/query.jade
rename to modules/web-console/frontend/app/modules/states/configuration/domains/query.pug
index 8ed7b6839c2c7..a057f59abbb2e 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/domains/query.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/domains/query.pug
@@ -14,42 +14,42 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'query'
 -var model = 'backupItem'
--var queryFields = model + '.fields'
--var queryAliases = model + '.aliases'
--var queryIndexes = model + '.indexes'
+-var queryFields = `${model}.fields`
+-var queryAliases = `${model}.aliases`
+-var queryIndexes = `${model}.indexes`
 -var queryFieldsForm = 'queryFields'
 -var queryAliasesForm = 'queryAliases'
 -var queryIndexesForm = 'queryIndexes'
 
 // LEGACY mixin for LEGACY index fields table.
 mixin table-index-item-edit(prefix, index, sortAvailable, idAddition)
-    -var fieldName = prefix + 'FieldName'
-    -var direction = prefix + 'Direction'
+    -var fieldName = `${prefix}FieldName`
+    -var direction = `${prefix}Direction`
 
-    -var fieldNameModel = 'indexesTbl.' + fieldName
-    -var directionModel = 'indexesTbl.' + direction
+    -var fieldNameModel = `indexesTbl.${fieldName}`
+    -var directionModel = `indexesTbl.${direction}`
 
-    -var btnVisible = 'tableIndexItemSaveVisible(indexesTbl, ' + index + ')'
-    -var btnSave = 'tableIndexItemSave(indexesTbl, itemIndex, ' + index + ')'
-    -var btnVisibleAndSave = btnVisible + ' && ' + btnSave
+    -var btnVisible = `tableIndexItemSaveVisible(indexesTbl, ${index})`
+    -var btnSave = `tableIndexItemSave(indexesTbl, itemIndex, ${index})`
+    -var btnVisibleAndSave = `${btnVisible} && ${btnSave}`
 
     div(ng-if=sortAvailable)
         .col-xs-8.col-sm-8.col-md-8
             label.fieldSep /
             .input-tip
-                button.select-toggle.form-control(id='{{::"#{fieldName}" + #{idAddition}}}' ignite-on-enter-focus-move='{{::"#{direction}S" + #{idAddition}}}' ng-model=fieldNameModel placeholder='{{fields("#{prefix}", #{fieldNameModel}).length > 0 ? "Choose field" : "No fields configured"}}' bs-select bs-options='item.value as item.label for item in fields("#{prefix}", #{fieldNameModel})' ng-disabled='fields("#{prefix}", #{fieldNameModel}).length === 0' ignite-on-escape='tableReset(false)' tabindex='0')
+                button.select-toggle.form-control(id=`{{::'${fieldName}' + ${idAddition}}}` ignite-on-enter-focus-move=`{{::'${direction}S' + ${idAddition}}}` ng-model=fieldNameModel placeholder=`{{fields('${prefix}', ${fieldNameModel}).length > 0 ? 'Choose field' : 'No fields configured'}}` bs-select bs-options=`item.value as item.label for item in fields('${prefix}', ${fieldNameModel})` ng-disabled=`fields('${prefix}', ${fieldNameModel}).length === 0` ignite-on-escape='tableReset(false)' tabindex='0')
         .col-xs-4.col-sm-4.col-md-4
             +btn-save(btnVisible, btnSave)
             .input-tip
-                button.select-toggle.form-control(id='{{::"#{direction}" + #{idAddition}}}' ng-model=directionModel bs-select bs-options='item.value as item.label for item in {{sortDirections}}' ignite-on-enter=btnVisibleAndSave ignite-on-escape='tableReset(false)' tabindex='0')
-    .col-xs-12(ng-if='!(#{sortAvailable})')
+                button.select-toggle.form-control(id=`{{::'${direction}' + ${idAddition}}}` ng-model=directionModel bs-select bs-options='item.value as item.label for item in {{sortDirections}}' ignite-on-enter=btnVisibleAndSave ignite-on-escape='tableReset(false)' tabindex='0')
+    .col-xs-12(ng-if=`!(${sortAvailable})`)
         +btn-save(btnVisible, btnSave)
         .input-tip
-            button.select-toggle.form-control(id='{{::"#{fieldName}" + #{idAddition}}}' ng-model=fieldNameModel placeholder='{{fields("#{prefix}", #{fieldNameModel}).length > 0 ? "Choose index field" : "No fields configured"}}' bs-select bs-options='item.value as item.label for item in fields("#{prefix}", #{fieldNameModel})' ng-disabled='fields("#{prefix}", #{fieldNameModel}).length === 0' ignite-on-escape='tableReset(false)' tabindex='0')
+            button.select-toggle.form-control(id=`{{::'${fieldName}' + ${idAddition}}}` ng-model=fieldNameModel placeholder=`{{fields('${prefix}', ${fieldNameModel}).length > 0 ? 'Choose index field' : 'No fields configured'}}` bs-select bs-options=`item.value as item.label for item in fields('${prefix}', ${fieldNameModel})` ng-disabled=`fields('${prefix}', ${fieldNameModel}).length === 0` ignite-on-escape='tableReset(false)' tabindex='0')
 
 .panel.panel-default(ng-form=form novalidate)
     .panel-heading(bs-collapse-toggle)
@@ -57,28 +57,28 @@ mixin table-index-item-edit(prefix, index, sortAvailable, idAddition)
         label(id='query-title') Domain model for SQL query
         ignite-form-field-tooltip.tipLabel
             | Domain model properties for fields queries#[br]
-            | #[a(href="https://apacheignite.readme.io/docs/cache-queries" target="_blank") More info]
+            | #[a(href='https://apacheignite.readme.io/docs/cache-queries' target='_blank') More info]
         ignite-form-revert
     .panel-collapse(role='tabpanel' bs-collapse-target id='query')
         .panel-body
             .col-sm-6
-                .content-not-available(ng-if='#{model}.queryMetadata === "Annotations"')
+                .content-not-available(ng-if=`${model}.queryMetadata === 'Annotations'`)
                     label Not available for annotated types
-                div(ng-if='#{model}.queryMetadata === "Configuration"')
+                div(ng-if=`${model}.queryMetadata === 'Configuration'`)
                     .settings-row
-                        +ignite-form-group(ng-model='#{queryFields}' ng-form='#{queryFieldsForm}')
+                        +ignite-form-group(ng-model=queryFields ng-form=queryFieldsForm)
                             ignite-form-field-label(id='queryFields')
                                 | Fields
                             ignite-form-group-tooltip
                                 | Collection of name-to-type mappings to be queried, in addition to indexed fields
                             ignite-form-group-add(ng-click='tableNewItem(queryFieldsTbl)')
                                 | Add field to query
-                            .group-content-empty(ng-if='!((#{queryFields} && #{queryFields}.length > 0) || tableNewItemActive(queryFieldsTbl))')
+                            .group-content-empty(ng-if=`!((${queryFields} && ${queryFields}.length > 0) || tableNewItemActive(queryFieldsTbl))`)
                                 | Not defined
-                            .group-content(ng-show='(#{queryFields} && #{queryFields}.length > 0) || tableNewItemActive(queryFieldsTbl)')
+                            .group-content(ng-show=`(${queryFields} && ${queryFields}.length > 0) || tableNewItemActive(queryFieldsTbl)`)
                                 table.links-edit(id='fields' st-table=queryFields)
                                     tbody
-                                        tr(ng-repeat='item in #{queryFields} track by $index')
+                                        tr(ng-repeat=`item in ${queryFields} track by $index`)
                                             td.col-sm-12(ng-hide='tableEditing(queryFieldsTbl, $index)')
                                                 a.labelFormField(ng-click='tableStartEdit(backupItem, queryFieldsTbl, $index)') {{item.name}}  / {{item.className}}
                                                 +btn-remove('tableRemove(backupItem, queryFieldsTbl, $index)', '"Remove path"')
@@ -89,7 +89,7 @@ mixin table-index-item-edit(prefix, index, sortAvailable, idAddition)
                                             td.col-sm-12
                                                 +table-pair-edit('queryFieldsTbl', 'new', 'Field name', 'Field full class name', false, true, '{{::queryFieldsTbl.focusId + $index}}', '-1', '/')
                     .settings-row
-                        +ignite-form-group(ng-model='#{queryAliases}' ng-form='#{queryAliasesForm}')
+                        +ignite-form-group(ng-model=queryAliases ng-form=queryAliasesForm)
                             ignite-form-field-label
                                 | Aliases
                             ignite-form-group-tooltip
@@ -97,12 +97,12 @@ mixin table-index-item-edit(prefix, index, sortAvailable, idAddition)
                                 | For example: "parent.name" as "parentName"
                             ignite-form-group-add(ng-click='tableNewItem(aliasesTbl)')
                                 | Add alias to query
-                            .group-content-empty(ng-if='!((#{queryAliases} && #{queryAliases}.length > 0) || tableNewItemActive(aliasesTbl))')
+                            .group-content-empty(ng-if=`!((${queryAliases} && ${queryAliases}.length > 0) || tableNewItemActive(aliasesTbl))`)
                                 | Not defined
-                            .group-content(ng-show='(#{queryAliases} && #{queryAliases}.length > 0) || tableNewItemActive(aliasesTbl)')
+                            .group-content(ng-show=`(${queryAliases} && ${queryAliases}.length > 0) || tableNewItemActive(aliasesTbl)`)
                                 table.links-edit(id='aliases' st-table=queryAliases)
                                     tbody
-                                        tr(ng-repeat='item in #{queryAliases} track by $index')
+                                        tr(ng-repeat=`item in ${queryAliases} track by $index`)
                                             td.col-sm-12(ng-hide='tableEditing(aliasesTbl, $index)')
                                                 a.labelFormField(ng-click='tableStartEdit(backupItem, aliasesTbl, $index)') {{item.field}} &rarr; {{item.alias}}
                                                 +btn-remove('tableRemove(backupItem, aliasesTbl, $index)', '"Remove alias"')
@@ -113,21 +113,21 @@ mixin table-index-item-edit(prefix, index, sortAvailable, idAddition)
                                             td.col-sm-12
                                                 +table-pair-edit('aliasesTbl', 'new', 'Field name', 'Field Alias', false, false, '{{::aliasesTbl.focusId + $index}}', '-1', '&rarr;')
                     .settings-row(ng-init='indexesTbl={type: "table-indexes", model: "indexes", focusId: "IndexName", ui: "table-indexes"}')
-                        +ignite-form-group(ng-model='#{queryIndexes}' ng-form='#{queryIndexesForm}')
+                        +ignite-form-group(ng-model=queryIndexes ng-form=queryIndexesForm)
                             ignite-form-field-label
                                 | Indexes
                             ignite-form-group-tooltip
                                 | Collection of indexes
                             ignite-form-group-add(ng-click='tableNewItem(indexesTbl)')
                                 | Add new index
-                            .group-content-empty(id='indexes-add' ng-show='!((#{queryIndexes} && #{queryIndexes}.length > 0) || tableNewItemActive(indexesTbl))')
+                            .group-content-empty(id='indexes-add' ng-show=`!((${queryIndexes} && ${queryIndexes}.length > 0) || tableNewItemActive(indexesTbl))`)
                                 | Not defined
-                            .group-content(ng-show='(#{queryIndexes} && #{queryIndexes}.length > 0) || tableNewItemActive(indexesTbl)')
+                            .group-content(ng-show=`(${queryIndexes} && ${queryIndexes}.length > 0) || tableNewItemActive(indexesTbl)`)
                                 -var btnVisibleAndSave = 'tableIndexSaveVisible(indexesTbl, $index) && tableIndexSave(indexesTbl, $index)'
 
                                 table.links-edit(st-table=queryIndexes ng-init='newDirection = false')
                                     tbody
-                                        tr(ng-repeat='item in #{queryIndexes} track by $index')
+                                        tr(ng-repeat=`item in ${queryIndexes} track by $index`)
                                             td
                                                 .col-sm-12(ng-hide='tableEditing(indexesTbl, $index)')
                                                     a.labelFormField(id='indexes{{$index}}' ng-click='tableStartEdit(backupItem, indexesTbl, $index)') {{$index + 1}}) {{item.name}} [{{item.indexType}}]
diff --git a/modules/web-console/frontend/app/modules/states/configuration/domains/store.jade b/modules/web-console/frontend/app/modules/states/configuration/domains/store.pug
similarity index 74%
rename from modules/web-console/frontend/app/modules/states/configuration/domains/store.jade
rename to modules/web-console/frontend/app/modules/states/configuration/domains/store.pug
index f5d6b7d075e5f..4dee98677950a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/domains/store.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/domains/store.pug
@@ -14,63 +14,63 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'store'
 -var model = 'backupItem'
--var keyFields = model + '.keyFields'
--var valueFields = model + '.valueFields'
+-var keyFields = `${model}.keyFields`
+-var valueFields = `${model}.valueFields`
 -var keyFieldsForm = 'storeKeyFields'
 -var valueFieldsForm = 'storeValueFields'
 
 //- LEGACY mixin for LEGACY db fields tables.
 mixin table-db-field-edit(tbl, prefix, focusId, index)
-    -var databaseName = prefix + 'DatabaseFieldName'
-    -var databaseType = prefix + 'DatabaseFieldType'
-    -var javaName = prefix + 'JavaFieldName'
-    -var javaType = prefix + 'JavaFieldType'
+    -var databaseName = `${prefix}DatabaseFieldName`
+    -var databaseType = `${prefix}DatabaseFieldType`
+    -var javaName = `${prefix}JavaFieldName`
+    -var javaType = `${prefix}JavaFieldType`
 
-    -var databaseNameModel = tbl + '.' + databaseName
-    -var databaseTypeModel = tbl + '.' + databaseType
-    -var javaNameModel = tbl + '.' + javaName
-    -var javaTypeModel = tbl + '.' + javaType
+    -var databaseNameModel = `${tbl}.${databaseName}`
+    -var databaseTypeModel = `${tbl}.${databaseType}`
+    -var javaNameModel = `${tbl}.${javaName}`
+    -var javaTypeModel = `${tbl}.${javaType}`
 
-    -var databaseNameId = databaseName + focusId
-    -var databaseTypeId = databaseType + focusId
-    -var javaNameId = javaName + focusId
-    -var javaTypeId = javaType + focusId
+    -var databaseNameId = `${databaseName}${focusId}`
+    -var databaseTypeId = `${databaseType}${focusId}`
+    -var javaNameId = `${javaName}${focusId}`
+    -var javaTypeId = `${javaType}${focusId}`
 
     .col-xs-3.col-sm-3.col-md-3
         .fieldSep /
         .input-tip
-            input.form-control(id=databaseNameId ignite-on-enter-focus-move=databaseTypeId type='text' ng-model=databaseNameModel placeholder='DB name' ignite-on-enter='#{javaNameModel} = #{javaNameModel} ? #{javaNameModel} : #{databaseNameModel}' ignite-on-escape='tableReset(false)')
+            input.form-control(id=databaseNameId ignite-on-enter-focus-move=databaseTypeId type='text' ng-model=databaseNameModel placeholder='DB name' ignite-on-enter=`${javaNameModel} = ${javaNameModel} ? ${javaNameModel} : ${databaseNameModel}` ignite-on-escape='tableReset(false)')
     .col-xs-3.col-sm-3.col-md-3
         .fieldSep /
         .input-tip
-            button.select-toggle.form-control(id=databaseTypeId ignite-on-enter-focus-move=javaNameId ng-model=databaseTypeModel data-placeholder='DB type' ng-class='{placeholder: !#{databaseTypeModel}}' bs-select bs-options='item.value as item.label for item in {{supportedJdbcTypes}}' ignite-on-escape='tableReset(false)' tabindex='0')
+            button.select-toggle.form-control(id=databaseTypeId ignite-on-enter-focus-move=javaNameId ng-model=databaseTypeModel data-placeholder='DB type' ng-class=`{placeholder: !${databaseTypeModel}}` bs-select bs-options='item.value as item.label for item in {{supportedJdbcTypes}}' ignite-on-escape='tableReset(false)' tabindex='0')
     .col-xs-3.col-sm-3.col-md-3
         .fieldSep /
         .input-tip
             input.form-control(id=javaNameId ignite-on-enter-focus-move=javaTypeId type='text' ng-model=javaNameModel placeholder='Java name' ignite-on-escape='tableReset(false)')
     .col-xs-3.col-sm-3.col-md-3
-        -var btnVisible = 'tableDbFieldSaveVisible(' + tbl + ', ' + index +')'
-        -var btnSave = 'tableDbFieldSave(' + tbl + ', ' + index +')'
-        -var btnVisibleAndSave = btnVisible + ' && ' + btnSave
+        -var btnVisible = `tableDbFieldSaveVisible(${tbl}, ${index})`
+        -var btnSave = `tableDbFieldSave(${tbl}, ${index})`
+        -var btnVisibleAndSave = `${btnVisible} && ${btnSave}`
 
         +btn-save(btnVisible, btnSave)
         .input-tip
-            button.select-toggle.form-control(id=javaTypeId ng-model=javaTypeModel data-placeholder='Java type' ng-class='{placeholder: !#{javaTypeModel}}' bs-select bs-options='item.value as item.label for item in {{supportedJavaTypes}}' ignite-on-enter=btnVisibleAndSave ignite-on-escape='tableReset(false)' tabindex='0')
+            button.select-toggle.form-control(id=javaTypeId ng-model=javaTypeModel data-placeholder='Java type' ng-class=`{placeholder: !${javaTypeModel}}` bs-select bs-options='item.value as item.label for item in {{supportedJavaTypes}}' ignite-on-enter=btnVisibleAndSave ignite-on-escape='tableReset(false)' tabindex='0')
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label(id='store-title') Domain model for cache store
         ignite-form-field-tooltip.tipLabel
             | Domain model properties for binding database with cache via POJO cache store#[br]
             | #[a(href="https://apacheignite.readme.io/docs/persistent-store" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +text('Database schema:', model + '.databaseSchema', '"databaseSchema"', 'false', 'Input DB schema name', 'Schema name in database')
@@ -84,11 +84,11 @@ mixin table-db-field-edit(tbl, prefix, focusId, index)
                             | Collection of key fields descriptions for CacheJdbcPojoStore
                         ignite-form-group-add(ng-click='tableNewItem(keysTbl)')
                             | Add key field
-                        .group-content-empty(ng-show='!((#{keyFields} && #{keyFields}.length > 0) || tableNewItemActive(keysTbl))') Not defined
-                        .group-content(ng-show='(#{keyFields} && #{keyFields}.length > 0) || tableNewItemActive(keysTbl)')
+                        .group-content-empty(ng-show=`!((${keyFields} && ${keyFields}.length > 0) || tableNewItemActive(keysTbl))`) Not defined
+                        .group-content(ng-show=`(${keyFields} && ${keyFields}.length > 0) || tableNewItemActive(keysTbl)`)
                             table.links-edit(st-table=keyFields)
                                 tbody
-                                    tr(ng-repeat='item in #{keyFields} track by $index')
+                                    tr(ng-repeat=`item in ${keyFields} track by $index`)
                                         td
                                             div(ng-hide='tableEditing(keysTbl, $index)')
                                                 a.labelFormField(ng-click='tableStartEdit(backupItem, keysTbl, $index)') {{$index + 1}}) {{item.databaseFieldName}} / {{item.databaseFieldType}} / {{item.javaFieldName}} / {{item.javaFieldType}}
@@ -107,11 +107,11 @@ mixin table-db-field-edit(tbl, prefix, focusId, index)
                             | Collection of value fields descriptions for CacheJdbcPojoStore
                         ignite-form-group-add(ng-click='tableNewItem(valuesTbl)')
                             | Add value field
-                        .group-content-empty(ng-show='!((#{valueFields} && #{valueFields}.length > 0) || tableNewItemActive(valuesTbl))') Not defined
-                        .group-content(ng-show='(#{valueFields} && #{valueFields}.length > 0) || tableNewItemActive(valuesTbl)')
+                        .group-content-empty(ng-show=`!((${valueFields} && ${valueFields}.length > 0) || tableNewItemActive(valuesTbl))`) Not defined
+                        .group-content(ng-show=`(${valueFields} && ${valueFields}.length > 0) || tableNewItemActive(valuesTbl)`)
                             table.links-edit(st-table=valueFields)
                                 tbody
-                                    tr(ng-repeat='item in #{valueFields} track by $index')
+                                    tr(ng-repeat=`item in ${valueFields} track by $index`)
                                         td
                                             div(ng-hide='tableEditing(valuesTbl, $index)')
                                                 a.labelFormField(ng-click='tableStartEdit(backupItem, valuesTbl, $index)') {{$index + 1}}) {{item.databaseFieldName}} / {{item.databaseFieldType}} / {{item.javaFieldName}} / {{item.javaFieldType}}
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/dual.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/dual.pug
similarity index 75%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/dual.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/dual.pug
index f6ac89f223102..613070cbcdc44 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/dual.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/dual.pug
@@ -14,29 +14,29 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'dualMode'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Dual mode
         ignite-form-field-tooltip.tipLabel
             | IGFS supports dual-mode that allows it to work as either a standalone file system in Hadoop cluster, or work in tandem with HDFS, providing a primary caching layer for the secondary HDFS#[br]
             | As a caching layer it provides highly configurable read-through and write-through behaviour
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Maximum pending puts size:', model + '.dualModeMaxPendingPutsSize', '"dualModeMaxPendingPutsSize"', 'true', '0', 'Number.MIN_SAFE_INTEGER',
+                    +number('Maximum pending puts size:', `${model}.dualModeMaxPendingPutsSize`, '"dualModeMaxPendingPutsSize"', 'true', '0', 'Number.MIN_SAFE_INTEGER',
                         'Maximum amount of pending data read from the secondary file system and waiting to be written to data cache<br/>\
                         Zero or negative value stands for unlimited size')
                 .settings-row
-                    +java-class('Put executor service:', model + '.dualModePutExecutorService', '"dualModePutExecutorService"', 'true', 'false', 'DUAL mode put operation executor service')
+                    +java-class('Put executor service:', `${model}.dualModePutExecutorService`, '"dualModePutExecutorService"', 'true', 'false', 'DUAL mode put operation executor service')
                 .settings-row
-                    +checkbox('Put executor service shutdown', model + '.dualModePutExecutorServiceShutdown', '"dualModePutExecutorServiceShutdown"', 'DUAL mode put operation executor service shutdown flag')
+                    +checkbox('Put executor service shutdown', `${model}.dualModePutExecutorServiceShutdown`, '"dualModePutExecutorServiceShutdown"', 'DUAL mode put operation executor service shutdown flag')
             .col-sm-6
                 +preview-xml-java(model, 'igfsDualMode')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.pug
similarity index 70%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.pug
index 16f3749103904..b1126976272b8 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.pug
@@ -14,30 +14,30 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'fragmentizer'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Fragmentizer
         ignite-form-field-tooltip.tipLabel
             | Fragmentizer settings
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var enabled = model + '.fragmentizerEnabled'
+                -var enabled = `${model}.fragmentizerEnabled`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"fragmentizerEnabled"', 'Fragmentizer enabled flag')
                 .settings-row
-                    +number('Concurrent files:', model + '.fragmentizerConcurrentFiles', '"fragmentizerConcurrentFiles"', enabled, '0', '0', 'Number of files to process concurrently by fragmentizer')
+                    +number('Concurrent files:', `${model}.fragmentizerConcurrentFiles`, '"fragmentizerConcurrentFiles"', enabled, '0', '0', 'Number of files to process concurrently by fragmentizer')
                 .settings-row
-                    +number('Throttling block length:', model + '.fragmentizerThrottlingBlockLength', '"fragmentizerThrottlingBlockLength"', enabled, '16777216', '1', 'Length of file chunk to transmit before throttling is delayed')
+                    +number('Throttling block length:', `${model}.fragmentizerThrottlingBlockLength`, '"fragmentizerThrottlingBlockLength"', enabled, '16777216', '1', 'Length of file chunk to transmit before throttling is delayed')
                 .settings-row
-                    +number('Throttling delay:', model + '.fragmentizerThrottlingDelay', '"fragmentizerThrottlingDelay"', enabled, '200', '0', 'Delay in milliseconds for which fragmentizer is paused')
+                    +number('Throttling delay:', `${model}.fragmentizerThrottlingDelay`, '"fragmentizerThrottlingDelay"', enabled, '200', '0', 'Delay in milliseconds for which fragmentizer is paused')
             .col-sm-6
                 +preview-xml-java(model, 'igfsFragmentizer')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/general.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/general.pug
similarity index 89%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/general.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/general.pug
index 62cda770ceddc..e002d364675a9 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/general.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/general.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var model = 'backupItem'
@@ -31,11 +31,11 @@ include /app/helpers/jade/mixins.jade
         .panel-body
             .col-sm-6
                 .settings-row
-                    +text('Name:', model + '.name', '"igfsName"', 'true', 'Input name', 'IGFS name')
+                    +text('Name:', `${model}.name`, '"igfsName"', 'true', 'Input name', 'IGFS name')
                 .settings-row
                     +clusters(model, 'Associate clusters with the current IGFS')
                 .settings-row
-                    +dropdown('IGFS mode:', model + '.defaultMode', '"defaultMode"', 'true', 'DUAL_ASYNC',
+                    +dropdown('IGFS mode:', `${model}.defaultMode`, '"defaultMode"', 'true', 'DUAL_ASYNC',
                     '[\
                         {value: "PRIMARY", label: "PRIMARY"},\
                         {value: "PROXY", label: "PROXY"},\
@@ -50,7 +50,7 @@ include /app/helpers/jade/mixins.jade
                         <li>DUAL_ASYNC - in this mode IGFS will cache files locally and also <b> asynchronously </b> write them through to secondary file system</li>\
                     </ul>')
                 .settings-row
-                    +number('Group size:', model + '.affinnityGroupSize', '"affinnityGroupSize"', 'true', '512', '1',
+                    +number('Group size:', `${model}.affinnityGroupSize`, '"affinnityGroupSize"', 'true', '512', '1',
                         'Size of the group in blocks<br/>\
                         Required for construction of affinity mapper in IGFS data cache')
             .col-sm-6
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.pug
similarity index 66%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.pug
index 83c5de6eae68a..7c8c056a0ea2c 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.pug
@@ -14,28 +14,28 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'ipc'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label IPC
         ignite-form-field-tooltip.tipLabel
             | IGFS Inter-process communication properties
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var ipcEndpointConfiguration = model + '.ipcEndpointConfiguration'
-                -var enabled = model + '.ipcEndpointEnabled'
+                -var ipcEndpointConfiguration = `${model}.ipcEndpointConfiguration`
+                -var enabled = `${model}.ipcEndpointEnabled`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"ipcEndpointEnabled"', 'IPC endpoint enabled flag')
                 .settings-row
-                    +dropdown('Type:', ipcEndpointConfiguration + '.type', '"ipcEndpointConfigurationType"', enabled, 'TCP',
+                    +dropdown('Type:', `${ipcEndpointConfiguration}.type`, '"ipcEndpointConfigurationType"', enabled, 'TCP',
                         '[\
                             {value: "SHMEM", label: "SHMEM"},\
                             {value: "TCP", label: "TCP"}\
@@ -46,15 +46,15 @@ include /app/helpers/jade/mixins.jade
                             <li>TCP - TCP endpoint</li>\
                         </ul>')
                 .settings-row
-                    +text-ip-address('Host:', ipcEndpointConfiguration + '.host', '"ipcEndpointConfigurationHost"', enabled, '127.0.0.1', 'Host endpoint is bound to')
+                    +text-ip-address('Host:', `${ipcEndpointConfiguration}.host`, '"ipcEndpointConfigurationHost"', enabled, '127.0.0.1', 'Host endpoint is bound to')
                 .settings-row
-                    +number-min-max('Port:', ipcEndpointConfiguration + '.port', '"ipcEndpointConfigurationPort"', enabled, '10500', '1', '65535', 'Port endpoint is bound to')
+                    +number-min-max('Port:', `${ipcEndpointConfiguration}.port`, '"ipcEndpointConfigurationPort"', enabled, '10500', '1', '65535', 'Port endpoint is bound to')
                 .settings-row
-                    +number('Memory size:', ipcEndpointConfiguration + '.memorySize', '"ipcEndpointConfigurationMemorySize"', enabled, '262144', '1', 'Shared memory size in bytes allocated for endpoint communication')
+                    +number('Memory size:', `${ipcEndpointConfiguration}.memorySize`, '"ipcEndpointConfigurationMemorySize"', enabled, '262144', '1', 'Shared memory size in bytes allocated for endpoint communication')
                 .settings-row
-                    +text-enabled('Token directory:', ipcEndpointConfiguration + '.tokenDirectoryPath', '"ipcEndpointConfigurationTokenDirectoryPath"', enabled, 'false', 'ipc/shmem', 'Directory where shared memory tokens are stored')
+                    +text-enabled('Token directory:', `${ipcEndpointConfiguration}.tokenDirectoryPath`, '"ipcEndpointConfigurationTokenDirectoryPath"', enabled, 'false', 'ipc/shmem', 'Directory where shared memory tokens are stored')
                 .settings-row
-                    +number('Thread count:', ipcEndpointConfiguration + '.threadCount', '"ipcEndpointConfigurationThreadCount"', enabled, 'availableProcessors', '1',
+                    +number('Thread count:', `${ipcEndpointConfiguration}.threadCount`, '"ipcEndpointConfigurationThreadCount"', enabled, 'availableProcessors', '1',
                         'Number of threads used by this endpoint to process incoming requests')
             .col-sm-6
                 +preview-xml-java(model, 'igfsIPC')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/misc.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/misc.pug
similarity index 66%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/misc.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/misc.pug
index dc0e9fc73991c..aa4b95740566f 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/misc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/misc.pug
@@ -14,68 +14,68 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'misc'
 -var model = 'backupItem'
 -var pathModesForm = 'miscPathModes'
--var pathModes = model + '.pathModes'
+-var pathModes = `${model}.pathModes`
 
 //- LEGACY mixin for LEGACY IGFS path modes table.
 mixin table-igfs-path-mode-edit(prefix, focusId, index)
-    -var keyModel = 'tblPathModes.' + prefix + 'Key'
-    -var valModel = 'tblPathModes.' + prefix + 'Value'
+    -var keyModel = `tblPathModes.${prefix}Key`
+    -var valModel = `tblPathModes.${prefix}Value`
 
-    -var keyFocusId = prefix + 'Key' + focusId
-    -var valFocusId = prefix + 'Value' + focusId
+    -var keyFocusId = `${prefix}Key${focusId}`
+    -var valFocusId = `${prefix}Value${focusId}`
 
     .col-xs-8.col-sm-8.col-md-8
         .fieldSep /
         .input-tip
             input.form-control(id=keyFocusId ignite-on-enter-focus-move=valFocusId type='text' ng-model=keyModel placeholder='Path' ignite-on-escape='tableReset(false)')
     .col-xs-4.col-sm-4.col-md-4
-        -var arg = keyModel + ', ' + valModel
-        -var btnVisible = 'tablePairSaveVisible(tblPathModes, ' + index + ')'
-        -var btnSave = 'tablePairSave(tablePairValid, backupItem, tblPathModes, ' + index + ')'
-        -var btnVisibleAndSave = btnVisible + ' && ' + btnSave
+        -var arg = `${keyModel}, ${valModel}`
+        -var btnVisible = `tablePairSaveVisible(tblPathModes, ${index})`
+        -var btnSave = `tablePairSave(tablePairValid, backupItem, tblPathModes, ${index})`
+        -var btnVisibleAndSave = `${btnVisible} && ${btnSave}`
         +btn-save(btnVisible, btnSave)
         .input-tip
             button.select-toggle.form-control(id=valFocusId bs-select ng-model=valModel data-placeholder='Mode' bs-options='item.value as item.label for item in igfsModes' tabindex='0' ignite-on-enter=btnVisibleAndSave ignite-on-escape='tableReset(false)')
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Miscellaneous
         ignite-form-field-tooltip.tipLabel
             | Various miscellaneous IGFS settings
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Block size:', model + '.blockSize', '"blockSize"', 'true', '65536', '0', 'File data block size in bytes')
+                    +number('Block size:', `${model}.blockSize`, '"blockSize"', 'true', '65536', '0', 'File data block size in bytes')
                 .settings-row
-                    +number('Stream buffer size:', model + '.streamBufferSize', '"streamBufferSize"', 'true', '65536', '0', 'Read/write buffer size for IGFS stream operations in bytes')
+                    +number('Stream buffer size:', `${model}.streamBufferSize`, '"streamBufferSize"', 'true', '65536', '0', 'Read/write buffer size for IGFS stream operations in bytes')
                 .settings-row
-                    +number('Maximum space size:', model + '.maxSpaceSize', '"maxSpaceSize"', 'true', '0', '0', 'Maximum space available for data cache to store file system entries')
+                    +number('Maximum space size:', `${model}.maxSpaceSize`, '"maxSpaceSize"', 'true', '0', '0', 'Maximum space available for data cache to store file system entries')
                 .settings-row
-                    +number('Maximum task range length:', model + '.maximumTaskRangeLength', '"maximumTaskRangeLength"', 'true', '0', '0', 'Maximum default range size of a file being split during IGFS task execution')
+                    +number('Maximum task range length:', `${model}.maximumTaskRangeLength`, '"maximumTaskRangeLength"', 'true', '0', '0', 'Maximum default range size of a file being split during IGFS task execution')
                 .settings-row
-                    +number-min-max('Management port:', model + '.managementPort', '"managementPort"', 'true', '11400', '0', '65535', 'Port number for management endpoint')
+                    +number-min-max('Management port:', `${model}.managementPort`, '"managementPort"', 'true', '11400', '0', '65535', 'Port number for management endpoint')
                 .settings-row
-                    +number('Per node batch size:', model + '.perNodeBatchSize', '"perNodeBatchSize"', 'true', '100', '0', 'Number of file blocks collected on local node before sending batch to remote node')
+                    +number('Per node batch size:', `${model}.perNodeBatchSize`, '"perNodeBatchSize"', 'true', '100', '0', 'Number of file blocks collected on local node before sending batch to remote node')
                 .settings-row
-                    +number('Per node parallel batch count:', model + '.perNodeParallelBatchCount', '"perNodeParallelBatchCount"', 'true', '8', '0', 'Number of file block batches that can be concurrently sent to remote node')
+                    +number('Per node parallel batch count:', `${model}.perNodeParallelBatchCount`, '"perNodeParallelBatchCount"', 'true', '8', '0', 'Number of file block batches that can be concurrently sent to remote node')
                 .settings-row
-                    +number('Prefetch blocks:', model + '.prefetchBlocks', '"prefetchBlocks"', 'true', '0', '0', 'Number of pre-fetched blocks if specific file chunk is requested')
+                    +number('Prefetch blocks:', `${model}.prefetchBlocks`, '"prefetchBlocks"', 'true', '0', '0', 'Number of pre-fetched blocks if specific file chunk is requested')
                 .settings-row
-                    +number('Sequential reads before prefetch:', model + '.sequentialReadsBeforePrefetch', '"sequentialReadsBeforePrefetch"', 'true', '0', '0', 'Amount of sequential block reads before prefetch is triggered')
+                    +number('Sequential reads before prefetch:', `${model}.sequentialReadsBeforePrefetch`, '"sequentialReadsBeforePrefetch"', 'true', '0', '0', 'Amount of sequential block reads before prefetch is triggered')
                 .settings-row
-                    +number('Trash purge timeout:', model + '.trashPurgeTimeout', '"trashPurgeTimeout"', 'true', '1000', '0', 'Maximum timeout awaiting for trash purging in case data cache oversize is detected')
+                    +number('Trash purge timeout:', `${model}.trashPurgeTimeout`, '"trashPurgeTimeout"', 'true', '1000', '0', 'Maximum timeout awaiting for trash purging in case data cache oversize is detected')
                 .settings-row
-                    +checkbox('Colocate metadata', model + '.colocateMetadata', '"colocateMetadata"', 'Whether to co-locate metadata on a single node')
+                    +checkbox('Colocate metadata', `${model}.colocateMetadata`, '"colocateMetadata"', 'Whether to co-locate metadata on a single node')
                 .settings-row
-                    +checkbox('Relaxed consistency', model + '.relaxedConsistency', '"relaxedConsistency"',
+                    +checkbox('Relaxed consistency', `${model}.relaxedConsistency`, '"relaxedConsistency"',
                         'If value of this flag is <b>true</b>, IGFS will skip expensive consistency checks<br/>\
                         It is recommended to set this flag to <b>false</b> if your application has conflicting\
                         operations, or you do not know how exactly users will use your system')
@@ -88,12 +88,12 @@ mixin table-igfs-path-mode-edit(prefix, focusId, index)
                         ignite-form-group-add(ng-click='tableNewItem(tblPathModes)')
                             | Add path mode
 
-                        .group-content-empty(ng-if='!((#{pathModes} && #{pathModes}.length > 0) || tableNewItemActive(tblPathModes))') Not defined
+                        .group-content-empty(ng-if=`!((${pathModes} && ${pathModes}.length > 0) || tableNewItemActive(tblPathModes))`) Not defined
 
-                        .group-content(ng-show='(#{pathModes} && #{pathModes}.length > 0) || tableNewItemActive(tblPathModes)')
+                        .group-content(ng-show=`(${pathModes} && ${pathModes}.length > 0) || tableNewItemActive(tblPathModes)`)
                             table.links-edit(id='pathModes' st-table=pathModes)
                                 tbody
-                                    tr(ng-repeat='item in #{pathModes} track by $index')
+                                    tr(ng-repeat=`item in ${pathModes} track by $index`)
                                         td.col-sm-12(ng-hide='tableEditing(tblPathModes, $index)')
                                             a.labelFormField(ng-click='tableStartEdit(backupItem, tblPathModes, $index)') {{item.path + " [" + item.mode + "]"}}
                                             +btn-remove('tableRemove(backupItem, tblPathModes, $index)', '"Remove path"')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.pug
similarity index 69%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.pug
index b605e7775a632..87e1f4fb8edb9 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.pug
@@ -14,32 +14,32 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'secondaryFileSystem'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label(id="secondaryFileSystem-title") Secondary file system
         ignite-form-field-tooltip.tipLabel
             | Secondary file system is provided for pass-through, write-through, and read-through purposes#[br]
             | #[a(href="https://apacheignite-fs.readme.io/docs/secondary-file-system" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var enabled = model + '.secondaryFileSystemEnabled'
-                -var secondaryFileSystem = model + '.secondaryFileSystem'
+                -var enabled = `${model}.secondaryFileSystemEnabled`
+                -var secondaryFileSystem = `${model}.secondaryFileSystem`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"secondaryFileSystemEnabled"', 'Secondary file system enabled flag')
                 .settings-row
-                    +text-enabled('URI:', secondaryFileSystem + '.uri', '"hadoopURI"', enabled, 'false', 'hdfs://[namenodehost]:[port]/[path]', 'URI of file system')
+                    +text-enabled('URI:', `${secondaryFileSystem}.uri`, '"hadoopURI"', enabled, 'false', 'hdfs://[namenodehost]:[port]/[path]', 'URI of file system')
                 .settings-row
-                    +text-enabled('Config path:', secondaryFileSystem + '.cfgPath', '"cfgPath"', enabled, 'false', 'Path to additional config', 'Additional path to Hadoop configuration')
+                    +text-enabled('Config path:', `${secondaryFileSystem}.cfgPath`, '"cfgPath"', enabled, 'false', 'Path to additional config', 'Additional path to Hadoop configuration')
                 .settings-row
-                    +text-enabled('User name:', secondaryFileSystem + '.userName', '"userName"', enabled, 'false', 'Input user name', 'User name')
+                    +text-enabled('User name:', `${secondaryFileSystem}.userName`, '"userName"', enabled, 'false', 'Input user name', 'User name')
             .col-sm-6
                 +preview-xml-java(model, 'igfsSecondFS')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
index 16d2faef04335..25203b30d1a27 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
@@ -18,6 +18,8 @@
 import _ from 'lodash';
 import saver from 'file-saver';
 
+import summaryProjectStructureTemplateUrl from 'views/configuration/summary-project-structure.tpl.pug';
+
 const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
 
 export default [
@@ -25,6 +27,9 @@ export default [
     function($root, $scope, $http, LegacyUtils, Messages, Loading, $filter, Resource, JavaTypes, Version, generator, spring, java, docker, pom, propsGenerator, readme, FormUtils, SummaryZipper, ActivitiesData) {
         const ctrl = this;
 
+        // Define template urls.
+        ctrl.summaryProjectStructureTemplateUrl = summaryProjectStructureTemplateUrl;
+
         $scope.ui = {
             isSafari: !!(/constructor/i.test(window.HTMLElement) || window.safari),
             ready: false
diff --git a/modules/web-console/frontend/app/modules/states/errors.state.js b/modules/web-console/frontend/app/modules/states/errors.state.js
index 2bdb80a432461..e219132beb34e 100644
--- a/modules/web-console/frontend/app/modules/states/errors.state.js
+++ b/modules/web-console/frontend/app/modules/states/errors.state.js
@@ -16,8 +16,8 @@
  */
 
 import angular from 'angular';
-import templateNotFoundPage from '../../../views/404.jade';
-import templateNotAuthorizedPage from '../../../views/403.jade';
+import templateNotFoundPage from 'views/404.pug';
+import templateNotAuthorizedPage from 'views/403.pug';
 
 angular
     .module('ignite-console.states.errors', [
diff --git a/modules/web-console/frontend/app/modules/states/password.state.js b/modules/web-console/frontend/app/modules/states/password.state.js
index 48d01df60aa7f..587f83da9a1c5 100644
--- a/modules/web-console/frontend/app/modules/states/password.state.js
+++ b/modules/web-console/frontend/app/modules/states/password.state.js
@@ -17,6 +17,8 @@
 
 import angular from 'angular';
 
+import templateUrl from 'views/reset.tpl.pug';
+
 angular
 .module('ignite-console.states.password', [
     'ui.router'
@@ -31,14 +33,14 @@ angular
     })
     .state('password.reset', {
         url: '/reset?{token}',
-        templateUrl: '/reset.html',
+        templateUrl,
         metaTags: {
             title: 'Reset password'
         }
     })
     .state('password.send', {
         url: '/send',
-        templateUrl: '/reset.html',
+        templateUrl,
         metaTags: {
             title: 'Password Send'
         }
diff --git a/modules/web-console/frontend/app/modules/states/profile.state.js b/modules/web-console/frontend/app/modules/states/profile.state.js
index 9c31340a9569b..7c270ad17d36a 100644
--- a/modules/web-console/frontend/app/modules/states/profile.state.js
+++ b/modules/web-console/frontend/app/modules/states/profile.state.js
@@ -17,6 +17,8 @@
 
 import angular from 'angular';
 
+import templateUrl from 'views/settings/profile.tpl.pug';
+
 angular
 .module('ignite-console.states.profile', [
     'ui.router'
@@ -26,7 +28,7 @@ angular
     $stateProvider
     .state('settings.profile', {
         url: '/profile',
-        templateUrl: '/settings/profile.html',
+        templateUrl,
         onEnter: AclRoute.checkAccess('profile'),
         metaTags: {
             title: 'User profile'
diff --git a/modules/web-console/frontend/app/modules/states/signin.state.js b/modules/web-console/frontend/app/modules/states/signin.state.js
index 14ebc1baa6aab..2625aaa32997e 100644
--- a/modules/web-console/frontend/app/modules/states/signin.state.js
+++ b/modules/web-console/frontend/app/modules/states/signin.state.js
@@ -16,7 +16,7 @@
  */
 
 import angular from 'angular';
-import templateUrl from 'views/signin.jade';
+import templateUrl from 'views/signin.tpl.pug';
 
 angular
 .module('ignite-console.states.login', [
diff --git a/modules/web-console/frontend/app/services/Clone.service.js b/modules/web-console/frontend/app/services/Clone.service.js
index 52a4e4e8d1cb9..d079141364b55 100644
--- a/modules/web-console/frontend/app/services/Clone.service.js
+++ b/modules/web-console/frontend/app/services/Clone.service.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import templateUrl from 'views/templates/clone.tpl.pug';
+
 // Service for clone objects.
 export default ['IgniteClone', ['$rootScope', '$q', '$modal', ($root, $q, $modal) => {
     const scope = $root.$new();
@@ -36,7 +38,7 @@ export default ['IgniteClone', ['$rootScope', '$q', '$modal', ($root, $q, $modal
         return tmpName;
     }
 
-    const cloneModal = $modal({templateUrl: '/templates/clone.html', scope, placement: 'center', show: false});
+    const cloneModal = $modal({templateUrl, scope, placement: 'center', show: false});
 
     scope.ok = function(newName) {
         if (!_validator || _validator(newName)) {
diff --git a/modules/web-console/frontend/app/services/Confirm.service.js b/modules/web-console/frontend/app/services/Confirm.service.js
index 8208ea2a15454..0b5b9be87a6ad 100644
--- a/modules/web-console/frontend/app/services/Confirm.service.js
+++ b/modules/web-console/frontend/app/services/Confirm.service.js
@@ -15,11 +15,13 @@
  * limitations under the License.
  */
 
+import templateUrl from 'views/templates/confirm.tpl.pug';
+
 // Confirm popup service.
 export default ['IgniteConfirm', ['$rootScope', '$q', '$modal', '$animate', ($root, $q, $modal, $animate) => {
     const scope = $root.$new();
 
-    const modal = $modal({templateUrl: '/templates/confirm.html', scope, placement: 'center', show: false, backdrop: true});
+    const modal = $modal({templateUrl, scope, placement: 'center', show: false, backdrop: true});
 
     const _hide = () => {
         $animate.enabled(modal.$element, false);
diff --git a/modules/web-console/frontend/app/services/ConfirmBatch.service.js b/modules/web-console/frontend/app/services/ConfirmBatch.service.js
index ef663353539aa..2fd55b709d8c4 100644
--- a/modules/web-console/frontend/app/services/ConfirmBatch.service.js
+++ b/modules/web-console/frontend/app/services/ConfirmBatch.service.js
@@ -15,12 +15,14 @@
  * limitations under the License.
  */
 
+import templateUrl from 'views/templates/batch-confirm.tpl.pug';
+
 // Service for confirm or skip several steps.
 export default ['IgniteConfirmBatch', ['$rootScope', '$q', '$modal', ($root, $q, $modal) => {
     const scope = $root.$new();
 
     scope.confirmModal = $modal({
-        templateUrl: '/templates/batch-confirm.html',
+        templateUrl,
         scope,
         placement: 'center',
         show: false,
diff --git a/modules/web-console/frontend/controllers/caches-controller.js b/modules/web-console/frontend/controllers/caches-controller.js
index e7521b55ad403..b50fde3188b1a 100644
--- a/modules/web-console/frontend/controllers/caches-controller.js
+++ b/modules/web-console/frontend/controllers/caches-controller.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import infoMessageTemplateUrl from 'views/templates/message.tpl.pug';
+
 // Controller for Caches screen.
 export default ['cachesController', [
     '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'IgniteLegacyTable',
@@ -542,7 +544,7 @@ export default ['cachesController', [
                         ];
 
                         // Show a basic modal from a controller
-                        $modal({scope, template: '/templates/message.html', placement: 'center', show: true});
+                        $modal({scope, templateUrl: infoMessageTemplateUrl, placement: 'center', show: true});
                     }
 
                     save(item);
diff --git a/modules/web-console/frontend/controllers/domains-controller.js b/modules/web-console/frontend/controllers/domains-controller.js
index bfffe92f62747..5c9e511a139ca 100644
--- a/modules/web-console/frontend/controllers/domains-controller.js
+++ b/modules/web-console/frontend/controllers/domains-controller.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import templateUrl from 'views/configuration/domains-import.tpl.pug';
+
 // Controller for Domain model screen.
 export default ['domainsController', [
     '$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteAgentMonitor', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes', 'IgniteActivitiesData',
@@ -409,7 +411,7 @@ export default ['domainsController', [
         $scope.$watch('importDomain.displayedTables', $scope.selectTable);
 
         // Pre-fetch modal dialogs.
-        const importDomainModal = $modal({scope: $scope, templateUrl: '/configuration/domains-import.html', show: false});
+        const importDomainModal = $modal({scope: $scope, templateUrl, show: false});
 
         const hideImportDomain = importDomainModal.hide;
 
diff --git a/modules/web-console/frontend/gulpfile.babel.js/paths.js b/modules/web-console/frontend/gulpfile.babel.js/paths.js
index 6ebfbad42ba84..627b2f9599058 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/paths.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/paths.js
@@ -24,33 +24,15 @@ const destDir = path.resolve('build');
 const igniteModulesDir = process.env.IGNITE_MODULES ? path.join(path.normalize(process.env.IGNITE_MODULES), 'frontend') : './ignite_modules';
 const igniteModulesTemp = path.resolve('ignite_modules_temp');
 
-const jadeViewsPaths = [
-    './views/**/*.jade',
-    '!./views/configuration/*.jade'
-];
-
-const jadeAppModulePaths = [
-    './app/modules/states/configuration/**/*.jade',
-    './app/modules/sql/*.jade',
-    './views/**/*.jade',
-    '!./views/*.jade',
-    '!./views/includes/*.jade',
-    '!./views/settings/*.jade',
-    '!./views/sql/*.jade',
-    '!./views/templates/*.jade'
-];
-
-const jadeModulePaths = [
-    igniteModulesDir + '/**/view/**/*.jade'
-];
-
 const appModulePaths = [
     igniteModulesDir + '/index.js',
     igniteModulesDir + '/**/main.js',
     igniteModulesDir + '/**/module.js',
     igniteModulesDir + '/**/app/modules/*.js',
     igniteModulesDir + '/**/app/modules/**/*.js',
-    igniteModulesDir + '/**/app/modules/**/*.jade',
+    igniteModulesDir + '/**/app/modules/**/*.pug',
+    igniteModulesDir + '/**/*.pug',
+    igniteModulesDir + '/**/*.tpl.pug',
     igniteModulesDir + '/**/app/**/*.css',
     igniteModulesDir + '/**/app/**/*.scss',
     igniteModulesDir + '/**/app/data/*.json'
@@ -73,10 +55,6 @@ export {
     igniteModulesDir,
     igniteModulesTemp,
 
-    jadeViewsPaths,
-    jadeAppModulePaths,
-    jadeModulePaths,
-
     resourcePaths,
     resourceModulePaths,
     appModulePaths
diff --git a/modules/web-console/frontend/gulpfile.babel.js/tasks/build.js b/modules/web-console/frontend/gulpfile.babel.js/tasks/build.js
index 7d7401b7cfb4a..149b07680c8a9 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/tasks/build.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/tasks/build.js
@@ -18,4 +18,4 @@
 import gulp from 'gulp';
 import sequence from 'gulp-sequence';
 
-gulp.task('build', (cb) => sequence(['clean', 'clean:ignite-modules-temp'], 'ignite:modules', ['copy:resource', 'jade'], 'bundle', cb));
+gulp.task('build', (cb) => sequence(['clean', 'clean:ignite-modules-temp'], 'ignite:modules', ['copy:resource'], 'bundle', cb));
diff --git a/modules/web-console/frontend/gulpfile.babel.js/tasks/bundle.js b/modules/web-console/frontend/gulpfile.babel.js/tasks/bundle.js
index d3e8dca576cd5..ac3ac76be89d1 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/tasks/bundle.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/tasks/bundle.js
@@ -21,10 +21,14 @@ import webpackConfig from '../webpack';
 import WebpackDevServer from 'webpack-dev-server';
 
 gulp.task('bundle', (cb) => {
+    const devServerConfig = webpackConfig.devServer;
+
+    delete webpackConfig.devServer;
+
     if (process.env.NODE_ENV === 'development') {
         // Important! Call webpack and WebpackDevServer must be inline.
-        new WebpackDevServer(webpack(webpackConfig), webpackConfig.devServer)
-            .listen(webpackConfig.devServer.port, 'localhost', cb);
+        new WebpackDevServer(webpack(webpackConfig), devServerConfig)
+            .listen(devServerConfig.port, 'localhost', cb);
     }
     else
         webpack(webpackConfig, cb);
diff --git a/modules/web-console/frontend/gulpfile.babel.js/tasks/watch.js b/modules/web-console/frontend/gulpfile.babel.js/tasks/watch.js
index dfaa1a3a88c35..9a02b889a6b41 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/tasks/watch.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/tasks/watch.js
@@ -18,15 +18,12 @@
 import gulp from 'gulp';
 import sequence from 'gulp-sequence';
 
-import { jadeViewsPaths, jadeAppModulePaths, jadeModulePaths, resourcePaths, resourceModulePaths, appModulePaths } from '../paths';
+import { resourcePaths, resourceModulePaths, appModulePaths } from '../paths';
 
 gulp.task('watch:ignite-modules', (cb) => sequence('clean:ignite-modules-temp', 'ignite:modules', cb));
 
 // Build + watch task.
 gulp.task('watch', ['build'], () => {
-    gulp.watch(jadeViewsPaths, ['jade:views']);
-    gulp.watch(jadeAppModulePaths.concat('./app/helpers/**/*.jade'), ['jade:app']);
-    gulp.watch(jadeModulePaths.concat('./app/helpers/**/*.jade'), ['jade:ignite_modules']);
     gulp.watch(resourcePaths, ['copy:resource:app']);
     gulp.watch(resourceModulePaths, ['copy:resource:ignite_modules']);
     gulp.watch(appModulePaths, ['watch:ignite-modules']);
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js
index 7360ac4c0da3c..3d9616f788d1d 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js
@@ -18,21 +18,22 @@
 import path from 'path';
 import fs from 'fs';
 import webpack from 'webpack';
-import autoprefixer from 'autoprefixer-core';
-import jade from 'jade';
+
 import ProgressBarPlugin from 'progress-bar-webpack-plugin';
 import eslintFormatter from 'eslint-friendly-formatter';
 
-import ExtractTextPlugin from 'extract-text-webpack-plugin';
 import HtmlWebpackPlugin from 'html-webpack-plugin';
 
+import ExtractTextPlugin from 'extract-text-webpack-plugin';
+
 import {srcDir, destDir, rootDir, igniteModulesDir} from '../paths';
 
+const viewsDir = path.resolve('views');
+const imagesDir = path.resolve('public/images');
+
 const NODE_ENV = process.env.NODE_ENV || 'production';
 const development = NODE_ENV === 'development';
 const node_modules_path = path.resolve('node_modules');
-const cssLoader = 'css-loader?sourceMap!postcss-loader';
-const stylesLoader = cssLoader + '!sass-loader?outputStyle=expanded&sourceMap=true&sourceMapContents=true';
 
 let favicon = 'build/ignite_modules/favicon.ico';
 
@@ -42,150 +43,155 @@ try {
     favicon = 'build/favicon.ico';
 }
 
-export default () => {
-    const assetsLoader = 'file-loader';
-
-    return {
-        cache: true,
-        node: {
-            fs: 'empty'
-        },
-
-        // Entry points.
-        entry: {
-            polyfill: 'babel-polyfill',
-            app: path.join(srcDir, 'app.js'),
-            vendor: path.join(srcDir, 'vendor.js')
-        },
-
-        // Output system.
-        output: {
-            path: destDir,
-            filename: '[name].js'
-        },
-
-        // Resolves modules.
-        resolve: {
-            extensions: [
-                '',
-                '.js'
-            ],
-            root: [rootDir],
-            modulesDirectories: [
-                node_modules_path,
-                './'
-            ]
-        },
-        jade: {
-            basedir: rootDir,
-            locals: {}
-        },
-        // Modules resolvers.
-        /* global require */
-        module: {
-            noParse: [],
-            preLoaders: [
-                {
-                    test: /\.js$/,
-                    exclude: [node_modules_path],
-                    loader: 'eslint-loader'
-                }
-            ],
-            loaders: [
-                {
-                    test: /\.json$/,
-                    loader: 'json-loader'
-                },
-                {
-                    test: /\.jade$/,
-                    loaders: [
-                        `ngtemplate-loader?relativeTo=${rootDir}`,
-                        'html-loader?attrs[]=img:src&attrs[]=img:data-src',
-                        'jade-html-loader'
-                    ]
-                },
-                {
-                    test: /\.js$/,
-                    exclude: [node_modules_path],
-                    loader: 'babel-loader',
-                    query: {
+export default {
+    cache: true,
+    node: {
+        fs: 'empty'
+    },
+    // Entry points.
+    entry: {
+        polyfill: 'babel-polyfill',
+        vendor: path.join(srcDir, 'vendor.js'),
+        app: path.join(srcDir, 'app.js')
+    },
+
+    // Output system.
+    output: {
+        path: destDir,
+        filename: '[name].js'
+    },
+
+    // Resolves modules.
+    resolve: {
+        extensions: [
+            '.js'
+        ],
+        modules: [
+            srcDir,
+            rootDir,
+            node_modules_path
+        ],
+        // A list of module source folders.
+        alias: {
+            app: srcDir,
+            views: viewsDir,
+            images: imagesDir
+        }
+    },
+
+    // Resolve loader use postfix.
+    resolveLoader: {
+        moduleExtensions: ['-loader']
+    },
+
+    module: {
+        rules: [
+            {
+                test: /\.json$/,
+                loader: 'json'
+            },
+
+            // Exclude tpl.pug files to import in bundle.
+            {
+                test: /^(?:(?!tpl\.pug$).)*\.pug$/, // TODO: check this regexp for correct.
+                loader: `pug-html?basedir=${rootDir}`
+            },
+
+            // Render .tpl.pug files to assets folder.
+            {
+                test: /\.tpl\.pug$/,
+                use: [
+                    'file?exports=false&name=assets/templates/[name].[hash].html',
+                    `pug-html?exports=false&basedir=${rootDir}`
+                ]
+            },
+            {
+                test: /\.js$/,
+                enforce: 'pre',
+                exclude: [node_modules_path],
+                use: [{
+                    loader: 'eslint',
+                    options: {
+                        failOnWarning: false,
+                        failOnError: false,
+                        formatter: eslintFormatter
+                    }
+                }]
+            },
+            {
+                test: /\.js$/,
+                exclude: [node_modules_path],
+                use: [{
+                    loader: 'babel',
+                    options: {
                         cacheDirectory: true,
                         plugins: [
                             'transform-runtime',
                             'add-module-exports'
                         ],
                         presets: ['angular']
-
                     }
-                },
-                {
-                    test: /\.css$/,
-                    loader: development ? `style-loader!${cssLoader}` : ExtractTextPlugin.extract('style-loader', cssLoader)
-                },
-                {
-                    test: /\.(scss|sass)$/,
-                    loader: development ? `style-loader!${stylesLoader}` : ExtractTextPlugin.extract('style-loader', stylesLoader)
-                },
-                {
-                    test: /\.(ttf|eot|svg|woff(2)?)(\?v=[\d.]+)?(\?[a-z0-9#-]+)?$/,
-                    loaders: [`${assetsLoader}?name=assets/fonts/[name].[ext]`]
-                },
-                {
-                    test: /\.(jpe?g|png|gif)$/i,
-                    loaders: [`${assetsLoader}?name=assets/images/[name]_[hash].[ext]`]
-                },
-                {
-                    test: require.resolve('jquery'),
-                    loaders: [
-                        'expose-loader?$',
-                        'expose-loader?jQuery'
-                    ]
-                },
-                {
-                    test: require.resolve('nvd3'),
-                    loaders: [
-                        'expose-loader?nv'
-                    ]
-                }
-            ]
-        },
-
-        // Postcss configuration.
-        postcss: [autoprefixer({browsers: ['last 2 versions']})],
-
-        // ESLint loader configuration.
-        eslint: {
-            failOnWarning: false,
-            failOnError: false,
-            formatter: eslintFormatter
-        },
-
-        // Load plugins.
-        plugins: [
-            new webpack.ProvidePlugin({
-                $: 'jquery',
-                jQuery: 'jquery',
-                _: 'lodash',
-                nv: 'nvd3'
-            }),
-            new webpack.DefinePlugin({NODE_ENV: JSON.stringify(NODE_ENV)}),
-            // new webpack.NoErrorsPlugin(),
-            new webpack.optimize.DedupePlugin(),
-            new webpack.optimize.CommonsChunkPlugin({
-                name: 'common',
-                chunks: ['vendor', 'app']
-            }),
-            new webpack.optimize.AggressiveMergingPlugin({moveToParents: true}),
-            new webpack.optimize.OccurenceOrderPlugin(),
-            new ExtractTextPlugin('assets/css/[name]' + (development ? '' : '.[chunkhash]') + '.css', {allChunks: true}),
-            new HtmlWebpackPlugin({
-                filename: 'index.html',
-                templateContent: () => {
-                    return jade.renderFile(path.join(rootDir, 'views', 'index.jade'));
-                },
-                favicon
-            }),
-            new ProgressBarPlugin()
+                }]
+            },
+            {
+                test: /\.css$/,
+                use: development ? ['style', 'css'] : ExtractTextPlugin.extract({
+                    fallback: 'style',
+                    use: ['css']
+                })
+            },
+            {
+                test: /\.scss$/,
+                use: development ? ['style', 'css', 'sass'] : ExtractTextPlugin.extract({
+                    fallback: 'style-loader',
+                    use: ['css', 'sass']
+                })
+            },
+            {
+                test: /\.(ttf|eot|svg|woff(2)?)(\?v=[\d.]+)?(\?[a-z0-9#-]+)?$/,
+                loader: 'file?name=assets/fonts/[name].[ext]'
+            },
+            {
+                test: /\.(jpe?g|png|gif)$/i,
+                loader: 'file?name=assets/images/[name]_[hash].[ext]'
+            },
+            {
+                test: require.resolve('jquery'),
+                use: [
+                    'expose-loader?$',
+                    'expose-loader?jQuery'
+                ]
+            },
+            {
+                test: require.resolve('nvd3'),
+                use: ['expose-loader?nv']
+            }
         ]
-    };
+    },
+
+    // Load plugins.
+    plugins: [
+        new webpack.LoaderOptionsPlugin({
+            options: {
+                pug: {
+                    basedir: rootDir
+                }
+            }
+        }),
+        new webpack.ProvidePlugin({
+            $: 'jquery',
+            jQuery: 'jquery',
+            _: 'lodash',
+            nv: 'nvd3'
+        }),
+        new webpack.DefinePlugin({NODE_ENV: JSON.stringify(NODE_ENV)}),
+        new webpack.optimize.CommonsChunkPlugin({name: 'vendor'}),
+        new webpack.optimize.AggressiveMergingPlugin({moveToParents: true}),
+        new HtmlWebpackPlugin({
+            template: './views/index.pug',
+            favicon
+        }),
+        new ExtractTextPlugin({filename: 'assets/css/[name].css', allChunks: true}),
+        new ProgressBarPlugin()
+    ]
 };
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js
index 34e1f6ae858b0..c837c43d828d5 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js
@@ -15,65 +15,47 @@
  * limitations under the License.
  */
 
-import path from 'path';
-import webpack from 'webpack';
-
-import {destDir, rootDir, srcDir} from '../../paths';
+import {destDir} from '../../paths';
 
 const backendPort = 3000;
 const devServerPort = 9000;
 
-export default () => {
-    const plugins = [
-        new webpack.HotModuleReplacementPlugin()
-    ];
-
-    return {
-        entry: {
-            app: [path.join(srcDir, 'app.js'), 'webpack/hot/only-dev-server']
-        },
-        output: {
-            publicPath: `http://localhost:${devServerPort}/`
-        },
-        context: rootDir,
-        debug: true,
-        devtool: 'source-map',
-        watch: true,
-        devServer: {
-            compress: true,
-            historyApiFallback: true,
-            contentBase: destDir,
-            hot: true,
-            inline: true,
-            proxy: {
-                '/socket.io': {
-                    target: `http://localhost:${backendPort}`,
-                    changeOrigin: true,
-                    ws: true
-                },
-                '/agents': {
-                    target: `http://localhost:${backendPort}`,
-                    changeOrigin: true,
-                    ws: true
-                },
-                '/api/v1/*': {
-                    target: `http://localhost:${backendPort}`,
-                    changeOrigin: true,
-                    pathRewrite: {
-                        '^/api/v1': ''
-                    }
-                }
+export default {
+    devtool: 'source-map',
+    watch: true,
+    devServer: {
+        compress: true,
+        historyApiFallback: true,
+        contentBase: destDir,
+        // hot: true,
+        inline: true,
+        proxy: {
+            '/socket.io': {
+                target: `http://localhost:${backendPort}`,
+                changeOrigin: true,
+                ws: true
             },
-            watchOptions: {
-                aggregateTimeout: 1000,
-                poll: 2000
+            '/agents': {
+                target: `http://localhost:${backendPort}`,
+                changeOrigin: true,
+                ws: true
             },
-            stats: {
-                colors: true,
-                chunks: false
-            },
-            port: devServerPort
+            '/api/v1/*': {
+                target: `http://localhost:${backendPort}`,
+                changeOrigin: true,
+                pathRewrite: {
+                    '^/api/v1': ''
+                }
+            }
+        },
+        watchOptions: {
+            aggregateTimeout: 1000,
+            poll: 2000
+        },
+        stats: {
+            colors: true,
+            chunks: false
         },
-        plugins
-    };
+        port: devServerPort
+    }
 };
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js
index 1194568f79217..cbbe9ca3a9f2e 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js
@@ -15,30 +15,27 @@
  * limitations under the License.
  */
 
+
 import webpack from 'webpack';
 
-import {destDir, rootDir} from '../../paths';
+import {destDir} from '../../paths';
 
-export default () => {
-    const plugins = [
-        new webpack.optimize.UglifyJsPlugin({
-            path: destDir,
-            minimize: true,
-            warnings: false,
-            sourceMap: false,
-            mangle: true
-        })
-    ];
+const plugins = [
+    new webpack.optimize.UglifyJsPlugin({
+        path: destDir,
+        minimize: true,
+        warnings: false,
+        sourceMap: false,
+        mangle: true
+    })
+];
 
-    return {
-        context: rootDir,
-        bail: true, // Cancel build on error.
-        debug: false,
-        devtool: 'cheap-source-map',
-        output: {
-            publicPath: '/',
-            filename: '[name].[chunkhash].js'
-        },
-        plugins
-    };
+export default {
+    bail: true, // Cancel build on error.
+    devtool: 'cheap-source-map',
+    output: {
+        publicPath: '/',
+        filename: '[name].[chunkhash].js'
+    },
+    plugins
 };
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/index.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/index.js
index 9b344dd6e1cab..3caf06c9c4728 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/index.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/index.js
@@ -19,6 +19,8 @@ import _ from 'lodash';
 import commonConfig from './common';
 import devConfig from './environments/development';
 import prodConfig from './environments/production';
+
+// TODO check test config
 import testConfig from './environments/test';
 
 const env = process.env.NODE_ENV || 'production';
@@ -31,4 +33,4 @@ const configs = {
 };
 
 // Load config file by environment
-export default _.merge(commonConfig(), configs[env]());
+export default _.merge(commonConfig, configs[env]);
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 2d126550f16b4..b11f690d66e9e 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -5,7 +5,7 @@
   "private": true,
   "scripts": {
     "dev": "cross-env NODE_ENV=development gulp watch",
-    "build": "cross-env NODE_ENV=production gulp build --no-ll",
+    "build": "cross-env NODE_ENV=production gulp build",
     "test": "cross-env NODE_ENV=test karma start ./test/karma.conf.js",
     "eslint": "eslint --format node_modules/eslint-friendly-formatter gulpfile.babel.js/ app/ controllers/ ignite_modules/ ignite_modules_temp/ -- --eff-by-issue"
   },
@@ -50,7 +50,7 @@
     "angular-ui-router": "0.4.2",
     "bootstrap-sass": "3.3.7",
     "brace": "0.8.0",
-    "es6-promise": "3.3.1",
+    "es6-promise": "4.0.5",
     "file-saver": "1.3.3",
     "font-awesome": "4.7.0",
     "glob": "7.1.1",
@@ -60,12 +60,12 @@
     "nvd3": "1.8.4",
     "raleway-webfont": "3.0.1",
     "roboto-font": "0.1.0",
-    "socket.io-client": "1.7.2",
+    "socket.io-client": "1.7.3",
     "ui-router-metatags": "1.0.3"
   },
   "devDependencies": {
     "assets-webpack-plugin": "3.5.1",
-    "autoprefixer-core": "6.0.1",
+    "autoprefixer": "6.7.5",
     "babel-core": "6.20.0",
     "babel-eslint": "7.0.0",
     "babel-loader": "6.2.10",
@@ -77,52 +77,50 @@
     "babel-preset-es2015": "6.18.0",
     "babel-runtime": "6.20.0",
     "chai": "3.5.0",
-    "cross-env": "1.0.8",
-    "css-loader": "0.26.1",
-    "eslint": "3.12.2",
+    "cross-env": "3.1.4",
+    "css-loader": "0.26.2",
+    "eslint": "3.16.1",
     "eslint-friendly-formatter": "2.0.7",
-    "eslint-loader": "1.6.1",
-    "expose-loader": "0.7.1",
-    "extract-text-webpack-plugin": "1.0.1",
-    "file-loader": "0.9.0",
+    "eslint-loader": "1.6.3",
+    "expose-loader": "0.7.3",
+    "extract-text-webpack-plugin": "2.0.0",
+    "file-loader": "0.10.1",
     "gulp": "3.9.1",
     "gulp-eslint": "3.0.1",
-    "gulp-inject": "4.1.0",
-    "gulp-jade": "1.1.0",
-    "gulp-ll": "1.0.4",
+    "gulp-inject": "4.2.0",
     "gulp-rimraf": "0.2.1",
     "gulp-sequence": "0.4.6",
     "gulp-util": "3.0.8",
-    "html-loader": "0.4.4",
-    "html-webpack-plugin": "2.24.1",
-    "jade": "1.11.0",
-    "jade-html-loader": "git://github.com/courcelan/jade-html-loader",
+    "html-loader": "0.4.5",
+    "html-webpack-plugin": "2.28.0",
     "jasmine-core": "2.5.2",
     "json-loader": "0.5.4",
-    "karma": "0.13.22",
+    "karma": "1.5.0",
     "karma-babel-preprocessor": "6.0.1",
     "karma-jasmine": "1.1.0",
     "karma-mocha": "1.3.0",
     "karma-mocha-reporter": "2.2.2",
     "karma-phantomjs-launcher": "1.0.2",
     "karma-teamcity-reporter": "1.0.0",
-    "karma-webpack": "1.8.1",
-    "mocha": "2.5.3",
+    "karma-webpack": "2.0.2",
+    "mocha": "3.2.0",
     "mocha-teamcity-reporter": "1.1.1",
-    "morgan": "1.7.0",
     "ngtemplate-loader": "1.3.1",
-    "node-sass": "3.13.1",
+    "node-sass": "4.5.0",
     "phantomjs-prebuilt": "2.1.14",
-    "postcss-loader": "0.9.1",
+    "postcss-loader": "1.3.2",
     "progress-bar-webpack-plugin": "1.9.3",
+    "pug-html-loader": "1.1.0",
+    "pug-loader": "2.3.0",
     "require-dir": "0.3.1",
-    "resolve-url-loader": "1.6.1",
-    "sass-loader": "3.2.2",
-    "style-loader": "0.13.1",
+    "resolve-url-loader": "2.0.0",
+    "sass-loader": "6.0.2",
+    "style-loader": "0.13.2",
     "url": "0.11.0",
-    "url-loader": "0.5.7",
-    "webpack": "1.14.0",
-    "webpack-dev-server": "1.16.3",
-    "worker-loader": "0.7.1"
+    "url-loader": "0.5.8",
+    "webpack": "2.2.1",
+    "webpack-dev-server": "2.4.1",
+    "webpack-stream": "3.2.0",
+    "worker-loader": "0.8.0"
   }
 }
diff --git a/modules/web-console/frontend/test/e2e/exampe.test.js b/modules/web-console/frontend/test/e2e/exampe.test.js
index c778c797e61b7..00788bbfbe8dd 100644
--- a/modules/web-console/frontend/test/e2e/exampe.test.js
+++ b/modules/web-console/frontend/test/e2e/exampe.test.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import { suite, test, setup } from 'mocha';
+
 suite('ExampleTestSuite', () => {
     setup(() => {
         // browser.get('http://localhost:9000/');
@@ -37,4 +39,4 @@ suite('ExampleTestSuite', () => {
 
         // element(by.model('ui.email')).sendKeys('jhon@doe.com');
     });
-});
\ No newline at end of file
+});
diff --git a/modules/web-console/frontend/test/karma.conf.babel.js b/modules/web-console/frontend/test/karma.conf.babel.js
index 76a0ba0a240a5..992c4d07a2990 100644
--- a/modules/web-console/frontend/test/karma.conf.babel.js
+++ b/modules/web-console/frontend/test/karma.conf.babel.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import webpackConfig from '../gulpfile.babel.js/webpack';
+import webpack from '../gulpfile.babel.js/webpack';
 import path from 'path';
 
 const basePath = path.resolve('./');
@@ -23,7 +23,7 @@ const basePath = path.resolve('./');
 export default (config) => {
     config.set({
         // Base path that will be used to resolve all patterns (eg. files, exclude).
-        basePath: basePath,
+        basePath,
 
         // Frameworks to use available frameworks: https://npmjs.org/browse/keyword/karma-adapter
         frameworks: ['mocha'],
@@ -46,7 +46,8 @@ export default (config) => {
         preprocessors: {
             'test/**/*.js': ['webpack']
         },
-        webpack: webpackConfig,
+
+        webpack,
 
         webpackMiddleware: {
             noInfo: true
diff --git a/modules/web-console/frontend/test/protractor.conf.js b/modules/web-console/frontend/test/protractor.conf.js
index 3386e66b9f115..4ee1c69078108 100644
--- a/modules/web-console/frontend/test/protractor.conf.js
+++ b/modules/web-console/frontend/test/protractor.conf.js
@@ -23,10 +23,10 @@
 // };
 
 exports.config = {
-  seleniumAddress: 'http://localhost:4444/wd/hub',
+    seleniumAddress: 'http://localhost:4444/wd/hub',
 
-  capabilities: {
-    'browserName': 'chrome'
+    capabilities: {
+        browserName: 'chrome'
     // 'browserName': 'phantomjs',
 
     // /*
@@ -40,11 +40,11 @@ exports.config = {
     //  * See https://github.com/detro/ghostdriver#faq
     //  */
     // 'phantomjs.ghostdriver.cli.args': ['--loglevel=DEBUG']
-  },
+    },
 
-  specs: ['test/e2e/*.js'],
+    specs: ['test/e2e/*.js'],
 
-  jasmineNodeOpts: {
-    showColors: true
-  }
+    jasmineNodeOpts: {
+        showColors: true
+    }
 };
diff --git a/modules/web-console/frontend/test/unit/JavaTransformer.test.js b/modules/web-console/frontend/test/unit/JavaTransformer.test.js
index 3f390003722da..7dacc1681e093 100644
--- a/modules/web-console/frontend/test/unit/JavaTransformer.test.js
+++ b/modules/web-console/frontend/test/unit/JavaTransformer.test.js
@@ -20,7 +20,7 @@ import JavaTypes from '../../app/services/JavaTypes.service.js';
 import generator from '../../app/modules/configuration/generator/ConfigurationGenerator';
 import transformer from '../../app/modules/configuration/generator/JavaTransformer.service';
 
-import { assert } from 'chai';
+import { suite, test } from 'mocha';
 
 suite.skip('JavaTransformerTestsSuite', () => {
     test('AtomicConfiguration', () => {
diff --git a/modules/web-console/frontend/test/unit/JavaTypes.test.js b/modules/web-console/frontend/test/unit/JavaTypes.test.js
index 49e78cc75efd4..e42d687e15eaf 100644
--- a/modules/web-console/frontend/test/unit/JavaTypes.test.js
+++ b/modules/web-console/frontend/test/unit/JavaTypes.test.js
@@ -21,113 +21,114 @@ import ClusterDflts from '../../app/modules/configuration/generator/defaults/Clu
 import CacheDflts from '../../app/modules/configuration/generator/defaults/Cache.service';
 import IgfsDflts from '../../app/modules/configuration/generator/defaults/IGFS.service';
 
-const INSTANCE = new JavaTypes(new ClusterDflts(), new CacheDflts(), new IgfsDflts());
+const instance = new JavaTypes(new ClusterDflts(), new CacheDflts(), new IgfsDflts());
 
+import { suite, test } from 'mocha';
 import { assert } from 'chai';
 
 suite('JavaTypesTestsSuite', () => {
     test('nonBuiltInClass', () => {
-        assert.equal(INSTANCE.nonBuiltInClass('BigDecimal'), false);
-        assert.equal(INSTANCE.nonBuiltInClass('java.math.BigDecimal'), false);
+        assert.equal(instance.nonBuiltInClass('BigDecimal'), false);
+        assert.equal(instance.nonBuiltInClass('java.math.BigDecimal'), false);
 
-        assert.equal(INSTANCE.nonBuiltInClass('String'), false);
-        assert.equal(INSTANCE.nonBuiltInClass('java.lang.String'), false);
+        assert.equal(instance.nonBuiltInClass('String'), false);
+        assert.equal(instance.nonBuiltInClass('java.lang.String'), false);
 
-        assert.equal(INSTANCE.nonBuiltInClass('Timestamp'), false);
-        assert.equal(INSTANCE.nonBuiltInClass('java.sql.Timestamp'), false);
+        assert.equal(instance.nonBuiltInClass('Timestamp'), false);
+        assert.equal(instance.nonBuiltInClass('java.sql.Timestamp'), false);
 
-        assert.equal(INSTANCE.nonBuiltInClass('Date'), false);
-        assert.equal(INSTANCE.nonBuiltInClass('java.sql.Date'), false);
+        assert.equal(instance.nonBuiltInClass('Date'), false);
+        assert.equal(instance.nonBuiltInClass('java.sql.Date'), false);
 
-        assert.equal(INSTANCE.nonBuiltInClass('Date'), false);
-        assert.equal(INSTANCE.nonBuiltInClass('java.util.Date'), false);
+        assert.equal(instance.nonBuiltInClass('Date'), false);
+        assert.equal(instance.nonBuiltInClass('java.util.Date'), false);
 
-        assert.equal(INSTANCE.nonBuiltInClass('CustomClass'), true);
-        assert.equal(INSTANCE.nonBuiltInClass('java.util.CustomClass'), true);
-        assert.equal(INSTANCE.nonBuiltInClass('my.package.CustomClass'), true);
+        assert.equal(instance.nonBuiltInClass('CustomClass'), true);
+        assert.equal(instance.nonBuiltInClass('java.util.CustomClass'), true);
+        assert.equal(instance.nonBuiltInClass('my.package.CustomClass'), true);
     });
 
     test('nonEnum', () => {
-        assert.equal(INSTANCE.nonEnum('org.apache.ignite.cache.CacheMode'), false);
-        assert.equal(INSTANCE.nonEnum('org.apache.ignite.transactions.TransactionConcurrency'), false);
-        assert.equal(INSTANCE.nonEnum('org.apache.ignite.cache.CacheWriteSynchronizationMode'), false);
-        assert.equal(INSTANCE.nonEnum('org.apache.ignite.igfs.IgfsIpcEndpointType'), false);
-        assert.equal(INSTANCE.nonEnum('java.io.Serializable'), true);
-        assert.equal(INSTANCE.nonEnum('BigDecimal'), true);
+        assert.equal(instance.nonEnum('org.apache.ignite.cache.CacheMode'), false);
+        assert.equal(instance.nonEnum('org.apache.ignite.transactions.TransactionConcurrency'), false);
+        assert.equal(instance.nonEnum('org.apache.ignite.cache.CacheWriteSynchronizationMode'), false);
+        assert.equal(instance.nonEnum('org.apache.ignite.igfs.IgfsIpcEndpointType'), false);
+        assert.equal(instance.nonEnum('java.io.Serializable'), true);
+        assert.equal(instance.nonEnum('BigDecimal'), true);
     });
 
     test('shortClassName', () => {
-        assert.equal(INSTANCE.shortClassName('java.math.BigDecimal'), 'BigDecimal');
-        assert.equal(INSTANCE.shortClassName('BigDecimal'), 'BigDecimal');
-        assert.equal(INSTANCE.shortClassName('int'), 'int');
-        assert.equal(INSTANCE.shortClassName('java.lang.Integer'), 'Integer');
-        assert.equal(INSTANCE.shortClassName('Integer'), 'Integer');
-        assert.equal(INSTANCE.shortClassName('java.util.UUID'), 'UUID');
-        assert.equal(INSTANCE.shortClassName('java.sql.Date'), 'Date');
-        assert.equal(INSTANCE.shortClassName('Date'), 'Date');
-        assert.equal(INSTANCE.shortClassName('com.my.Abstract'), 'Abstract');
-        assert.equal(INSTANCE.shortClassName('Abstract'), 'Abstract');
+        assert.equal(instance.shortClassName('java.math.BigDecimal'), 'BigDecimal');
+        assert.equal(instance.shortClassName('BigDecimal'), 'BigDecimal');
+        assert.equal(instance.shortClassName('int'), 'int');
+        assert.equal(instance.shortClassName('java.lang.Integer'), 'Integer');
+        assert.equal(instance.shortClassName('Integer'), 'Integer');
+        assert.equal(instance.shortClassName('java.util.UUID'), 'UUID');
+        assert.equal(instance.shortClassName('java.sql.Date'), 'Date');
+        assert.equal(instance.shortClassName('Date'), 'Date');
+        assert.equal(instance.shortClassName('com.my.Abstract'), 'Abstract');
+        assert.equal(instance.shortClassName('Abstract'), 'Abstract');
     });
 
     test('fullClassName', () => {
-        assert.equal(INSTANCE.fullClassName('BigDecimal'), 'java.math.BigDecimal');
+        assert.equal(instance.fullClassName('BigDecimal'), 'java.math.BigDecimal');
     });
 
     test('validIdentifier', () => {
-        assert.equal(INSTANCE.validIdentifier('myIdent'), true);
-        assert.equal(INSTANCE.validIdentifier('java.math.BigDecimal'), false);
-        assert.equal(INSTANCE.validIdentifier('2Demo'), false);
-        assert.equal(INSTANCE.validIdentifier('abra kadabra'), false);
-        assert.equal(INSTANCE.validIdentifier(undefined), false);
-        assert.equal(INSTANCE.validIdentifier(null), false);
-        assert.equal(INSTANCE.validIdentifier(''), false);
-        assert.equal(INSTANCE.validIdentifier(' '), false);
+        assert.equal(instance.validIdentifier('myIdent'), true);
+        assert.equal(instance.validIdentifier('java.math.BigDecimal'), false);
+        assert.equal(instance.validIdentifier('2Demo'), false);
+        assert.equal(instance.validIdentifier('abra kadabra'), false);
+        assert.equal(instance.validIdentifier(), false);
+        assert.equal(instance.validIdentifier(null), false);
+        assert.equal(instance.validIdentifier(''), false);
+        assert.equal(instance.validIdentifier(' '), false);
     });
 
     test('validClassName', () => {
-        assert.equal(INSTANCE.validClassName('java.math.BigDecimal'), true);
-        assert.equal(INSTANCE.validClassName('2Demo'), false);
-        assert.equal(INSTANCE.validClassName('abra kadabra'), false);
-        assert.equal(INSTANCE.validClassName(undefined), false);
-        assert.equal(INSTANCE.validClassName(null), false);
-        assert.equal(INSTANCE.validClassName(''), false);
-        assert.equal(INSTANCE.validClassName(' '), false);
+        assert.equal(instance.validClassName('java.math.BigDecimal'), true);
+        assert.equal(instance.validClassName('2Demo'), false);
+        assert.equal(instance.validClassName('abra kadabra'), false);
+        assert.equal(instance.validClassName(), false);
+        assert.equal(instance.validClassName(null), false);
+        assert.equal(instance.validClassName(''), false);
+        assert.equal(instance.validClassName(' '), false);
     });
 
     test('validPackage', () => {
-        assert.equal(INSTANCE.validPackage('java.math.BigDecimal'), true);
-        assert.equal(INSTANCE.validPackage('my.org.SomeClass'), true);
-        assert.equal(INSTANCE.validPackage('25'), false);
-        assert.equal(INSTANCE.validPackage('abra kadabra'), false);
-        assert.equal(INSTANCE.validPackage(''), false);
-        assert.equal(INSTANCE.validPackage(' '), false);
+        assert.equal(instance.validPackage('java.math.BigDecimal'), true);
+        assert.equal(instance.validPackage('my.org.SomeClass'), true);
+        assert.equal(instance.validPackage('25'), false);
+        assert.equal(instance.validPackage('abra kadabra'), false);
+        assert.equal(instance.validPackage(''), false);
+        assert.equal(instance.validPackage(' '), false);
     });
 
     test('packageSpecified', () => {
-        assert.equal(INSTANCE.packageSpecified('java.math.BigDecimal'), true);
-        assert.equal(INSTANCE.packageSpecified('BigDecimal'), false);
+        assert.equal(instance.packageSpecified('java.math.BigDecimal'), true);
+        assert.equal(instance.packageSpecified('BigDecimal'), false);
     });
 
     test('isKeyword', () => {
-        assert.equal(INSTANCE.isKeyword('abstract'), true);
-        assert.equal(INSTANCE.isKeyword('Abstract'), true);
-        assert.equal(INSTANCE.isKeyword('abra kadabra'), false);
-        assert.equal(INSTANCE.isKeyword(undefined), false);
-        assert.equal(INSTANCE.isKeyword(null), false);
-        assert.equal(INSTANCE.isKeyword(''), false);
-        assert.equal(INSTANCE.isKeyword(' '), false);
+        assert.equal(instance.isKeyword('abstract'), true);
+        assert.equal(instance.isKeyword('Abstract'), true);
+        assert.equal(instance.isKeyword('abra kadabra'), false);
+        assert.equal(instance.isKeyword(), false);
+        assert.equal(instance.isKeyword(null), false);
+        assert.equal(instance.isKeyword(''), false);
+        assert.equal(instance.isKeyword(' '), false);
     });
 
     test('isPrimitive', () => {
-        assert.equal(INSTANCE.isPrimitive('boolean'), true);
+        assert.equal(instance.isPrimitive('boolean'), true);
     });
 
     test('validUUID', () => {
-        assert.equal(INSTANCE.validUUID('123e4567-e89b-12d3-a456-426655440000'), true);
-        assert.equal(INSTANCE.validUUID('12345'), false);
-        assert.equal(INSTANCE.validUUID(undefined), false);
-        assert.equal(INSTANCE.validUUID(null), false);
-        assert.equal(INSTANCE.validUUID(''), false);
-        assert.equal(INSTANCE.validUUID(' '), false);
+        assert.equal(instance.validUUID('123e4567-e89b-12d3-a456-426655440000'), true);
+        assert.equal(instance.validUUID('12345'), false);
+        assert.equal(instance.validUUID(), false);
+        assert.equal(instance.validUUID(null), false);
+        assert.equal(instance.validUUID(''), false);
+        assert.equal(instance.validUUID(' '), false);
     });
 });
diff --git a/modules/web-console/frontend/test/unit/SharpTransformer.test.js b/modules/web-console/frontend/test/unit/SharpTransformer.test.js
index 20de266af6bb1..3fb9265b73c55 100644
--- a/modules/web-console/frontend/test/unit/SharpTransformer.test.js
+++ b/modules/web-console/frontend/test/unit/SharpTransformer.test.js
@@ -18,7 +18,7 @@
 import generator from '../../app/modules/configuration/generator/PlatformGenerator';
 import transformer from '../../app/modules/configuration/generator/SharpTransformer.service';
 
-import { assert } from 'chai';
+import { suite, test } from 'mocha';
 
 suite.skip('SharpTransformerTestsSuite', () => {
     test('AtomicConfiguration', () => {
diff --git a/modules/web-console/frontend/test/unit/SpringTransformer.test.js b/modules/web-console/frontend/test/unit/SpringTransformer.test.js
index 7998f666befe4..e96cab3c2d2f4 100644
--- a/modules/web-console/frontend/test/unit/SpringTransformer.test.js
+++ b/modules/web-console/frontend/test/unit/SpringTransformer.test.js
@@ -20,7 +20,7 @@ import JavaTypes from '../../app/services/JavaTypes.service.js';
 import generator from '../../app/modules/configuration/generator/ConfigurationGenerator';
 import transformer from '../../app/modules/configuration/generator/SpringTransformer.service';
 
-import { assert } from 'chai';
+import { suite, test } from 'mocha';
 
 suite.skip('SpringTransformerTestsSuite', () => {
     test('AtomicConfiguration', () => {
diff --git a/modules/web-console/frontend/test/unit/SqlTypes.test.js b/modules/web-console/frontend/test/unit/SqlTypes.test.js
index 2d54bdfdb1f25..17f899cf90900 100644
--- a/modules/web-console/frontend/test/unit/SqlTypes.test.js
+++ b/modules/web-console/frontend/test/unit/SqlTypes.test.js
@@ -19,6 +19,7 @@ import SqlTypes from '../../app/services/SqlTypes.service.js';
 
 const INSTANCE = new SqlTypes();
 
+import { suite, test } from 'mocha';
 import { assert } from 'chai';
 
 suite('SqlTypesTestsSuite', () => {
@@ -27,7 +28,7 @@ suite('SqlTypesTestsSuite', () => {
         assert.equal(INSTANCE.validIdentifier('java.math.BigDecimal'), false);
         assert.equal(INSTANCE.validIdentifier('2Demo'), false);
         assert.equal(INSTANCE.validIdentifier('abra kadabra'), false);
-        assert.equal(INSTANCE.validIdentifier(undefined), false);
+        assert.equal(INSTANCE.validIdentifier(), false);
         assert.equal(INSTANCE.validIdentifier(null), false);
         assert.equal(INSTANCE.validIdentifier(''), false);
         assert.equal(INSTANCE.validIdentifier(' '), false);
@@ -38,7 +39,7 @@ suite('SqlTypesTestsSuite', () => {
         assert.equal(INSTANCE.isKeyword('Group'), true);
         assert.equal(INSTANCE.isKeyword('select'), true);
         assert.equal(INSTANCE.isKeyword('abra kadabra'), false);
-        assert.equal(INSTANCE.isKeyword(undefined), false);
+        assert.equal(INSTANCE.isKeyword(), false);
         assert.equal(INSTANCE.isKeyword(null), false);
         assert.equal(INSTANCE.isKeyword(''), false);
         assert.equal(INSTANCE.isKeyword(' '), false);
@@ -47,5 +48,5 @@ suite('SqlTypesTestsSuite', () => {
     test('findJdbcType', () => {
         assert.equal(INSTANCE.findJdbcType(0).dbName, 'NULL');
         assert.equal(INSTANCE.findJdbcType(5555).dbName, 'Unknown');
-    })
+    });
 });
diff --git a/modules/web-console/frontend/test/unit/UserAuth.test.js b/modules/web-console/frontend/test/unit/UserAuth.test.js
index dbba1f63fa747..7b6b24c2944f7 100644
--- a/modules/web-console/frontend/test/unit/UserAuth.test.js
+++ b/modules/web-console/frontend/test/unit/UserAuth.test.js
@@ -15,7 +15,9 @@
  * limitations under the License.
  */
 
-import AuthService from '../../app/modules/user/Auth.service';
+// import AuthService from '../../app/modules/user/Auth.service';
+
+import { suite, test } from 'mocha';
 
 suite('AuthServiceTestsSuite', () => {
     test('SignIn', (done) => {
diff --git a/modules/web-console/frontend/test/unit/Version.test.js b/modules/web-console/frontend/test/unit/Version.test.js
index 72685eae03239..d8c8f656367ff 100644
--- a/modules/web-console/frontend/test/unit/Version.test.js
+++ b/modules/web-console/frontend/test/unit/Version.test.js
@@ -19,6 +19,7 @@ import VersionService from '../../app/modules/configuration/Version.service';
 
 const INSTANCE = new VersionService();
 
+import { suite, test } from 'mocha';
 import { assert } from 'chai';
 
 suite('VersionServiceTestsSuite', () => {
diff --git a/modules/web-console/frontend/test/unit/defaultName.filter.test.js b/modules/web-console/frontend/test/unit/defaultName.filter.test.js
index 5f282900b4f97..2eaf2238e17ee 100644
--- a/modules/web-console/frontend/test/unit/defaultName.filter.test.js
+++ b/modules/web-console/frontend/test/unit/defaultName.filter.test.js
@@ -17,22 +17,25 @@
 
 import defaultName from '../../app/filters/default-name.filter';
 
+import { suite, test } from 'mocha';
 import { assert } from 'chai';
 
-const INSTANCE = defaultName[0]();
+const instance = defaultName[0]();
 
 suite('defaultName', () => {
     test('defaultName filter', () => {
-        assert.equal(INSTANCE(''), '<default>');
-        assert.equal(INSTANCE(null), '<default>');
-        assert.equal(INSTANCE(undefined), '<default>');
-        assert.equal(INSTANCE('', false), '<default>');
-        assert.equal(INSTANCE(null, false), '<default>');
-        assert.equal(INSTANCE(undefined, false), '<default>');
-        assert.equal(INSTANCE('', true), '&lt;default&gt;');
-        assert.equal(INSTANCE(null, true), '&lt;default&gt;');
-        assert.equal(INSTANCE(undefined, true), '&lt;default&gt;');
-        assert.equal(INSTANCE("name", false), 'name');
-        assert.equal(INSTANCE("name", true), 'name');
+        let undef;
+
+        assert.equal(instance(''), '<default>');
+        assert.equal(instance(null), '<default>');
+        assert.equal(instance(), '<default>');
+        assert.equal(instance('', false), '<default>');
+        assert.equal(instance(null, false), '<default>');
+        assert.equal(instance(undef, false), '<default>');
+        assert.equal(instance('', true), '&lt;default&gt;');
+        assert.equal(instance(null, true), '&lt;default&gt;');
+        assert.equal(instance(undef, true), '&lt;default&gt;');
+        assert.equal(instance('name', false), 'name');
+        assert.equal(instance('name', true), 'name');
     });
 });
diff --git a/modules/web-console/frontend/views/403.jade b/modules/web-console/frontend/views/403.pug
similarity index 100%
rename from modules/web-console/frontend/views/403.jade
rename to modules/web-console/frontend/views/403.pug
diff --git a/modules/web-console/frontend/views/404.jade b/modules/web-console/frontend/views/404.pug
similarity index 100%
rename from modules/web-console/frontend/views/404.jade
rename to modules/web-console/frontend/views/404.pug
diff --git a/modules/web-console/frontend/views/base.jade b/modules/web-console/frontend/views/base.pug
similarity index 94%
rename from modules/web-console/frontend/views/base.jade
rename to modules/web-console/frontend/views/base.pug
index a910d1b2d84b5..b05cdbabeaef1 100644
--- a/modules/web-console/frontend/views/base.jade
+++ b/modules/web-console/frontend/views/base.pug
@@ -14,9 +14,9 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include includes/header
+include ./includes/header
 
 .container.body-container
     .main-content(ui-view='')
 
-include includes/footer
+include ./includes/footer
diff --git a/modules/web-console/frontend/views/configuration/caches.jade b/modules/web-console/frontend/views/configuration/caches.tpl.pug
similarity index 89%
rename from modules/web-console/frontend/views/configuration/caches.jade
rename to modules/web-console/frontend/views/configuration/caches.tpl.pug
index 73a6309d223c6..fca41a895a36f 100644
--- a/modules/web-console/frontend/views/configuration/caches.jade
+++ b/modules/web-console/frontend/views/configuration/caches.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 .docs-header
     h1 Configure Ignite Caches
@@ -36,20 +36,20 @@ include /app/helpers/jade/mixins.jade
             div(bs-collapse='' data-allow-multiple='true' ng-model='ui.activePanels')
                 form.form-horizontal(name='ui.inputForm' novalidate ng-if='contentVisible()')
                     .panel-group
-                        include /app/modules/states/configuration/caches/general.jade
-                        include /app/modules/states/configuration/caches/memory.jade
-                        include /app/modules/states/configuration/caches/query.jade
-                        include /app/modules/states/configuration/caches/store.jade
+                        include /app/modules/states/configuration/caches/general
+                        include /app/modules/states/configuration/caches/memory
+                        include /app/modules/states/configuration/caches/query
+                        include /app/modules/states/configuration/caches/store
 
                         +advanced-options-toggle-default
 
                         div(ng-show='ui.expanded')
-                            include /app/modules/states/configuration/caches/affinity.jade
-                            include /app/modules/states/configuration/caches/concurrency.jade
-                            include /app/modules/states/configuration/caches/near-cache-client.jade
-                            include /app/modules/states/configuration/caches/near-cache-server.jade
-                            include /app/modules/states/configuration/caches/node-filter.jade
-                            include /app/modules/states/configuration/caches/rebalance.jade
-                            include /app/modules/states/configuration/caches/statistics.jade
+                            include /app/modules/states/configuration/caches/affinity
+                            include /app/modules/states/configuration/caches/concurrency
+                            include /app/modules/states/configuration/caches/near-cache-client
+                            include /app/modules/states/configuration/caches/near-cache-server
+                            include /app/modules/states/configuration/caches/node-filter
+                            include /app/modules/states/configuration/caches/rebalance
+                            include /app/modules/states/configuration/caches/statistics
 
                             +advanced-options-toggle-default
diff --git a/modules/web-console/frontend/views/configuration/clusters.jade b/modules/web-console/frontend/views/configuration/clusters.tpl.pug
similarity index 83%
rename from modules/web-console/frontend/views/configuration/clusters.jade
rename to modules/web-console/frontend/views/configuration/clusters.tpl.pug
index 78264647b800f..c97901260f243 100644
--- a/modules/web-console/frontend/views/configuration/clusters.jade
+++ b/modules/web-console/frontend/views/configuration/clusters.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 .docs-header
     h1 Configure Ignite Clusters
@@ -36,33 +36,33 @@ include /app/helpers/jade/mixins.jade
             div(bs-collapse='' data-allow-multiple='true' ng-model='ui.activePanels')
                 form.form-horizontal(name='ui.inputForm' novalidate ng-if='contentVisible()')
                     .panel-group
-                        include /app/modules/states/configuration/clusters/general.jade
+                        include /app/modules/states/configuration/clusters/general
 
                         +advanced-options-toggle-default
 
                         div(ng-show='ui.expanded')
-                            include /app/modules/states/configuration/clusters/atomic.jade
-                            include /app/modules/states/configuration/clusters/binary.jade
-                            include /app/modules/states/configuration/clusters/cache-key-cfg.jade
-                            include /app/modules/states/configuration/clusters/checkpoint.jade
-                            include /app/modules/states/configuration/clusters/collision.jade
-                            include /app/modules/states/configuration/clusters/communication.jade
-                            include /app/modules/states/configuration/clusters/connector.jade
-                            include /app/modules/states/configuration/clusters/deployment.jade
-                            include /app/modules/states/configuration/clusters/discovery.jade
-                            include /app/modules/states/configuration/clusters/events.jade
-                            include /app/modules/states/configuration/clusters/failover.jade
-                            include /app/modules/states/configuration/clusters/igfs.jade
-                            include /app/modules/states/configuration/clusters/load-balancing.jade
-                            include /app/modules/states/configuration/clusters/logger.jade
-                            include /app/modules/states/configuration/clusters/marshaller.jade
-                            include /app/modules/states/configuration/clusters/metrics.jade
-                            include /app/modules/states/configuration/clusters/odbc.jade
-                            include /app/modules/states/configuration/clusters/ssl.jade
-                            include /app/modules/states/configuration/clusters/swap.jade
-                            include /app/modules/states/configuration/clusters/thread.jade
-                            include /app/modules/states/configuration/clusters/time.jade
-                            include /app/modules/states/configuration/clusters/transactions.jade
-                            include /app/modules/states/configuration/clusters/attributes.jade
+                            include /app/modules/states/configuration/clusters/atomic
+                            include /app/modules/states/configuration/clusters/binary
+                            include /app/modules/states/configuration/clusters/cache-key-cfg
+                            include /app/modules/states/configuration/clusters/checkpoint
+                            include /app/modules/states/configuration/clusters/collision
+                            include /app/modules/states/configuration/clusters/communication
+                            include /app/modules/states/configuration/clusters/connector
+                            include /app/modules/states/configuration/clusters/deployment
+                            include /app/modules/states/configuration/clusters/discovery
+                            include /app/modules/states/configuration/clusters/events
+                            include /app/modules/states/configuration/clusters/failover
+                            include /app/modules/states/configuration/clusters/igfs
+                            include /app/modules/states/configuration/clusters/load-balancing
+                            include /app/modules/states/configuration/clusters/logger
+                            include /app/modules/states/configuration/clusters/marshaller
+                            include /app/modules/states/configuration/clusters/metrics
+                            include /app/modules/states/configuration/clusters/odbc
+                            include /app/modules/states/configuration/clusters/ssl
+                            include /app/modules/states/configuration/clusters/swap
+                            include /app/modules/states/configuration/clusters/thread
+                            include /app/modules/states/configuration/clusters/time
+                            include /app/modules/states/configuration/clusters/transactions
+                            include /app/modules/states/configuration/clusters/attributes
 
                             +advanced-options-toggle-default
diff --git a/modules/web-console/frontend/views/configuration/domains-import.jade b/modules/web-console/frontend/views/configuration/domains-import.tpl.pug
similarity index 99%
rename from modules/web-console/frontend/views/configuration/domains-import.jade
rename to modules/web-console/frontend/views/configuration/domains-import.tpl.pug
index bbcb391cbedef..baaa2d34cf1ad 100644
--- a/modules/web-console/frontend/views/configuration/domains-import.jade
+++ b/modules/web-console/frontend/views/configuration/domains-import.tpl.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 mixin chk(mdl, change, tip)
     input(type='checkbox' ng-model=mdl ng-change=change bs-tooltip='' data-title=tip data-trigger='hover' data-placement='top')
 
 mixin td-ellipses-lbl(w, lbl)
-    td.td-ellipsis(width='#{w}' style='min-width: #{w}; max-width: #{w}')
+    td.td-ellipsis(width=`${w}` style=`min-width: ${w}; max-width: ${w}`)
         label #{lbl}
 
 .modal.modal-domain-import.center(role='dialog')
diff --git a/modules/web-console/frontend/views/configuration/domains.jade b/modules/web-console/frontend/views/configuration/domains.tpl.pug
similarity index 97%
rename from modules/web-console/frontend/views/configuration/domains.jade
rename to modules/web-console/frontend/views/configuration/domains.tpl.pug
index 14e9ca5240ea0..33528c7c8dd68 100644
--- a/modules/web-console/frontend/views/configuration/domains.jade
+++ b/modules/web-console/frontend/views/configuration/domains.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 .docs-header
     h1 Configure Domain Model And SQL Queries
@@ -61,6 +61,6 @@ include /app/helpers/jade/mixins.jade
             div(bs-collapse='' data-allow-multiple='true' ng-model='ui.activePanels')
                 form.form-horizontal(name='ui.inputForm' novalidate ng-if='contentVisible()')
                     .panel-group
-                        include /app/modules/states/configuration/domains/general.jade
-                        include /app/modules/states/configuration/domains/query.jade
-                        include /app/modules/states/configuration/domains/store.jade
+                        include /app/modules/states/configuration/domains/general
+                        include /app/modules/states/configuration/domains/query
+                        include /app/modules/states/configuration/domains/store
diff --git a/modules/web-console/frontend/views/configuration/igfs.jade b/modules/web-console/frontend/views/configuration/igfs.tpl.pug
similarity index 94%
rename from modules/web-console/frontend/views/configuration/igfs.jade
rename to modules/web-console/frontend/views/configuration/igfs.tpl.pug
index 10c902be3b3be..89e09209d0cc3 100644
--- a/modules/web-console/frontend/views/configuration/igfs.jade
+++ b/modules/web-console/frontend/views/configuration/igfs.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 .docs-header
     h1 Configure Ignite In-memory File Systems
@@ -37,15 +37,15 @@ include /app/helpers/jade/mixins.jade
             div(bs-collapse='' data-allow-multiple='true' ng-model='ui.activePanels')
                 form.form-horizontal(name='ui.inputForm' novalidate ng-if='contentVisible()')
                     .panel-group
-                        include /app/modules/states/configuration/igfs/general.jade
+                        include /app/modules/states/configuration/igfs/general
 
                         +advanced-options-toggle-default
 
                         div(ng-show='ui.expanded')
-                            include /app/modules/states/configuration/igfs/secondary.jade
-                            include /app/modules/states/configuration/igfs/ipc.jade
-                            include /app/modules/states/configuration/igfs/fragmentizer.jade
-                            include /app/modules/states/configuration/igfs/dual.jade
-                            include /app/modules/states/configuration/igfs/misc.jade
+                            include /app/modules/states/configuration/igfs/secondary
+                            include /app/modules/states/configuration/igfs/ipc
+                            include /app/modules/states/configuration/igfs/fragmentizer
+                            include /app/modules/states/configuration/igfs/dual
+                            include /app/modules/states/configuration/igfs/misc
 
                             +advanced-options-toggle-default
diff --git a/modules/web-console/frontend/views/configuration/sidebar.jade b/modules/web-console/frontend/views/configuration/sidebar.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/configuration/sidebar.jade
rename to modules/web-console/frontend/views/configuration/sidebar.tpl.pug
diff --git a/modules/web-console/frontend/views/configuration/summary-project-structure.jade b/modules/web-console/frontend/views/configuration/summary-project-structure.tpl.pug
similarity index 99%
rename from modules/web-console/frontend/views/configuration/summary-project-structure.jade
rename to modules/web-console/frontend/views/configuration/summary-project-structure.tpl.pug
index 29d4538c7cb90..31a557fca15da 100644
--- a/modules/web-console/frontend/views/configuration/summary-project-structure.jade
+++ b/modules/web-console/frontend/views/configuration/summary-project-structure.tpl.pug
@@ -13,6 +13,7 @@
     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     See the License for the specific language governing permissions and
     limitations under the License.
+
 .popover.summary-project-structure
     h3.popover-title
         label.labelField Project structure
diff --git a/modules/web-console/frontend/views/configuration/summary-tabs.jade b/modules/web-console/frontend/views/configuration/summary-tabs.pug
similarity index 97%
rename from modules/web-console/frontend/views/configuration/summary-tabs.jade
rename to modules/web-console/frontend/views/configuration/summary-tabs.pug
index 847b42f70b2f5..d05e3a9eae43b 100644
--- a/modules/web-console/frontend/views/configuration/summary-tabs.jade
+++ b/modules/web-console/frontend/views/configuration/summary-tabs.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-ul.nav(ng-class='$navClass', role='tablist')
+ul.nav(ng-class='$navClass' role='tablist')
     li(role='presentation' ng-repeat='$pane in $panes track by $index' ng-class='[ $isActive($pane, $index) ? $activeClass : "", $pane.disabled ? "disabled" : "" ]')
         a.summary-tab(ng-show='$pane.title != "POJO" || (cluster | hasPojo)' ng-switch='$pane.title' role='tab' data-toggle='tab' ng-click='!$pane.disabled && $setActive($pane.name || $index)' data-index='{{ $index }}' aria-controls='$pane.title') {{$pane.title}}
             img(ng-switch-when='XML' src='/images/xml.png')
diff --git a/modules/web-console/frontend/views/configuration/summary.jade b/modules/web-console/frontend/views/configuration/summary.tpl.pug
similarity index 93%
rename from modules/web-console/frontend/views/configuration/summary.jade
rename to modules/web-console/frontend/views/configuration/summary.tpl.pug
index a04f0dbbf074b..9fdd0de3f3097 100644
--- a/modules/web-console/frontend/views/configuration/summary.jade
+++ b/modules/web-console/frontend/views/configuration/summary.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 mixin hard-link(ref, txt)
     a(style='color:#ec1c24' href=ref target='_blank') #{txt}
@@ -44,7 +44,7 @@ mixin hard-link(ref, txt)
                             i.fa.fa-fw.fa-refresh.fa-spin(ng-show='isPrepareDownloading')
                             span.tipLabel Download project
                     button.btn.btn-primary(bs-tooltip='' data-title='Preview generated project structure' data-placement='bottom')
-                        div(bs-popover data-template-url='/configuration/summary-project-structure.html', data-placement='bottom', data-trigger='click' data-auto-close='true')
+                        div(bs-popover data-template-url='{{ ctrl.summaryProjectStructureTemplateUrl }}', data-placement='bottom', data-trigger='click' data-auto-close='true')
                             i.fa.fa-sitemap
                             label.tipLabel Project structure
                     button.btn.btn-primary(id='proprietary-jdbc-drivers' ng-if='downloadJdbcDriversVisible()' ng-click='downloadJdbcDrivers()' bs-tooltip='' data-title='Open proprietary JDBC drivers download pages' data-placement='bottom') Download JDBC drivers
@@ -61,7 +61,7 @@ mixin hard-link(ref, txt)
 
                     .panel-collapse(id='server' role='tabpanel' bs-collapse-target)
                         .summary-tabs(ignite-ui-ace-tabs)
-                            div(bs-tabs data-bs-active-pane="tabsServer.activeTab" template='configuration/summary-tabs.html')
+                            div(bs-tabs data-bs-active-pane='tabsServer.activeTab' data-template='summary-tabs.html')
                                 div(bs-pane title='XML')
                                     ignite-ui-ace-spring(ng-if='tabsServer.activeTab == 0 || tabsServer.init[0]' ng-init='tabsServer.init[0] = true' data-master='cluster' data-generator='igniteConfiguration' data-no-deep-watch)
                                 div(bs-pane title='Java')
@@ -78,7 +78,7 @@ mixin hard-link(ref, txt)
 
                     .panel-collapse(id='client' role='tabpanel' bs-collapse-target)
                         .summary-tabs(ignite-ui-ace-tabs)
-                            div(bs-tabs data-bs-active-pane="tabsClient.activeTab" template='configuration/summary-tabs.html')
+                            div(bs-tabs data-bs-active-pane='tabsClient.activeTab' data-template='summary-tabs.html')
                                 div(bs-pane title='XML')
                                     ignite-ui-ace-spring(ng-if='tabsClient.activeTab == 0 || tabsClient.init[0]' ng-init='tabsClient.init[0] = true' data-master='cluster' data-generator='igniteConfiguration' data-client='true' data-no-deep-watch)
                                 div(bs-pane title='Java')
diff --git a/modules/web-console/frontend/views/includes/footer.jade b/modules/web-console/frontend/views/includes/footer.pug
similarity index 100%
rename from modules/web-console/frontend/views/includes/footer.jade
rename to modules/web-console/frontend/views/includes/footer.pug
diff --git a/modules/web-console/frontend/views/includes/header.jade b/modules/web-console/frontend/views/includes/header.pug
similarity index 100%
rename from modules/web-console/frontend/views/includes/header.jade
rename to modules/web-console/frontend/views/includes/header.pug
diff --git a/modules/web-console/frontend/views/index.jade b/modules/web-console/frontend/views/index.pug
similarity index 100%
rename from modules/web-console/frontend/views/index.jade
rename to modules/web-console/frontend/views/index.pug
diff --git a/modules/web-console/frontend/views/reset.jade b/modules/web-console/frontend/views/reset.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/reset.jade
rename to modules/web-console/frontend/views/reset.tpl.pug
diff --git a/modules/web-console/frontend/views/settings/admin.jade b/modules/web-console/frontend/views/settings/admin.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/settings/admin.jade
rename to modules/web-console/frontend/views/settings/admin.tpl.pug
diff --git a/modules/web-console/frontend/views/settings/profile.jade b/modules/web-console/frontend/views/settings/profile.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/settings/profile.jade
rename to modules/web-console/frontend/views/settings/profile.tpl.pug
diff --git a/modules/web-console/frontend/views/signin.jade b/modules/web-console/frontend/views/signin.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/signin.jade
rename to modules/web-console/frontend/views/signin.tpl.pug
diff --git a/modules/web-console/frontend/views/sql/cache-metadata.jade b/modules/web-console/frontend/views/sql/cache-metadata.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/sql/cache-metadata.jade
rename to modules/web-console/frontend/views/sql/cache-metadata.tpl.pug
diff --git a/modules/web-console/frontend/views/sql/chart-settings.jade b/modules/web-console/frontend/views/sql/chart-settings.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/sql/chart-settings.jade
rename to modules/web-console/frontend/views/sql/chart-settings.tpl.pug
diff --git a/modules/web-console/frontend/views/sql/notebook-new.jade b/modules/web-console/frontend/views/sql/notebook-new.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/sql/notebook-new.jade
rename to modules/web-console/frontend/views/sql/notebook-new.tpl.pug
diff --git a/modules/web-console/frontend/views/sql/paragraph-rate.jade b/modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/sql/paragraph-rate.jade
rename to modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug
diff --git a/modules/web-console/frontend/views/sql/sql.jade b/modules/web-console/frontend/views/sql/sql.tpl.pug
similarity index 95%
rename from modules/web-console/frontend/views/sql/sql.jade
rename to modules/web-console/frontend/views/sql/sql.tpl.pug
index 61d5b3037825a..931d6327888d3 100644
--- a/modules/web-console/frontend/views/sql/sql.jade
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
-include /app/components/ui-grid-settings/ui-grid-settings.jade
+include /app/helpers/jade/mixins
+include /app/components/ui-grid-settings/ui-grid-settings
 
 mixin btn-toolbar(btn, click, tip, focusId)
     i.btn.btn-default.fa(class=btn ng-click=click bs-tooltip='' data-title=tip ignite-on-click-focus=focusId data-trigger='hover' data-placement='bottom')
 
 mixin btn-toolbar-data(btn, kind, tip)
-    i.btn.btn-default.fa(class=btn ng-click='setResult(paragraph, "#{kind}")' ng-class='{active: resultEq(paragraph, "#{kind}")}' bs-tooltip='' data-title=tip data-trigger='hover' data-placement='bottom')
+    i.btn.btn-default.fa(class=btn ng-click=`setResult(paragraph, '${kind}')` ng-class=`{active: resultEq(paragraph, '${kind}')}` bs-tooltip='' data-title=tip data-trigger='hover' data-placement='bottom')
 
 mixin result-toolbar
     .btn-group(ng-model='paragraph.result' ng-click='$event.stopPropagation()' style='left: 50%; margin: 0 0 0 -70px;display: block;')
@@ -35,7 +35,7 @@ mixin chart-settings
     .total.row
         .col-xs-4
             .chart-settings-link(ng-show='paragraph.chart && paragraph.chartColumns.length > 0')
-                a(title='Click to show chart settings dialog' ng-click='$event.stopPropagation()' bs-popover data-template-url='/sql/chart-settings.html' data-placement='bottom' data-auto-close='1' data-trigger='click')
+                a(title='Click to show chart settings dialog' ng-click='$event.stopPropagation()' bs-popover data-template-url='{{ $ctrl.chartSettingsTemplateUrl }}' data-placement='bottom' data-auto-close='1' data-trigger='click')
                     i.fa.fa-bars
                     | Chart settings
                 div(ng-show='paragraphTimeSpanVisible(paragraph)')
@@ -92,7 +92,7 @@ mixin paragraph-rename
 
 mixin query-settings
     label.tipLabel(bs-tooltip data-placement='bottom' data-title='Configure periodical execution of last successfully executed query') Refresh rate:
-        button.btn.btn-default.fa.fa-clock-o.tipLabel(ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='/sql/paragraph-rate.html' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
+        button.btn.btn-default.fa.fa-clock-o.tipLabel(ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='{{ $ctrl.paragraphRateTemplateUrl }}' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
 
     label.tipLabel(bs-tooltip data-placement='bottom' data-title='Max number of rows to show in query result as one page') Page size:
         button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-select bs-options='item for item in pageSizes')
@@ -201,7 +201,7 @@ mixin paragraph-scan
 
                 -var nextVisibleCondition = 'paragraph.resultType() != "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
 
-                .pull-right(ng-show='#{nextVisibleCondition}' ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
+                .pull-right(ng-show=`${nextVisibleCondition}` ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
                     i.fa.fa-chevron-circle-right
                     a Next
 
@@ -217,7 +217,7 @@ mixin paragraph-query
             .col-xs-4.col-sm-3
                 div(ng-show='caches.length > 0' style='padding: 5px 10px' st-table='displayedCaches' st-safe-src='caches')
                     lable.labelField.labelFormField Caches:
-                    i.fa.fa-database.tipField(title='Click to show cache types metadata dialog' bs-popover data-template-url='/sql/cache-metadata.html' data-placement='bottom' data-trigger='click' data-container='#{{ paragraph.id }}')
+                    i.fa.fa-database.tipField(title='Click to show cache types metadata dialog' bs-popover data-template-url='{{ $ctrl.cacheMetadataTemplateUrl }}' data-placement='bottom' data-trigger='click' data-container='#{{ paragraph.id }}')
                     .input-tip
                         input.form-control(type='text' st-search='label' placeholder='Filter caches...')
                     table.links
@@ -248,11 +248,11 @@ mixin paragraph-query
 
                 -var nextVisibleCondition = 'paragraph.resultType() != "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
 
-                .pull-right(ng-show='#{nextVisibleCondition}' ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
+                .pull-right(ng-show=`${nextVisibleCondition}` ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
                     i.fa.fa-chevron-circle-right
                     a Next
 
-.row(ng-controller='sqlController')
+.row
     .docs-content
         .row(ng-if='notebook' bs-affix style='margin-bottom: 20px;')
             +notebook-rename
diff --git a/modules/web-console/frontend/views/templates/agent-download.jade b/modules/web-console/frontend/views/templates/agent-download.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/agent-download.jade
rename to modules/web-console/frontend/views/templates/agent-download.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/alert.jade b/modules/web-console/frontend/views/templates/alert.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/alert.jade
rename to modules/web-console/frontend/views/templates/alert.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/batch-confirm.jade b/modules/web-console/frontend/views/templates/batch-confirm.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/batch-confirm.jade
rename to modules/web-console/frontend/views/templates/batch-confirm.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/clone.jade b/modules/web-console/frontend/views/templates/clone.tpl.pug
similarity index 98%
rename from modules/web-console/frontend/views/templates/clone.jade
rename to modules/web-console/frontend/views/templates/clone.tpl.pug
index 99ec58be4697d..e62cb18a8fb60 100644
--- a/modules/web-console/frontend/views/templates/clone.jade
+++ b/modules/web-console/frontend/views/templates/clone.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 .modal(tabindex='-1' role='dialog')
     .modal-dialog
diff --git a/modules/web-console/frontend/views/templates/confirm.jade b/modules/web-console/frontend/views/templates/confirm.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/confirm.jade
rename to modules/web-console/frontend/views/templates/confirm.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/demo-info.jade b/modules/web-console/frontend/views/templates/demo-info.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/demo-info.jade
rename to modules/web-console/frontend/views/templates/demo-info.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/dropdown.jade b/modules/web-console/frontend/views/templates/dropdown.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/dropdown.jade
rename to modules/web-console/frontend/views/templates/dropdown.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/getting-started.jade b/modules/web-console/frontend/views/templates/getting-started.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/getting-started.jade
rename to modules/web-console/frontend/views/templates/getting-started.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/message.jade b/modules/web-console/frontend/views/templates/message.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/message.jade
rename to modules/web-console/frontend/views/templates/message.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/pagination.jade b/modules/web-console/frontend/views/templates/pagination.jade
deleted file mode 100644
index 08ced60f3500a..0000000000000
--- a/modules/web-console/frontend/views/templates/pagination.jade
+++ /dev/null
@@ -1,32 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-nav(ng-if='numPages && pages.length >= 2')
-    ul.pagination
-        li(ng-if='currentPage > 1')
-            a(href='javascript:void(0);' ng-click='selectPage(1)' bs-tooltip='' data-title='First page' data-placement='bottom')
-                i.fa.fa-angle-double-left
-        li(ng-if='currentPage > 1')
-            a(href='javascript:void(0);' ng-click='selectPage(currentPage-1)' bs-tooltip='' data-title='Previous page' data-placement='bottom')
-                i.fa.fa-angle-left
-        li(ng-repeat='page in pages' ng-class='{active: page==currentPage}')
-            a(href='javascript:void(0);' ng-click='selectPage(page)') {{page}}
-        li(ng-if='currentPage < numPages')
-            a(href='javascript:void(0);' ng-click='selectPage(currentPage+1)' bs-tooltip='' data-title='Next page' data-placement='bottom')
-                i.fa.fa-angle-right
-        li(ng-if='currentPage < numPages')
-            a(href='javascript:void(0);' ng-click='selectPage(numPages)' bs-tooltip='' data-title='Last page' data-placement='bottom')
-                i.fa.fa-angle-double-right
\ No newline at end of file
diff --git a/modules/web-console/frontend/views/templates/select.jade b/modules/web-console/frontend/views/templates/select.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/select.jade
rename to modules/web-console/frontend/views/templates/select.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/validation-error.jade b/modules/web-console/frontend/views/templates/validation-error.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/validation-error.jade
rename to modules/web-console/frontend/views/templates/validation-error.tpl.pug

From 1c05eb30797ab9862c87e9d11237e7bcfb9782a0 Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgain.com>
Date: Thu, 9 Mar 2017 10:00:34 +0700
Subject: [PATCH 187/446] IGNITE_4659 Minor templates fixes. (cherry picked
 from commit 2d385c5)

---
 .../app/modules/states/configuration/clusters/binary.pug        | 2 +-
 .../app/modules/states/configuration/clusters/communication.pug | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
index ee9f37bd1645b..2e14e7fc29803 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
@@ -72,6 +72,6 @@ include /app/helpers/jade/mixins
                                 +checkbox('Enum', 'model.enum', 'enum', 'Flag indicating that this type is the enum')
 
                 .settings-row
-                    +checkbox('Compact footer', model + '.compactFooter', '"compactFooter"', 'When enabled, Ignite will not write fields metadata when serializing objects(this will increase serialization performance), because internally #[b BinaryMarshaller] already distribute metadata inside cluster')
+                    +checkbox('Compact footer', model + '.compactFooter', '"compactFooter"', 'When enabled, Ignite will not write fields metadata when serializing objects (this will increase serialization performance), because internally <b>BinaryMarshaller</b> already distribute metadata inside cluster')
             .col-sm-6
                 +preview-xml-java(model, 'clusterBinary')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
index cd93565720e2b..93bb5cea34d7c 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
@@ -53,7 +53,7 @@ include /app/helpers/jade/mixins
                 .settings-row
                     +number-min-max('Shared memory port:', `${communication}.sharedMemoryPort`, '"sharedMemoryPort"', 'true', '48100', '-1', '65535',
                         'Local port to accept shared memory connections<br/>\
-                        If set to #[b -1] shared memory communication will be disabled')
+                        If set to <b>-1</b> shared memory communication will be disabled')
                 .settings-row
                     +number('Idle connection timeout:', `${communication}.idleConnectionTimeout`, '"idleConnectionTimeout"', 'true', '30000', '1',
                         'Maximum idle connection timeout upon which a connection to client will be closed')

From 0c72d825418f785a2b0017d4730b5f6e55f2f757 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Wed, 15 Mar 2017 11:43:23 +0700
Subject: [PATCH 188/446] IGNITE-4821 Implemented enforce join order option on
 query tab. (cherry picked from commit 94c1e7c)

---
 modules/web-console/backend/app/agent.js      | 14 ++++++-
 modules/web-console/backend/app/browser.js    |  8 ++--
 .../app/modules/agent/agent.module.js         | 10 +++--
 .../app/modules/sql/sql.controller.js         | 25 +++++++++---
 .../frontend/public/stylesheets/style.scss    |  8 +++-
 .../frontend/views/sql/sql.tpl.pug            | 39 ++++++++++++-------
 6 files changed, 74 insertions(+), 30 deletions(-)

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index 4cae8ee124175..b09e10a9c16c7 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -227,17 +227,27 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo,
          * @param {String} cacheName Cache name.
          * @param {String} query Query.
          * @param {Boolean} nonCollocatedJoins Flag whether to execute non collocated joins.
+         * @param {Boolean} enforceJoinOrder Flag whether enforce join order is enabled.
          * @param {Boolean} local Flag whether to execute query locally.
          * @param {int} pageSize Page size.
          * @returns {Promise}
          */
-        fieldsQuery(demo, nid, cacheName, query, nonCollocatedJoins, local, pageSize) {
+        fieldsQuery(demo, nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, local, pageSize) {
             const cmd = new Command(demo, 'exe')
                 .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
                 .addParam('p1', nid)
                 .addParam('p2', 'org.apache.ignite.internal.visor.query.VisorQueryTask');
 
-            if (nonCollocatedJoins) {
+            if (enforceJoinOrder) {
+                cmd.addParam('p3', 'org.apache.ignite.internal.visor.query.VisorQueryArgV3')
+                    .addParam('p4', cacheName)
+                    .addParam('p5', query)
+                    .addParam('p6', nonCollocatedJoins)
+                    .addParam('p7', enforceJoinOrder)
+                    .addParam('p8', local)
+                    .addParam('p9', pageSize);
+            }
+            else if (nonCollocatedJoins) {
                 cmd.addParam('p3', 'org.apache.ignite.internal.visor.query.VisorQueryArgV2')
                     .addParam('p4', cacheName)
                     .addParam('p5', query)
diff --git a/modules/web-console/backend/app/browser.js b/modules/web-console/backend/app/browser.js
index 2b1285e4f79c1..d272c70800435 100644
--- a/modules/web-console/backend/app/browser.js
+++ b/modules/web-console/backend/app/browser.js
@@ -98,9 +98,9 @@ module.exports.factory = (_, socketio, agentMgr, configure) => {
                 });
 
                 // Execute query on node and return first page to browser.
-                socket.on('node:query', (nid, cacheName, query, distributedJoins, local, pageSize, cb) => {
+                socket.on('node:query', (nid, cacheName, query, distributedJoins, enforceJoinOrder, local, pageSize, cb) => {
                     agentMgr.findAgent(accountId())
-                        .then((agent) => agent.fieldsQuery(demo, nid, cacheName, query, distributedJoins, local, pageSize))
+                        .then((agent) => agent.fieldsQuery(demo, nid, cacheName, query, distributedJoins, enforceJoinOrder, local, pageSize))
                         .then((res) => cb(null, res))
                         .catch((err) => cb(_errorToJson(err)));
                 });
@@ -114,13 +114,13 @@ module.exports.factory = (_, socketio, agentMgr, configure) => {
                 });
 
                 // Execute query on node and return full result to browser.
-                socket.on('node:query:getAll', (nid, cacheName, query, distributedJoins, local, cb) => {
+                socket.on('node:query:getAll', (nid, cacheName, query, distributedJoins, enforceJoinOrder, local, cb) => {
                     // Set page size for query.
                     const pageSize = 1024;
 
                     agentMgr.findAgent(accountId())
                         .then((agent) => {
-                            const firstPage = agent.fieldsQuery(demo, nid, cacheName, query, distributedJoins, local, pageSize)
+                            const firstPage = agent.fieldsQuery(demo, nid, cacheName, query, distributedJoins, enforceJoinOrder, local, pageSize)
                                 .then(({result}) => {
                                     if (result.key)
                                         return Promise.reject(result.key);
diff --git a/modules/web-console/frontend/app/modules/agent/agent.module.js b/modules/web-console/frontend/app/modules/agent/agent.module.js
index c0e92d5d2db99..724fc6c5d0914 100644
--- a/modules/web-console/frontend/app/modules/agent/agent.module.js
+++ b/modules/web-console/frontend/app/modules/agent/agent.module.js
@@ -265,12 +265,13 @@ class IgniteAgentMonitor {
      * @param {String} cacheName Cache name.
      * @param {String} [query] Query if null then scan query.
      * @param {Boolean} nonCollocatedJoins Flag whether to execute non collocated joins.
+     * @param {Boolean} enforceJoinOrder Flag whether enforce join order is enabled.
      * @param {Boolean} local Flag whether to execute query locally.
      * @param {int} pageSize
      * @returns {Promise}
      */
-    query(nid, cacheName, query, nonCollocatedJoins, local, pageSize) {
-        return this._rest('node:query', nid, maskNull(cacheName), maskNull(query), nonCollocatedJoins, local, pageSize)
+    query(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, local, pageSize) {
+        return this._rest('node:query', nid, maskNull(cacheName), maskNull(query), nonCollocatedJoins, enforceJoinOrder, local, pageSize)
             .then(({result}) => {
                 if (_.isEmpty(result.key))
                     return result.value;
@@ -284,11 +285,12 @@ class IgniteAgentMonitor {
      * @param {String} cacheName Cache name.
      * @param {String} [query] Query if null then scan query.
      * @param {Boolean} nonCollocatedJoins Flag whether to execute non collocated joins.
+     * @param {Boolean} enforceJoinOrder Flag whether enforce join order is enabled.
      * @param {Boolean} local Flag whether to execute query locally.
      * @returns {Promise}
      */
-    queryGetAll(nid, cacheName, query, nonCollocatedJoins, local) {
-        return this._rest('node:query:getAll', nid, maskNull(cacheName), maskNull(query), nonCollocatedJoins, local);
+    queryGetAll(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, local) {
+        return this._rest('node:query:getAll', nid, maskNull(cacheName), maskNull(query), nonCollocatedJoins, enforceJoinOrder, local);
     }
 
     /**
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index e2ead13ec7c41..2fd089d14f7a9 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -35,6 +35,8 @@ const SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE = 'VISOR_SCAN_CACHE_WITH_FILTER_CASE
 
 const NON_COLLOCATED_JOINS_SINCE = '1.7.0';
 
+const ENFORCE_JOIN_VERS = [['1.7.9', '1.8.0'], ['1.8.4', '1.9.0'], ['1.9.1']];
+
 const _fullColName = (col) => {
     const res = [];
 
@@ -1337,7 +1339,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
             agentMonitor.awaitAgent()
                 .then(() => _closeOldQuery(paragraph))
                 .then(() => args.localNid || _chooseNode(args.cacheName, false))
-                .then((nid) => agentMonitor.query(nid, args.cacheName, args.query, args.nonCollocatedJoins, !!args.localNid, args.pageSize))
+                .then((nid) => agentMonitor.query(nid, args.cacheName, args.query, args.nonCollocatedJoins,
+                    args.enforceJoinOrder, !!args.localNid, args.pageSize))
                 .then(_processQueryResult.bind(this, paragraph, false))
                 .catch((err) => paragraph.errMsg = err.message);
         };
@@ -1370,8 +1373,18 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
             return false;
         };
 
+        $scope.enforceJoinOrderAvailable = (paragraph) => {
+            const cache = _.find($scope.caches, {name: paragraph.cacheName});
+
+            if (cache)
+                return !!_.find(cache.nodes, (node) => Version.includes(node.version, ...ENFORCE_JOIN_VERS));
+
+            return false;
+        };
+
         $scope.execute = (paragraph, local = false) => {
             const nonCollocatedJoins = !!paragraph.nonCollocatedJoins;
+            const enforceJoinOrder = !!paragraph.enforceJoinOrder;
 
             $scope.actionAvailable(paragraph, true) && _chooseNode(paragraph.cacheName, local)
                 .then((nid) => {
@@ -1391,6 +1404,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                                 pageSize: paragraph.pageSize,
                                 maxPages: paragraph.maxPages,
                                 nonCollocatedJoins,
+                                enforceJoinOrder,
                                 localNid: local ? nid : null
                             };
 
@@ -1398,7 +1412,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
                             ActivitiesData.post({ action: '/queries/execute' });
 
-                            return agentMonitor.query(nid, args.cacheName, qry, nonCollocatedJoins, local, args.pageSize);
+                            return agentMonitor.query(nid, args.cacheName, qry, nonCollocatedJoins, enforceJoinOrder, local, args.pageSize);
                         })
                         .then((res) => {
                             _processQueryResult(paragraph, true, res);
@@ -1451,7 +1465,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
                     ActivitiesData.post({ action: '/queries/explain' });
 
-                    return agentMonitor.query(nid, args.cacheName, args.query, false, false, args.pageSize);
+                    return agentMonitor.query(nid, args.cacheName, args.query, false, !!paragraph.enforceJoinOrder, false, args.pageSize);
                 })
                 .then(_processQueryResult.bind(this, paragraph, true))
                 .catch((err) => {
@@ -1489,7 +1503,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
                             ActivitiesData.post({ action: '/queries/scan' });
 
-                            return agentMonitor.query(nid, args.cacheName, query, false, local, args.pageSize);
+                            return agentMonitor.query(nid, args.cacheName, query, false, false, local, args.pageSize);
                         })
                         .then((res) => _processQueryResult(paragraph, true, res))
                         .catch((err) => {
@@ -1610,7 +1624,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
             const args = paragraph.queryArgs;
 
             return Promise.resolve(args.localNid || _chooseNode(args.cacheName, false))
-                .then((nid) => agentMonitor.queryGetAll(nid, args.cacheName, args.query, !!args.nonCollocatedJoins, !!args.localNid))
+                .then((nid) => agentMonitor.queryGetAll(nid, args.cacheName, args.query, !!args.nonCollocatedJoins,
+                    !!args.enforceJoinOrder, !!args.localNid))
                 .then((res) => _export(paragraph.name + '-all.csv', paragraph.gridOptions.columnDefs, res.columns, res.rows))
                 .catch(Messages.showError)
                 .then(() => paragraph.ace && paragraph.ace.focus());
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index ab7e3dd94286f..a07472e8840a4 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -809,7 +809,7 @@ button.form-control {
     }
 
     .btn-group {
-        vertical-align:top;
+        vertical-align: top;
         margin-left: 10px;
 
         i { line-height: 18px; }
@@ -1854,6 +1854,12 @@ th[st-sort] {
     display: inline-block;
 }
 
+.panel-top-align {
+    label {
+        vertical-align: top !important;
+    }
+}
+
 button.dropdown-toggle {
     margin-right: 5px;
 }
diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug
index 931d6327888d3..bc134a67e5bce 100644
--- a/modules/web-console/frontend/views/sql/sql.tpl.pug
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -91,20 +91,31 @@ mixin paragraph-rename
             input.form-control(id='paragraph-name-{{paragraph.id}}' ng-model='paragraph.editName' required ng-click='$event.stopPropagation();' ignite-on-enter='renameParagraph(paragraph, paragraph.editName)' ignite-on-escape='paragraph.edit = false')
 
 mixin query-settings
-    label.tipLabel(bs-tooltip data-placement='bottom' data-title='Configure periodical execution of last successfully executed query') Refresh rate:
-        button.btn.btn-default.fa.fa-clock-o.tipLabel(ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='{{ $ctrl.paragraphRateTemplateUrl }}' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
-
-    label.tipLabel(bs-tooltip data-placement='bottom' data-title='Max number of rows to show in query result as one page') Page size:
-        button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-select bs-options='item for item in pageSizes')
-
-    label.tipLabel(bs-tooltip data-placement='bottom' data-title='Limit query max results to specified number of pages') Max pages:
-        button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.maxPages' bs-select bs-options='item.value as item.label for item in maxPages')
-
-    label.tipLabel(ng-if='nonCollocatedJoinsAvailable(paragraph)' bs-tooltip data-placement='bottom' data-title='Non-collocated joins is a special mode that allow to join data across cluster without collocation.<br/>\
-        Nested joins are not supported for now.<br/>\
-        <b>NOTE</b>: In some cases it may consume more heap memory or may take a long time than collocated joins.' data-trigger='hover')
-        input(type='checkbox' ng-model='paragraph.nonCollocatedJoins')
-        span Allow non-collocated joins
+    .panel-top-align
+        label.tipLabel(bs-tooltip data-placement='bottom' data-title='Configure periodical execution of last successfully executed query') Refresh rate:
+            button.btn.btn-default.fa.fa-clock-o.tipLabel(ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='{{ $ctrl.paragraphRateTemplateUrl }}' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
+
+        label.tipLabel(bs-tooltip data-placement='bottom' data-title='Max number of rows to show in query result as one page') Page size:
+            button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-select bs-options='item for item in pageSizes')
+
+        label.tipLabel(bs-tooltip data-placement='bottom' data-title='Limit query max results to specified number of pages') Max pages:
+            button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.maxPages' bs-select bs-options='item.value as item.label for item in maxPages')
+
+        .panel-tip-container
+            .row(ng-if='nonCollocatedJoinsAvailable(paragraph)')
+                label.tipLabel(bs-tooltip data-placement='bottom' data-title='Non-collocated joins is a special mode that allow to join data across cluster without collocation.<br/>\
+                    Nested joins are not supported for now.<br/>\
+                    <b>NOTE</b>: In some cases it may consume more heap memory or may take a long time than collocated joins.' data-trigger='hover')
+                    input(type='checkbox' ng-model='paragraph.nonCollocatedJoins')
+                    span Allow non-collocated joins
+            .row(ng-if='enforceJoinOrderAvailable(paragraph)')
+                label.tipLabel(bs-tooltip data-placement='bottom' data-title='Enforce join order of tables in the query.<br/>\
+                    If <b>set</b> then query optimizer will not reorder tables in join.<br/>\
+                    <b>NOTE:</b> It is not recommended to enable this property until you are sure that\
+                    your indexes and the query itself are correct and tuned as much as possible but\
+                    query optimizer still produces wrong join order.' data-trigger='hover')
+                    input(type='checkbox' ng-model='paragraph.enforceJoinOrder')
+                    span Enforce join order
 
 mixin query-actions
     button.btn.btn-primary(ng-disabled='!actionAvailable(paragraph, true)' ng-click='execute(paragraph)') Execute

From baf8a8ca01d7e2fc69b04a77a39f7ff1f8b35727 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 17 Feb 2017 18:02:48 +0700
Subject: [PATCH 189/446] IGNITE-4472 Minor fix. (cherry picked from commit
 7cb3e68)

---
 .../backend/services/activities.js            |  7 ++---
 .../list-of-registered-users.controller.js    | 14 ++++------
 .../configuration/configuration.module.js     |  6 ++--
 .../configuration/summary/summary.worker.js   | 28 +++++++++----------
 .../console/agent/handlers/RestListener.java  |  9 +++++-
 5 files changed, 33 insertions(+), 31 deletions(-)

diff --git a/modules/web-console/backend/services/activities.js b/modules/web-console/backend/services/activities.js
index 124c775dc7a86..afde8e72f3ad3 100644
--- a/modules/web-console/backend/services/activities.js
+++ b/modules/web-console/backend/services/activities.js
@@ -37,10 +37,10 @@ module.exports.factory = (_, mongo) => {
          * @param {String} owner - User ID
          * @param {String} action - Action string presentation.
          * @param {String} group - Action group string presentation.
-         * @param {Date} [date] - Optional date to save in activity.
+         * @param {Date} [now] - Optional date to save in activity.
          * @returns {Promise.<mongo.ObjectId>} that resolve activity
          */
-        static merge(owner, {action, group}, date = new Date()) {
+        static merge(owner, {action, group}, now = new Date()) {
             mongo.Account.findById(owner)
                 .then((user) => {
                     user.lastActivity = new Date();
@@ -48,8 +48,7 @@ module.exports.factory = (_, mongo) => {
                     return user.save();
                 });
 
-            date.setDate(1);
-            date.setHours(0, 0, 0, 0);
+            const date = Date.UTC(now.getFullYear(), now.getMonth(), 1);
 
             return mongo.Activities.findOne({owner, action, date}).exec()
                 .then((activity) => {
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 33fe819068c78..54971b142b014 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -44,9 +44,6 @@ export default class IgniteListOfRegisteredUsersCtrl {
             startDate: new Date()
         };
 
-        $ctrl.params.startDate.setDate(1);
-        $ctrl.params.startDate.setHours(0, 0, 0, 0);
-
         const columnCompany = _.find(columnDefs, { displayName: 'Company' });
         const columnCountry = _.find(columnDefs, { displayName: 'Country' });
 
@@ -151,7 +148,7 @@ export default class IgniteListOfRegisteredUsersCtrl {
         };
 
         /**
-         * @param {{startDate: Date, endDate: Date}} params
+         * @param {{startDate: number, endDate: number}} params
          */
         const reloadUsers = (params) => {
             AdminData.loadUsers(params)
@@ -177,11 +174,10 @@ export default class IgniteListOfRegisteredUsersCtrl {
         $scope.$watch(() => $ctrl.params.startDate, (dt) => {
             $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(dt, 'yyyy_MM')}.csv`;
 
-            const endDate = new Date(dt);
-
-            endDate.setMonth(endDate.getMonth() + 1);
+            const startDate = Date.UTC(dt.getFullYear(), dt.getMonth(), 1);
+            const endDate = Date.UTC(dt.getFullYear(), dt.getMonth() + 1, 1);
 
-            reloadUsers({startDate: dtFilter(dt, 'yyyy-MM-dd'), endDate: dtFilter(endDate, 'yyyy-MM-dd')});
+            reloadUsers({ startDate, endDate });
         });
     }
 
@@ -237,6 +233,6 @@ export default class IgniteListOfRegisteredUsersCtrl {
     }
 
     exportCsv() {
-        this.gridApi.exporter.csvExport('all', 'visible');
+        this.gridApi.exporter.csvExport('visible', 'visible');
     }
 }
diff --git a/modules/web-console/frontend/app/modules/configuration/configuration.module.js b/modules/web-console/frontend/app/modules/configuration/configuration.module.js
index 4288ff79ea182..51306c6d8f8c5 100644
--- a/modules/web-console/frontend/app/modules/configuration/configuration.module.js
+++ b/modules/web-console/frontend/app/modules/configuration/configuration.module.js
@@ -49,10 +49,10 @@ angular
 ])
 .provider(...igniteSidebar)
 .directive(...igniteSidebarDirective)
-.service('IgniteConfigurationGenerator', IgniteConfigurationGenerator)
+.service('IgniteConfigurationGenerator', () => IgniteConfigurationGenerator)
 .service('IgnitePlatformGenerator', IgnitePlatformGenerator)
-.service('SpringTransformer', IgniteSpringTransformer)
-.service('JavaTransformer', IgniteJavaTransformer)
+.service('SpringTransformer', () => IgniteSpringTransformer)
+.service('JavaTransformer', () => IgniteJavaTransformer)
 .service('IgniteSharpTransformer', SharpTransformer)
 .service('IgniteVersion', IgniteVersion)
 .service('IgniteEventGroups', IgniteEventGroups)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
index 070b6ce87aa69..7ea1d5ad5beb0 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
@@ -17,24 +17,24 @@
 
 import JSZip from 'jszip';
 
-import MavenGenerator from 'app/modules/configuration/generator/Maven.service';
-import DockerGenerator from 'app/modules/configuration/generator/Docker.service';
-import ReadmeGenerator from 'app/modules/configuration/generator/Readme.service';
-import PropertiesGenerator from 'app/modules/configuration/generator/Properties.service';
-import ConfigurationGenerator from 'app/modules/configuration/generator/ConfigurationGenerator';
+import IgniteMavenGenerator from 'app/modules/configuration/generator/Maven.service';
+import IgniteDockerGenerator from 'app/modules/configuration/generator/Docker.service';
+import IgniteReadmeGenerator from 'app/modules/configuration/generator/Readme.service';
+import IgnitePropertiesGenerator from 'app/modules/configuration/generator/Properties.service';
+import IgniteConfigurationGenerator from 'app/modules/configuration/generator/ConfigurationGenerator';
 
-import JavaTransformer from 'app/modules/configuration/generator/JavaTransformer.service';
-import SpringTransformer from 'app/modules/configuration/generator/SpringTransformer.service';
+import IgniteJavaTransformer from 'app/modules/configuration/generator/JavaTransformer.service';
+import IgniteSpringTransformer from 'app/modules/configuration/generator/SpringTransformer.service';
 
-const maven = new MavenGenerator();
-const docker = new DockerGenerator();
-const readme = new ReadmeGenerator();
-const properties = new PropertiesGenerator();
+const maven = new IgniteMavenGenerator();
+const docker = new IgniteDockerGenerator();
+const readme = new IgniteReadmeGenerator();
+const properties = new IgnitePropertiesGenerator();
 
-const java = new JavaTransformer[0]();
-const spring = new SpringTransformer[0]();
+const java = IgniteJavaTransformer;
+const spring = IgniteSpringTransformer;
 
-const generator = new ConfigurationGenerator[0]();
+const generator = IgniteConfigurationGenerator;
 
 const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
 
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
index 1e86549e25927..fcacc88182bcc 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
@@ -38,6 +38,7 @@
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.ignite.console.agent.AgentConfiguration;
 import org.apache.ignite.console.demo.AgentClusterDemo;
 import org.apache.log4j.Logger;
@@ -65,7 +66,13 @@ public RestListener(AgentConfiguration cfg) {
 
         this.cfg = cfg;
 
-        httpClient = HttpClientBuilder.create().build();
+        // Create a connection manager with custom configuration.
+        PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager();
+
+        connMgr.setDefaultMaxPerRoute(Integer.MAX_VALUE);
+        connMgr.setMaxTotal(Integer.MAX_VALUE);
+
+        httpClient = HttpClientBuilder.create().setConnectionManager(connMgr).build();
     }
 
     /** {@inheritDoc} */

From 4ba2d12c1a797bc9cc5ce585354adfe3755a3ddb Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 17 Mar 2017 11:08:39 +0700
Subject: [PATCH 190/446] IGNITE-4666 Clone to Input service. (cherry picked
 from commit 83579ce)

---
 modules/web-console/frontend/app/app.js       | 10 +--
 .../app/components/input-dialog/index.js      | 24 +++++
 .../input-dialog/input-dialog.controller.js   | 35 ++++++++
 .../input-dialog/input-dialog.service.js      | 88 +++++++++++++++++++
 .../input-dialog/input-dialog.tpl.pug}        | 10 +--
 .../frontend/app/services/Clone.service.js    | 66 --------------
 .../frontend/controllers/caches-controller.js | 10 +--
 .../controllers/clusters-controller.js        |  6 +-
 .../controllers/domains-controller.js         | 12 ++-
 .../frontend/controllers/igfs-controller.js   | 10 +--
 10 files changed, 173 insertions(+), 98 deletions(-)
 create mode 100644 modules/web-console/frontend/app/components/input-dialog/index.js
 create mode 100644 modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js
 create mode 100644 modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js
 rename modules/web-console/frontend/{views/templates/clone.tpl.pug => app/components/input-dialog/input-dialog.tpl.pug} (83%)
 delete mode 100644 modules/web-console/frontend/app/services/Clone.service.js

diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 5e3bb07434b2c..1e21d247c060a 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -16,9 +16,9 @@
  */
 
 import '../public/stylesheets/style.scss';
-import '../app/components/ui-grid-header/ui-grid-header.scss';
-import '../app/components/ui-grid-settings/ui-grid-settings.scss';
-import '../app/components/form-field-datepicker/form-field-datepicker.scss';
+import './components/ui-grid-header/ui-grid-header.scss';
+import './components/ui-grid-settings/ui-grid-settings.scss';
+import './components/form-field-datepicker/form-field-datepicker.scss';
 
 import './app.config';
 
@@ -80,7 +80,6 @@ import igniteRetainSelection from './directives/retain-selection.directive';
 
 // Services.
 import ChartColors from './services/ChartColors.service';
-import Clone from './services/Clone.service.js';
 import Confirm from './services/Confirm.service.js';
 import ConfirmBatch from './services/ConfirmBatch.service.js';
 import CopyToClipboard from './services/CopyToClipboard.service';
@@ -117,6 +116,7 @@ import resetPassword from './controllers/reset-password.controller';
 // Components
 import igniteListOfRegisteredUsers from './components/list-of-registered-users';
 import IgniteActivitiesUserDialog from './components/activities-user-dialog';
+import './components/input-dialog';
 
 // Inject external modules.
 import 'ignite_modules_temp/index';
@@ -149,6 +149,7 @@ angular
     'ignite-console.core',
     'ignite-console.ace',
     'ignite-console.Form',
+    'ignite-console.input-dialog',
     'ignite-console.user',
     'ignite-console.branding',
     'ignite-console.socket',
@@ -203,7 +204,6 @@ angular
 .service('JavaTypes', JavaTypes)
 .service('SqlTypes', SqlTypes)
 .service(...ChartColors)
-.service(...Clone)
 .service(...Confirm)
 .service(...ConfirmBatch)
 .service(...CopyToClipboard)
diff --git a/modules/web-console/frontend/app/components/input-dialog/index.js b/modules/web-console/frontend/app/components/input-dialog/index.js
new file mode 100644
index 0000000000000..4bb96422160b4
--- /dev/null
+++ b/modules/web-console/frontend/app/components/input-dialog/index.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import inputDialog from './input-dialog.service';
+
+angular
+    .module('ignite-console.input-dialog', [])
+    .service('IgniteInput', inputDialog);
diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js b/modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js
new file mode 100644
index 0000000000000..3f6e97b96c9a6
--- /dev/null
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class InputDialogController {
+    static $inject = ['deferred', 'ui'];
+
+    constructor(deferred, {title, label, value, toValidValue}) {
+        this.deferred = deferred;
+        this.title = title;
+        this.label = label;
+        this.value = value;
+        this.toValidValue = toValidValue;
+    }
+
+    confirm() {
+        if (_.isFunction(this.toValidValue))
+            return this.deferred.resolve(this.toValidValue(this.value));
+
+        this.deferred.resolve(this.value);
+    }
+}
diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js
new file mode 100644
index 0000000000000..fc3cb851c720f
--- /dev/null
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import controller from './input-dialog.controller';
+import templateUrl from './input-dialog.tpl.pug';
+
+export default class InputDialog {
+    static $inject = ['$modal', '$q'];
+
+    constructor($modal, $q) {
+        this.$modal = $modal;
+        this.$q = $q;
+    }
+
+    /**
+     * Open input dialog to configure custom value.
+     *
+     * @param {String} title Dialog title.
+     * @param {String} label Input field label.
+     * @param {String} value Default value.
+     * @param {Function} [toValidValue] Validator function.
+     * @returns {Promise.<String>} User input.
+     */
+    input(title, label, value, toValidValue) {
+        const deferred = this.$q.defer();
+
+        const modal = this.$modal({
+            templateUrl,
+            resolve: {
+                deferred: () => deferred,
+                ui: () => ({
+                    title,
+                    label,
+                    value,
+                    toValidValue
+                })
+            },
+            placement: 'center',
+            controller,
+            controllerAs: 'ctrl'
+        });
+
+        const modalHide = modal.hide;
+
+        modal.hide = () => deferred.reject('cancelled');
+
+        return deferred.promise
+            .finally(modalHide);
+    }
+
+    /**
+     * Open input dialog to configure cloned object name.
+     *
+     * @param {String} srcName Name of source object.
+     * @param {Array.<String>} names List of already exist names.
+     * @returns {Promise.<String>} New name
+     */
+    clone(srcName, names) {
+        const uniqueName = (value) => {
+            let num = 1;
+            let tmpName = value;
+
+            while (_.includes(names, tmpName)) {
+                tmpName = `${value}_${num}`;
+
+                num++;
+            }
+
+            return tmpName;
+        };
+
+        return this.input('Clone', 'New name', uniqueName(srcName), uniqueName);
+    }
+}
diff --git a/modules/web-console/frontend/views/templates/clone.tpl.pug b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
similarity index 83%
rename from modules/web-console/frontend/views/templates/clone.tpl.pug
rename to modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
index e62cb18a8fb60..95549d7995e21 100644
--- a/modules/web-console/frontend/views/templates/clone.tpl.pug
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
@@ -23,17 +23,17 @@ include /app/helpers/jade/mixins
                 button.close(ng-click='$hide()') &times;
                 h4.modal-title 
                     i.fa.fa-clone
-                    | Clone
+                    | {{ ctrl.title }}
             form.form-horizontal.modal-body.row(name='ui.inputForm' novalidate)
                 div
                     .col-sm-2
-                        label.required.labelFormField New name:&nbsp;
+                        label.required.labelFormField {{ ctrl.label }}:&nbsp;
                     .col-sm-10
                         .input-tip
-                            +ignite-form-field-input('"copy-new-name"','newName', false, 'true', 'Enter new name')(
+                            +ignite-form-field-input('"input-field"', 'ctrl.value', false, 'true', 'Enter value')(
                                 data-ignite-form-field-input-autofocus='true'
-                                ignite-on-enter='form.$valid && ok(newName)'
+                                ignite-on-enter='form.$valid && ctrl.confirm()'
                             )
             .modal-footer
                 button.btn.btn-default(id='copy-btn-cancel' ng-click='$hide()') Cancel
-                button.btn.btn-primary(id='copy-btn-confirm' ng-disabled='ui.inputForm.$invalid' ng-click='ok(newName)') Confirm
+                button.btn.btn-primary(id='copy-btn-confirm' ng-disabled='ui.inputForm.$invalid' ng-click='ctrl.confirm()') Confirm
diff --git a/modules/web-console/frontend/app/services/Clone.service.js b/modules/web-console/frontend/app/services/Clone.service.js
deleted file mode 100644
index d079141364b55..0000000000000
--- a/modules/web-console/frontend/app/services/Clone.service.js
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import templateUrl from 'views/templates/clone.tpl.pug';
-
-// Service for clone objects.
-export default ['IgniteClone', ['$rootScope', '$q', '$modal', ($root, $q, $modal) => {
-    const scope = $root.$new();
-
-    let _names = [];
-    let deferred;
-    let _validator;
-
-    function _nextAvailableName(name) {
-        let num = 1;
-        let tmpName = name;
-
-        while (_.includes(_names, tmpName)) {
-            tmpName = name + '_' + num.toString();
-
-            num++;
-        }
-
-        return tmpName;
-    }
-
-    const cloneModal = $modal({templateUrl, scope, placement: 'center', show: false});
-
-    scope.ok = function(newName) {
-        if (!_validator || _validator(newName)) {
-            deferred.resolve(_nextAvailableName(newName));
-
-            cloneModal.hide();
-        }
-    };
-
-    cloneModal.confirm = function(oldName, names, validator) {
-        _names = names;
-
-        scope.newName = _nextAvailableName(oldName);
-
-        _validator = validator;
-
-        deferred = $q.defer();
-
-        cloneModal.$promise.then(cloneModal.show);
-
-        return deferred.promise;
-    };
-
-    return cloneModal;
-}]];
diff --git a/modules/web-console/frontend/controllers/caches-controller.js b/modules/web-console/frontend/controllers/caches-controller.js
index b50fde3188b1a..d4a13e56c74a2 100644
--- a/modules/web-console/frontend/controllers/caches-controller.js
+++ b/modules/web-console/frontend/controllers/caches-controller.js
@@ -19,8 +19,8 @@ import infoMessageTemplateUrl from 'views/templates/message.tpl.pug';
 
 // Controller for Caches screen.
 export default ['cachesController', [
-    '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'IgniteLegacyTable',
-    function($scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Confirm, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, Resource, ErrorPopover, FormUtils, LegacyTable) {
+    '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteInput', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'IgniteLegacyTable',
+    function($scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Confirm, Input, Loading, ModelNormalizer, UnsavedChangesGuard, Resource, ErrorPopover, FormUtils, LegacyTable) {
         UnsavedChangesGuard.install($scope);
 
         const emptyCache = {empty: true};
@@ -517,15 +517,13 @@ export default ['cachesController', [
         };
 
         function _cacheNames() {
-            return _.map($scope.caches, function(cache) {
-                return cache.name;
-            });
+            return _.map($scope.caches, (cache) => cache.name);
         }
 
         // Clone cache with new name.
         $scope.cloneItem = function() {
             if (validate($scope.backupItem)) {
-                Clone.confirm($scope.backupItem.name, _cacheNames()).then(function(newName) {
+                Input.clone($scope.backupItem.name, _cacheNames()).then((newName) => {
                     const item = angular.copy($scope.backupItem);
 
                     delete item._id;
diff --git a/modules/web-console/frontend/controllers/clusters-controller.js b/modules/web-console/frontend/controllers/clusters-controller.js
index 7f90b90e198fb..c8392cf7ef0aa 100644
--- a/modules/web-console/frontend/controllers/clusters-controller.js
+++ b/modules/web-console/frontend/controllers/clusters-controller.js
@@ -17,8 +17,8 @@
 
 // Controller for Clusters screen.
 export default ['clustersController', [
-    '$rootScope', '$scope', '$http', '$state', '$timeout', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteEventGroups', 'DemoInfo', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils',
-    function($root, $scope, $http, $state, $timeout, LegacyUtils, Messages, Confirm, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, igniteEventGroups, DemoInfo, LegacyTable, Resource, ErrorPopover, FormUtils) {
+    '$rootScope', '$scope', '$http', '$state', '$timeout', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteInput', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteEventGroups', 'DemoInfo', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils',
+    function($root, $scope, $http, $state, $timeout, LegacyUtils, Messages, Confirm, Input, Loading, ModelNormalizer, UnsavedChangesGuard, igniteEventGroups, DemoInfo, LegacyTable, Resource, ErrorPopover, FormUtils) {
         UnsavedChangesGuard.install($scope);
 
         const emptyCluster = {empty: true};
@@ -774,7 +774,7 @@ export default ['clustersController', [
         // Clone cluster with new name.
         $scope.cloneItem = function() {
             if (validate($scope.backupItem)) {
-                Clone.confirm($scope.backupItem.name, _clusterNames()).then(function(newName) {
+                Input.clone($scope.backupItem.name, _clusterNames()).then((newName) => {
                     const item = angular.copy($scope.backupItem);
 
                     delete item._id;
diff --git a/modules/web-console/frontend/controllers/domains-controller.js b/modules/web-console/frontend/controllers/domains-controller.js
index 5c9e511a139ca..806dd4577ead9 100644
--- a/modules/web-console/frontend/controllers/domains-controller.js
+++ b/modules/web-console/frontend/controllers/domains-controller.js
@@ -19,8 +19,8 @@ import templateUrl from 'views/configuration/domains-import.tpl.pug';
 
 // Controller for Domain model screen.
 export default ['domainsController', [
-    '$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteAgentMonitor', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes', 'IgniteActivitiesData',
-    function($root, $scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Focus, Confirm, ConfirmBatch, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, IgniteAgentMonitor, LegacyTable, Resource, ErrorPopover, FormUtils, JavaTypes, SqlTypes, ActivitiesData) {
+    '$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteInput', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteAgentMonitor', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes', 'IgniteActivitiesData',
+    function($root, $scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Focus, Confirm, ConfirmBatch, Input, Loading, ModelNormalizer, UnsavedChangesGuard, IgniteAgentMonitor, LegacyTable, Resource, ErrorPopover, FormUtils, JavaTypes, SqlTypes, ActivitiesData) {
         UnsavedChangesGuard.install($scope);
 
         const emptyDomain = {empty: true};
@@ -1429,7 +1429,7 @@ export default ['domainsController', [
 
                 item.cacheStoreChanges = [];
 
-                _.forEach(item.caches, function(cacheId) {
+                _.forEach(item.caches, (cacheId) => {
                     const cache = _.find($scope.caches, {value: cacheId}).cache;
 
                     const change = LegacyUtils.autoCacheStoreConfiguration(cache, [item]);
@@ -1444,9 +1444,7 @@ export default ['domainsController', [
         };
 
         function _domainNames() {
-            return _.map($scope.domains, function(domain) {
-                return domain.valueType;
-            });
+            return _.map($scope.domains, (domain) => domain.valueType);
         }
 
         function _newNameIsValidJavaClass(newName) {
@@ -1457,7 +1455,7 @@ export default ['domainsController', [
         // Save domain model with new name.
         $scope.cloneItem = function() {
             if ($scope.tableReset(true) && validate($scope.backupItem)) {
-                Clone.confirm($scope.backupItem.valueType, _domainNames(), _newNameIsValidJavaClass).then(function(newName) {
+                Input.clone($scope.backupItem.valueType, _domainNames(), _newNameIsValidJavaClass).then((newName) => {
                     const item = angular.copy($scope.backupItem);
 
                     delete item._id;
diff --git a/modules/web-console/frontend/controllers/igfs-controller.js b/modules/web-console/frontend/controllers/igfs-controller.js
index b3c6043ce32b6..504e28d96e417 100644
--- a/modules/web-console/frontend/controllers/igfs-controller.js
+++ b/modules/web-console/frontend/controllers/igfs-controller.js
@@ -17,8 +17,8 @@
 
 // Controller for IGFS screen.
 export default ['igfsController', [
-    '$scope', '$http', '$state', '$filter', '$timeout', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils',
-    function($scope, $http, $state, $filter, $timeout, LegacyUtils, Messages, Confirm, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, LegacyTable, Resource, ErrorPopover, FormUtils) {
+    '$scope', '$http', '$state', '$filter', '$timeout', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteInput', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils',
+    function($scope, $http, $state, $filter, $timeout, LegacyUtils, Messages, Confirm, Input, Loading, ModelNormalizer, UnsavedChangesGuard, LegacyTable, Resource, ErrorPopover, FormUtils) {
         UnsavedChangesGuard.install($scope);
 
         const emptyIgfs = {empty: true};
@@ -328,15 +328,13 @@ export default ['igfsController', [
         };
 
         function _igfsNames() {
-            return _.map($scope.igfss, function(igfs) {
-                return igfs.name;
-            });
+            return _.map($scope.igfss, (igfs) => igfs.name);
         }
 
         // Clone IGFS with new name.
         $scope.cloneItem = function() {
             if ($scope.tableReset(true) && validate($scope.backupItem)) {
-                Clone.confirm($scope.backupItem.name, _igfsNames()).then(function(newName) {
+                Input.clone($scope.backupItem.name, _igfsNames()).then((newName) => {
                     const item = angular.copy($scope.backupItem);
 
                     delete item._id;

From f1aa86f43776840ff14d5c709f3599da2e7b39e5 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Tue, 14 Mar 2017 15:54:14 +0700
Subject: [PATCH 191/446] Web Console: Cleanup (cherry picked from commit
 3da7794)

---
 .../ui-grid-settings/ui-grid-settings.scss    | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
index 4beb2a1baa40a..24f4d9b7efb7f 100644
--- a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
@@ -70,6 +70,41 @@
         }
     }
 
+    &-number-filter {
+        float: right;
+
+        .ignite-form-field {
+            width: 180px;
+            margin-right: 0;
+
+            &__label {
+            }
+
+            &__control {
+            }
+
+            &:nth-child(1) {
+                float: left;
+
+                .ignite-form-field__label {
+                    margin-right: 0;
+                    width: 70%;
+                    max-width: 100%;
+                }
+
+                .ignite-form-field__control {
+                    width: 30%;
+                }
+            }
+        }
+
+        button {
+            width: auto;
+            display: inline-block;
+            margin-left: 5px;
+        }
+    }
+
     &-dateperiod {
         float: right;
 

From d0c0042d3fbdade16e719cfb91acebae3f10bf04 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 2 Mar 2017 16:36:43 +0700
Subject: [PATCH 192/446] Minor fixes. (cherry picked from commit 684dc7b)

---
 modules/web-console/backend/app/agent.js      | 36 +++++++++++++++++++
 modules/web-console/backend/app/browser.js    | 28 +++++++++++++++
 .../app/modules/sql/sql.controller.js         |  1 -
 .../demo/service/DemoCachesLoadService.java   | 26 ++++++++++++++
 4 files changed, 90 insertions(+), 1 deletion(-)

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index b09e10a9c16c7..f65eabb29bee7 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -338,6 +338,42 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo,
             return this.executeRest(cmd);
         }
 
+        /**
+         * Collect running queries
+         * @param {Boolean} demo Is need run command on demo node.
+         * @param {Number} duration minimum duration time of running queries.
+         * @returns {Promise}
+         */
+        queryCollectRunning(demo, duration) {
+            const cmd = new Command(demo, 'exe')
+                .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
+                .addParam('p1', '')
+                .addParam('p2', 'org.apache.ignite.internal.visor.query.VisorCollectRunningQueriesTask')
+                .addParam('p3', 'java.lang.Long')
+                .addParam('p4', duration);
+
+            return this.executeRest(cmd);
+        }
+
+        /**
+         * Cancel running query.
+         * @param {Boolean} demo Is need run command on demo node.
+         * @param {String} nid Node id.
+         * @param {Number} queryId query id to cancel.
+         * @returns {Promise}
+         */
+        queryCancel(demo, nid, queryId) {
+            const cmd = new Command(demo, 'exe')
+                .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
+                .addParam('p1', nid)
+                .addParam('p2', 'org.apache.ignite.internal.visor.query.VisorCancelQueriesTask')
+                .addParam('p3', 'java.util.Collection')
+                .addParam('p4', 'java.lang.Long')
+                .addParam('p5', queryId);
+
+            return this.executeRest(cmd);
+        }
+
         /**
          * @param {Boolean} demo Is need run command on demo node.
          * @param {String} cacheName Cache name.
diff --git a/modules/web-console/backend/app/browser.js b/modules/web-console/backend/app/browser.js
index d272c70800435..00ae751f93dae 100644
--- a/modules/web-console/backend/app/browser.js
+++ b/modules/web-console/backend/app/browser.js
@@ -175,6 +175,34 @@ module.exports.factory = (_, socketio, agentMgr, configure) => {
                         .catch((err) => cb(_errorToJson(err)));
                 });
 
+                // Collect running queries from all nodes in grid.
+                socket.on('node:query:running', (duration, cb) => {
+                    agentMgr.findAgent(accountId())
+                        .then((agent) => agent.queryCollectRunning(demo, duration))
+                        .then((data) => {
+
+                            if (data.finished)
+                                return cb(null, data.result);
+
+                            cb(_errorToJson(data.error));
+                        })
+                        .catch((err) => cb(_errorToJson(err)));
+                });
+
+                // Cancel running query by query id on node.
+                socket.on('node:query:cancel', (nid, queryId, cb) => {
+                    agentMgr.findAgent(accountId())
+                        .then((agent) => agent.queryCancel(demo, nid, queryId))
+                        .then((data) => {
+
+                            if (data.finished)
+                                return cb(null, data.result);
+
+                            cb(_errorToJson(data.error));
+                        })
+                        .catch((err) => cb(_errorToJson(err)));
+                });
+
                 // Return cache metadata from all nodes in grid.
                 socket.on('node:cache:metadata', (cacheName, cb) => {
                     agentMgr.findAgent(accountId())
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 2fd089d14f7a9..7ded2d5d99b6f 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -20,7 +20,6 @@ import cacheMetadataTemplateUrl from 'views/sql/cache-metadata.tpl.pug';
 import chartSettingsTemplateUrl from 'views/sql/chart-settings.tpl.pug';
 import showQueryTemplateUrl from 'views/templates/message.tpl.pug';
 
-
 // Time line X axis descriptor.
 const TIME_LINE = {value: -1, type: 'java.sql.Date', label: 'TIME_LINE'};
 
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
index fbfa2ae6e09e2..5f7823b418e1d 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.cache.QueryIndex;
 import org.apache.ignite.cache.QueryIndexType;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.console.demo.AgentDemoUtils;
 import org.apache.ignite.console.demo.model.Car;
@@ -209,6 +210,7 @@ private static <K, V> CacheConfiguration<K, V> cacheConfiguration(String name) {
         ccfg.setQueryDetailMetricsSize(10);
         ccfg.setStartSize(100);
         ccfg.setStatisticsEnabled(true);
+        ccfg.setSqlFunctionClasses(SQLFunctions.class);
 
         return ccfg;
     }
@@ -453,4 +455,28 @@ private void populateCacheCar() {
         if (ignite.log().isDebugEnabled())
             ignite.log().debug("DEMO: Finished cars population.");
     }
+
+    /**
+     * Utility class with custom SQL functions.
+     */
+    public static class SQLFunctions {
+        /**
+         * Sleep function to simulate long running queries.
+         *
+         * @param x Time to sleep.
+         * @return Return specified argument.
+         */
+        @QuerySqlFunction
+        public static long sleep(long x) {
+            if (x >= 0)
+                try {
+                    Thread.sleep(x);
+                }
+                catch (InterruptedException ignored) {
+                    // No-op.
+                }
+
+            return x;
+        }
+    }
 }

From d939944f84aa415099a945b720dd4cd65132f6af Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 21 Mar 2017 10:05:33 +0700
Subject: [PATCH 193/446] IGNITE-1.8.4 Manual merge with master.

---
 .../frontend/gulpfile.babel.js/tasks/jade.js  | 49 -----------------
 .../webpack/environments/test.js              | 52 ++++++++-----------
 2 files changed, 22 insertions(+), 79 deletions(-)
 delete mode 100644 modules/web-console/frontend/gulpfile.babel.js/tasks/jade.js

diff --git a/modules/web-console/frontend/gulpfile.babel.js/tasks/jade.js b/modules/web-console/frontend/gulpfile.babel.js/tasks/jade.js
deleted file mode 100644
index 49359764bdeac..0000000000000
--- a/modules/web-console/frontend/gulpfile.babel.js/tasks/jade.js
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import gulp from 'gulp';
-import ll from 'gulp-ll';
-import jade from 'gulp-jade';
-
-import { jadeViewsPaths, jadeAppModulePaths, jadeModulePaths, destDir, rootDir } from '../paths';
-
-const jadeOptions = {
-    basedir: rootDir,
-    cache: true
-};
-
-ll.tasks(['jade:views', 'jade:app', 'jade:ignite_modules']);
-
-gulp.task('jade', ['jade:views', 'jade:app', 'jade:ignite_modules']);
-
-gulp.task('jade:views', () =>
-    gulp.src(jadeViewsPaths)
-        .pipe(jade(jadeOptions))
-        .pipe(gulp.dest(destDir))
-);
-
-gulp.task('jade:app', () =>
-    gulp.src(jadeAppModulePaths)
-        .pipe(jade(jadeOptions))
-        .pipe(gulp.dest(destDir))
-);
-
-gulp.task('jade:ignite_modules', () =>
-    gulp.src(jadeModulePaths)
-        .pipe(jade(jadeOptions))
-        .pipe(gulp.dest(`${destDir}/ignite_modules`))
-);
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/test.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/test.js
index 1c37196838ae4..0cb9b77980840 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/test.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/test.js
@@ -19,34 +19,26 @@ import webpack from 'webpack';
 
 const NODE_ENV = process.env.NODE_ENV || 'production';
 
-export default () => {
-
-    return {
-        cache: true,
-        node: {
-            fs: 'empty'
-        },
-
-        module: {
-            preLoaders: null
-        },
-
-        // Entry points.
-        entry: null,
-
-        // Output system.
-        output: null,
-        eslint: null,
-
-        // Load plugins.
-        plugins: [
-            new webpack.ProvidePlugin({
-                $: 'jquery',
-                jQuery: 'jquery',
-                _: 'lodash',
-                nv: 'nvd3'
-            }),
-            new webpack.DefinePlugin({NODE_ENV: JSON.stringify(NODE_ENV)})
-        ]
-    };
+export default {
+    cache: true,
+    node: {
+        fs: 'empty'
+    },
+
+    // Entry points.
+    entry: null,
+
+    // Output system.
+    output: null,
+
+    // Load plugins.
+    plugins: [
+        new webpack.ProvidePlugin({
+            $: 'jquery',
+            jQuery: 'jquery',
+            _: 'lodash',
+            nv: 'nvd3'
+        }),
+        new webpack.DefinePlugin({NODE_ENV: JSON.stringify(NODE_ENV)})
+    ]
 };

From 71c62f70df2b79f706b6cca157bccfbcea0ae5ef Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 21 Mar 2017 10:20:39 +0700
Subject: [PATCH 194/446] IGNITE-4830 Implemented better SQL errors handling.
 (cherry picked from commit fbb9940)

---
 .../app/modules/sql/sql.controller.js         | 84 ++++++++++++++-----
 .../frontend/views/sql/sql.tpl.pug            |  9 +-
 .../frontend/views/templates/message.tpl.pug  |  4 +-
 3 files changed, 71 insertions(+), 26 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 7ded2d5d99b6f..a3a5a5fb81dc7 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -18,7 +18,7 @@
 import paragraphRateTemplateUrl from 'views/sql/paragraph-rate.tpl.pug';
 import cacheMetadataTemplateUrl from 'views/sql/cache-metadata.tpl.pug';
 import chartSettingsTemplateUrl from 'views/sql/chart-settings.tpl.pug';
-import showQueryTemplateUrl from 'views/templates/message.tpl.pug';
+import messageTemplateUrl from 'views/templates/message.tpl.pug';
 
 // Time line X axis descriptor.
 const TIME_LINE = {value: -1, type: 'java.sql.Date', label: 'TIME_LINE'};
@@ -53,7 +53,7 @@ const _fullColName = (col) => {
 let paragraphId = 0;
 
 class Paragraph {
-    constructor($animate, $timeout, paragraph) {
+    constructor($animate, $timeout, JavaTypes, paragraph) {
         const self = this;
 
         self.id = 'paragraph-' + paragraphId++;
@@ -140,13 +140,36 @@ class Paragraph {
         }});
 
         Object.defineProperty(this, 'chartHistory', {value: []});
+
+        Object.defineProperty(this, 'error', {value: {
+            root: {},
+            message: ''
+        }});
+
+        this.setError = (err) => {
+            this.error.root = err;
+            this.error.message = err.message;
+
+            let cause = err;
+
+            while (_.nonNil(cause)) {
+                if (_.nonEmpty(cause.className) &&
+                    _.includes(['SQLException', 'JdbcSQLException'], JavaTypes.shortClassName(cause.className))) {
+                    this.error.message = cause.message;
+
+                    break;
+                }
+
+                cause = cause.cause;
+            }
+        };
     }
 
     resultType() {
         if (_.isNil(this.queryArgs))
             return null;
 
-        if (!_.isEmpty(this.errMsg))
+        if (_.nonEmpty(this.error.message))
             return 'error';
 
         if (_.isEmpty(this.rows))
@@ -172,7 +195,7 @@ class Paragraph {
     }
 
     queryExecuted() {
-        return !_.isEmpty(this.meta) || !_.isEmpty(this.errMsg);
+        return _.nonEmpty(this.meta) || _.nonEmpty(this.error.message);
     }
 
     scanExplain() {
@@ -184,17 +207,17 @@ class Paragraph {
     }
 
     chartColumnsConfigured() {
-        return !_.isEmpty(this.chartKeyCols) && !_.isEmpty(this.chartValCols);
+        return _.nonEmpty(this.chartKeyCols) && _.nonEmpty(this.chartValCols);
     }
 
     chartTimeLineEnabled() {
-        return !_.isEmpty(this.chartKeyCols) && _.eq(this.chartKeyCols[0], TIME_LINE);
+        return _.nonEmpty(this.chartKeyCols) && _.eq(this.chartKeyCols[0], TIME_LINE);
     }
 }
 
 // Controller for SQL notebook screen.
-export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData',
-    function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData) {
+export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData', 'JavaTypes',
+    function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes) {
         const $ctrl = this;
 
         // Define template urls.
@@ -909,7 +932,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                     $scope.notebook.paragraphs = [];
 
                 $scope.notebook.paragraphs = _.map($scope.notebook.paragraphs,
-                    (paragraph) => new Paragraph($animate, $timeout, paragraph));
+                    (paragraph) => new Paragraph($animate, $timeout, JavaTypes, paragraph));
 
                 if (_.isEmpty($scope.notebook.paragraphs))
                     $scope.addQuery();
@@ -981,7 +1004,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
             ActivitiesData.post({ action: '/queries/add/query' });
 
-            const paragraph = new Paragraph($animate, $timeout, {
+            const paragraph = new Paragraph($animate, $timeout, JavaTypes, {
                 name: 'Query' + (sz === 0 ? '' : sz),
                 query: '',
                 pageSize: $scope.pageSizes[1],
@@ -1009,7 +1032,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
             ActivitiesData.post({ action: '/queries/add/scan' });
 
-            const paragraph = new Paragraph($animate, $timeout, {
+            const paragraph = new Paragraph($animate, $timeout, JavaTypes, {
                 name: 'Scan' + (sz === 0 ? '' : sz),
                 query: '',
                 pageSize: $scope.pageSizes[1],
@@ -1246,7 +1269,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
             paragraph.resNodeId = res.responseNodeId;
 
-            delete paragraph.errMsg;
+            paragraph.setError({message: ''});
 
             // Prepare explain results for display in table.
             if (paragraph.queryArgs.query && paragraph.queryArgs.query.startsWith('EXPLAIN') && res.rows) {
@@ -1341,7 +1364,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 .then((nid) => agentMonitor.query(nid, args.cacheName, args.query, args.nonCollocatedJoins,
                     args.enforceJoinOrder, !!args.localNid, args.pageSize))
                 .then(_processQueryResult.bind(this, paragraph, false))
-                .catch((err) => paragraph.errMsg = err.message);
+                .catch((err) => paragraph.setError(err));
         };
 
         const _tryStartRefresh = function(paragraph) {
@@ -1419,7 +1442,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                             _tryStartRefresh(paragraph);
                         })
                         .catch((err) => {
-                            paragraph.errMsg = err.message;
+                            paragraph.setError(err);
 
                             _showLoading(paragraph, false);
 
@@ -1468,7 +1491,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 })
                 .then(_processQueryResult.bind(this, paragraph, true))
                 .catch((err) => {
-                    paragraph.errMsg = err.message;
+                    paragraph.setError(err);
 
                     _showLoading(paragraph, false);
                 })
@@ -1506,7 +1529,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                         })
                         .then((res) => _processQueryResult(paragraph, true, res))
                         .catch((err) => {
-                            paragraph.errMsg = err.message;
+                            paragraph.setError(err);
 
                             _showLoading(paragraph, false);
                         });
@@ -1560,7 +1583,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                         delete paragraph.queryId;
                 })
                 .catch((err) => {
-                    paragraph.errMsg = err.message;
+                    paragraph.setError(err);
 
                     _showLoading(paragraph, false);
                 })
@@ -1756,15 +1779,36 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 }
                 else if (paragraph.queryArgs.query .startsWith('EXPLAIN ')) {
                     scope.title = 'Explain query';
-                    scope.content = [paragraph.queryArgs.query];
+                    scope.content = paragraph.queryArgs.query.split(/\r?\n/);
                 }
                 else {
                     scope.title = 'SQL query';
-                    scope.content = [paragraph.queryArgs.query];
+                    scope.content = paragraph.queryArgs.query.split(/\r?\n/);
+                }
+
+                // Show a basic modal from a controller
+                $modal({scope, templateUrl: messageTemplateUrl, placement: 'center', show: true});
+            }
+        };
+
+        $scope.showStackTrace = function(paragraph) {
+            if (!_.isNil(paragraph)) {
+                const scope = $scope.$new();
+
+                scope.title = 'Error details';
+                scope.content = [];
+
+                let cause = paragraph.error.root;
+
+                while (_.nonNil(cause)) {
+                    scope.content.push((scope.content.length > 0 ? '&nbsp;&nbsp;&nbsp;&nbsp;' : '') +
+                        '[' + JavaTypes.shortClassName(cause.className) + '] ' + cause.message);
+
+                    cause = cause.cause;
                 }
 
                 // Show a basic modal from a controller
-                $modal({scope, templateUrl: showQueryTemplateUrl, placement: 'center', show: true});
+                $modal({scope, templateUrl: messageTemplateUrl, placement: 'center', show: true});
             }
         };
     }
diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug
index bc134a67e5bce..701ee143af0ca 100644
--- a/modules/web-console/frontend/views/sql/sql.tpl.pug
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -199,7 +199,7 @@ mixin paragraph-scan
                 | Scan on selected node
 
         .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
-            .error(ng-switch-when='error') Error: {{paragraph.errMsg}}
+            .error(ng-switch-when='error') Error: {{paragraph.error.message}}
             .empty(ng-switch-when='empty') Result set is empty
             .table(ng-switch-when='table')
                 +table-result-heading-scan
@@ -247,7 +247,7 @@ mixin paragraph-query
             .pull-right
                 +query-settings
         .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
-            .error(ng-switch-when='error') Error: {{paragraph.errMsg}}
+            .error(ng-switch-when='error') Error: {{paragraph.error.message}}
             .empty(ng-switch-when='empty') Result set is empty
             .table(ng-switch-when='table')
                 +table-result-heading-query
@@ -255,9 +255,10 @@ mixin paragraph-query
             .chart(ng-switch-when='chart')
                 +chart-result
             .footer.clearfix
-                a.pull-left(ng-click='showResultQuery(paragraph)') Show query
+                a.pull-left(ng-show='paragraph.resultType() === "error"' ng-click='showStackTrace(paragraph)') Show error details
+                a.pull-left(ng-show='paragraph.resultType() !== "error"' ng-click='showResultQuery(paragraph)') Show query
 
-                -var nextVisibleCondition = 'paragraph.resultType() != "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
+                -var nextVisibleCondition = 'paragraph.resultType() !== "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
 
                 .pull-right(ng-show=`${nextVisibleCondition}` ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
                     i.fa.fa-chevron-circle-right
diff --git a/modules/web-console/frontend/views/templates/message.tpl.pug b/modules/web-console/frontend/views/templates/message.tpl.pug
index 6eff74bba5173..aa3615fbee0c6 100644
--- a/modules/web-console/frontend/views/templates/message.tpl.pug
+++ b/modules/web-console/frontend/views/templates/message.tpl.pug
@@ -22,7 +22,7 @@
                 h4.modal-title
                     i.fa.fa-info-circle
                     | {{title}}
-            .modal-body(ng-show='content')
-                p(ng-bind-html='content.join("<br/>")' style='text-align: left;')
+            .modal-body(ng-show='content' style='overflow: auto; max-height: 300px;')
+                p(ng-bind-html='content.join("<br/>")' style='text-align: left; white-space: nowrap;')
             .modal-footer
                 button.btn.btn-primary(id='confirm-btn-confirm' ng-click='$hide()') Ok

From 93faee3579389c5754e3b91bcd39cc88dcf39125 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 21 Mar 2017 11:07:15 +0700
Subject: [PATCH 195/446] Web Console: Update classnames.properties.

---
 .../resources/META-INF/classnames.properties    | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 7bebe24f8a51b..0d3e9ea16a078 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -24,6 +24,7 @@ org.apache.ignite.IgniteException
 org.apache.ignite.IgniteIllegalStateException
 org.apache.ignite.IgniteInterruptedException
 org.apache.ignite.IgniteState
+org.apache.ignite.IgniteSystemProperties$1
 org.apache.ignite.binary.BinaryInvalidTypeException
 org.apache.ignite.binary.BinaryObject
 org.apache.ignite.binary.BinaryObjectException
@@ -382,9 +383,9 @@ org.apache.ignite.internal.processors.cache.GridCacheAdapter$12
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$13
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$14
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$15
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$16
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$16$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$17
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$18$1
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$18
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$2
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$26$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$28
@@ -842,13 +843,13 @@ org.apache.ignite.internal.processors.cache.query.GridCacheQueryAdapter$ScanQuer
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryDetailMetricsAdapter
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryFutureAdapter$1
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryFutureAdapter$2
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$10
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$11
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$12
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$13
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$14
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$15
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$16
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$17
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$18$1
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$17$1
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$4$1
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$4$2
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$5
@@ -1738,6 +1739,7 @@ org.apache.ignite.internal.visor.compute.VisorComputeToggleMonitoringTask
 org.apache.ignite.internal.visor.compute.VisorComputeToggleMonitoringTask$VisorComputeToggleMonitoringJob
 org.apache.ignite.internal.visor.compute.VisorGatewayTask
 org.apache.ignite.internal.visor.compute.VisorGatewayTask$VisorGatewayJob
+org.apache.ignite.internal.visor.compute.VisorGatewayTask$VisorGatewayJob$1
 org.apache.ignite.internal.visor.debug.VisorThreadDumpTask
 org.apache.ignite.internal.visor.debug.VisorThreadDumpTask$VisorDumpThreadJob
 org.apache.ignite.internal.visor.debug.VisorThreadInfo
@@ -1817,6 +1819,10 @@ org.apache.ignite.internal.visor.node.VisorRestConfiguration
 org.apache.ignite.internal.visor.node.VisorSegmentationConfiguration
 org.apache.ignite.internal.visor.node.VisorSpisConfiguration
 org.apache.ignite.internal.visor.node.VisorTransactionConfiguration
+org.apache.ignite.internal.visor.query.VisorCancelQueriesTask
+org.apache.ignite.internal.visor.query.VisorCancelQueriesTask$VisorCancelQueriesJob
+org.apache.ignite.internal.visor.query.VisorCollectRunningQueriesTask
+org.apache.ignite.internal.visor.query.VisorCollectRunningQueriesTask$VisorCollectRunningQueriesJob
 org.apache.ignite.internal.visor.query.VisorQueryArg
 org.apache.ignite.internal.visor.query.VisorQueryArgV2
 org.apache.ignite.internal.visor.query.VisorQueryArgV3
@@ -1830,6 +1836,7 @@ org.apache.ignite.internal.visor.query.VisorQueryResult
 org.apache.ignite.internal.visor.query.VisorQueryResultEx
 org.apache.ignite.internal.visor.query.VisorQueryScanSubstringFilter
 org.apache.ignite.internal.visor.query.VisorQueryTask
+org.apache.ignite.internal.visor.query.VisorRunningQuery
 org.apache.ignite.internal.visor.service.VisorCancelServiceTask
 org.apache.ignite.internal.visor.service.VisorCancelServiceTask$VisorCancelServiceJob
 org.apache.ignite.internal.visor.service.VisorServiceDescriptor

From 45812566694d016c07b47fa589bbba612c26f824 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 21 Mar 2017 11:23:18 +0700
Subject: [PATCH 196/446] Web Console: minor fix. (cherry picked from commit
 92bce6e)

---
 modules/web-console/frontend/app/modules/sql/sql.controller.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index a3a5a5fb81dc7..075bd559ee1e2 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -154,7 +154,7 @@ class Paragraph {
 
             while (_.nonNil(cause)) {
                 if (_.nonEmpty(cause.className) &&
-                    _.includes(['SQLException', 'JdbcSQLException'], JavaTypes.shortClassName(cause.className))) {
+                    _.includes(['SQLException', 'JdbcSQLException', 'QueryCancelledException'], JavaTypes.shortClassName(cause.className))) {
                     this.error.message = cause.message;
 
                     break;

From f4232e9557fca89065f4ee424afab940265e2d1b Mon Sep 17 00:00:00 2001
From: oleg-ostanin <oostanin@gridgain.com>
Date: Tue, 21 Mar 2017 15:32:33 +0300
Subject: [PATCH 197/446] IGNITE-4822 Fixed change jvm options for benchmarks

---
 .../config/benchmark-atomic.properties        | 18 +++++++-------
 .../config/benchmark-bin-identity.properties  |  9 ++-----
 .../config/benchmark-cache-load.properties    |  9 ++-----
 .../config/benchmark-client-mode.properties   |  9 ++-----
 .../config/benchmark-compute.properties       | 18 +++++++-------
 .../config/benchmark-failover.properties      | 18 ++++++--------
 .../config/benchmark-full.properties          | 18 ++++++--------
 .../config/benchmark-multicast.properties     |  9 ++-----
 .../benchmark-put-indexed-val.properties      | 18 +++++++-------
 .../benchmark-query-put-separated.properties  |  9 ++-----
 .../config/benchmark-query.properties         | 18 +++++++-------
 .../config/benchmark-sql-dml.properties       | 18 +++++++-------
 .../config/benchmark-store.properties         |  9 ++-----
 .../yardstick/config/benchmark-tx.properties  | 18 +++++++-------
 modules/yardstick/config/benchmark.properties | 18 ++++++--------
 .../test-max-int-values-offheap.properties    | 24 +++++++------------
 .../test-max-int-values-onheap.properties     | 24 +++++++------------
 .../test-max-int-values-swap.properties       | 24 +++++++------------
 18 files changed, 105 insertions(+), 183 deletions(-)

diff --git a/modules/yardstick/config/benchmark-atomic.properties b/modules/yardstick/config/benchmark-atomic.properties
index fc827e0edb1e6..674f5a15b3537 100644
--- a/modules/yardstick/config/benchmark-atomic.properties
+++ b/modules/yardstick/config/benchmark-atomic.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-bin-identity.properties b/modules/yardstick/config/benchmark-bin-identity.properties
index 3d754c55a4b06..13cbc61077b53 100644
--- a/modules/yardstick/config/benchmark-bin-identity.properties
+++ b/modules/yardstick/config/benchmark-bin-identity.properties
@@ -24,18 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-cache-load.properties b/modules/yardstick/config/benchmark-cache-load.properties
index 7312cb6acefd5..95e9315b3e9db 100644
--- a/modules/yardstick/config/benchmark-cache-load.properties
+++ b/modules/yardstick/config/benchmark-cache-load.properties
@@ -21,18 +21,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-client-mode.properties b/modules/yardstick/config/benchmark-client-mode.properties
index f7c83472cf570..0c3dedb7a481c 100644
--- a/modules/yardstick/config/benchmark-client-mode.properties
+++ b/modules/yardstick/config/benchmark-client-mode.properties
@@ -24,18 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 #Ignite version
 ver="RELEASE-"
diff --git a/modules/yardstick/config/benchmark-compute.properties b/modules/yardstick/config/benchmark-compute.properties
index a4816bdd008c6..30a4b09bb0686 100644
--- a/modules/yardstick/config/benchmark-compute.properties
+++ b/modules/yardstick/config/benchmark-compute.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-failover.properties b/modules/yardstick/config/benchmark-failover.properties
index 352e2d4058569..d4e17ee40780e 100644
--- a/modules/yardstick/config/benchmark-failover.properties
+++ b/modules/yardstick/config/benchmark-failover.properties
@@ -28,17 +28,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false -ea"
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
   -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -verbose:gc \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-full.properties b/modules/yardstick/config/benchmark-full.properties
index da3ae9ff8a716..9d9ea90c8acd8 100644
--- a/modules/yardstick/config/benchmark-full.properties
+++ b/modules/yardstick/config/benchmark-full.properties
@@ -25,17 +25,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
   -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -verbose:gc \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-multicast.properties b/modules/yardstick/config/benchmark-multicast.properties
index 3b31745b1d79d..e979070abf74c 100644
--- a/modules/yardstick/config/benchmark-multicast.properties
+++ b/modules/yardstick/config/benchmark-multicast.properties
@@ -24,18 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-put-indexed-val.properties b/modules/yardstick/config/benchmark-put-indexed-val.properties
index e81ae6f89e235..49be7b9cf941d 100644
--- a/modules/yardstick/config/benchmark-put-indexed-val.properties
+++ b/modules/yardstick/config/benchmark-put-indexed-val.properties
@@ -21,16 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/benchmark-query-put-separated.properties b/modules/yardstick/config/benchmark-query-put-separated.properties
index 6d2ce33e03d26..0e73bbcc927c1 100644
--- a/modules/yardstick/config/benchmark-query-put-separated.properties
+++ b/modules/yardstick/config/benchmark-query-put-separated.properties
@@ -24,18 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-query.properties b/modules/yardstick/config/benchmark-query.properties
index 486d00e14327f..44184f2997a30 100644
--- a/modules/yardstick/config/benchmark-query.properties
+++ b/modules/yardstick/config/benchmark-query.properties
@@ -21,16 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/benchmark-sql-dml.properties b/modules/yardstick/config/benchmark-sql-dml.properties
index 2ce2e1fc7a261..7aad883cb3e52 100644
--- a/modules/yardstick/config/benchmark-sql-dml.properties
+++ b/modules/yardstick/config/benchmark-sql-dml.properties
@@ -21,16 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/benchmark-store.properties b/modules/yardstick/config/benchmark-store.properties
index 746fa11ffcc42..4ebf8a60a1691 100644
--- a/modules/yardstick/config/benchmark-store.properties
+++ b/modules/yardstick/config/benchmark-store.properties
@@ -26,18 +26,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-tx.properties b/modules/yardstick/config/benchmark-tx.properties
index 0d5bb02dd3e1a..8a3620697f824 100644
--- a/modules/yardstick/config/benchmark-tx.properties
+++ b/modules/yardstick/config/benchmark-tx.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes, comma separated.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark.properties b/modules/yardstick/config/benchmark.properties
index cfc14999751d9..de0cb5aceac1b 100644
--- a/modules/yardstick/config/benchmark.properties
+++ b/modules/yardstick/config/benchmark.properties
@@ -25,17 +25,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
   -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -verbose:gc \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/test-max-int-values-offheap.properties b/modules/yardstick/config/test-max-int-values-offheap.properties
index 5b4da209db71c..fb501d7ae0c75 100644
--- a/modules/yardstick/config/test-max-int-values-offheap.properties
+++ b/modules/yardstick/config/test-max-int-values-offheap.properties
@@ -21,22 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -XX:-PrintGCTimeStamps \
-  -verbose:gc \
-  -Xmx8g \
-  -Xms8g \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=1g \
-  -XX:MaxNewSize=1g \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+    -Xms8g \
+    -Xmx8g \
+    -Xloggc:./gc${now0}.log \
+    -XX:+PrintGCDetails \
+    -verbose:gc \
+    -XX:+UseParNewGC \
+    -XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/test-max-int-values-onheap.properties b/modules/yardstick/config/test-max-int-values-onheap.properties
index d29800aa2ce70..6616003be573e 100644
--- a/modules/yardstick/config/test-max-int-values-onheap.properties
+++ b/modules/yardstick/config/test-max-int-values-onheap.properties
@@ -21,22 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -XX:-PrintGCTimeStamps \
-  -verbose:gc \
-  -Xmx92g \
-  -Xms32g \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=4g \
-  -XX:MaxNewSize=4g \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms92g \
+-Xmx32g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/test-max-int-values-swap.properties b/modules/yardstick/config/test-max-int-values-swap.properties
index 203f0040a78a3..cecad8b59b5bd 100644
--- a/modules/yardstick/config/test-max-int-values-swap.properties
+++ b/modules/yardstick/config/test-max-int-values-swap.properties
@@ -21,22 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -XX:-PrintGCTimeStamps \
-  -verbose:gc \
-  -Xmx8g \
-  -Xms8g \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=1g \
-  -XX:MaxNewSize=1g \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.

From ab3b2926213291a45305c21e7efe066855c1342f Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 21 Mar 2017 17:47:09 +0300
Subject: [PATCH 198/446] IGNITE-4200: Added copying of the C++ binaries.

---
 assembly/release-fabric-base.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml
index 7b3d8cf21e5f5..7f05c305ed4b2 100644
--- a/assembly/release-fabric-base.xml
+++ b/assembly/release-fabric-base.xml
@@ -174,6 +174,12 @@
             <outputDirectory>/platforms/cpp/docs</outputDirectory>
         </fileSet>
 
+        <!-- Move CPP binaries. -->
+        <fileSet>
+            <directory>modules/platforms/cpp/bin</directory>
+            <outputDirectory>/platforms/cpp/bin</outputDirectory>
+        </fileSet>
+
         <!-- Other files. -->
         <fileSet>
             <directory>bin</directory>

From e0c012d977b6db13dfdf2fb8347677998287c1e4 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 21 Mar 2017 17:50:06 +0300
Subject: [PATCH 199/446] IGNITE-4200: Added copying of the C++ binaries.

---
 assembly/release-fabric-base.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml
index 7b3d8cf21e5f5..7f05c305ed4b2 100644
--- a/assembly/release-fabric-base.xml
+++ b/assembly/release-fabric-base.xml
@@ -174,6 +174,12 @@
             <outputDirectory>/platforms/cpp/docs</outputDirectory>
         </fileSet>
 
+        <!-- Move CPP binaries. -->
+        <fileSet>
+            <directory>modules/platforms/cpp/bin</directory>
+            <outputDirectory>/platforms/cpp/bin</outputDirectory>
+        </fileSet>
+
         <!-- Other files. -->
         <fileSet>
             <directory>bin</directory>

From 6e2bc4bfb0b200e59c9fc3cf4fcbd408d52acb9c Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 21 Mar 2017 17:51:40 +0300
Subject: [PATCH 200/446] IGNITE-4200: Added copying of the C++ binaries.

---
 assembly/release-fabric-base.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml
index c008083221a30..78c77770e885a 100644
--- a/assembly/release-fabric-base.xml
+++ b/assembly/release-fabric-base.xml
@@ -174,6 +174,12 @@
             <outputDirectory>/platforms/cpp/docs</outputDirectory>
         </fileSet>
 
+        <!-- Move CPP binaries. -->
+        <fileSet>
+            <directory>modules/platforms/cpp/bin</directory>
+            <outputDirectory>/platforms/cpp/bin</outputDirectory>
+        </fileSet>
+
         <!-- Other files. -->
         <fileSet>
             <directory>bin</directory>

From 8273e6703944ea50b229a512398ac741eb713073 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Tue, 21 Mar 2017 18:04:01 +0300
Subject: [PATCH 201/446] IGNITE-4779 Missed discovery data snapshot during
 exchange processing (Backport from 2.0)

---
 .../managers/discovery/DiscoCache.java        | 310 ++++++++
 .../discovery/GridDiscoveryManager.java       | 710 +++++-------------
 .../eventstorage/DiscoveryEventListener.java  |  33 +
 .../eventstorage/GridEventStorageManager.java | 162 +++-
 .../affinity/GridAffinityAssignmentCache.java |   7 +-
 .../cache/CacheAffinitySharedManager.java     |  35 +-
 .../cache/GridCacheAffinityManager.java       |   2 +-
 .../GridCachePartitionExchangeManager.java    |  64 +-
 .../dht/GridClientPartitionTopology.java      |  20 +-
 .../dht/GridDhtAssignmentFetchFuture.java     |   7 +-
 .../dht/GridDhtPartitionTopologyImpl.java     |  44 +-
 .../dht/atomic/GridDhtAtomicCache.java        |   4 +-
 .../GridDhtPartitionsExchangeFuture.java      |  33 +-
 .../service/GridServiceProcessor.java         |  21 +-
 ...ridDiscoveryManagerAliveCacheSelfTest.java |  64 +-
 ...ridDiscoveryManagerAttributesSelfTest.java |  14 +-
 .../GridDiscoveryManagerSelfTest.java         | 214 ------
 .../IgniteTopologyPrintFormatSelfTest.java    |   8 +-
 .../testsuites/IgniteKernalSelfTestSuite.java |   3 -
 19 files changed, 854 insertions(+), 901 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/DiscoveryEventListener.java
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
new file mode 100644
index 0000000000000..5247ac1b28091
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.P1;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class DiscoCache {
+    /** Local node. */
+    private final ClusterNode loc;
+
+    /** Remote nodes. */
+    private final List<ClusterNode> rmtNodes;
+
+    /** All nodes. */
+    private final List<ClusterNode> allNodes;
+
+    /** All server nodes. */
+    private final List<ClusterNode> srvNodes;
+
+    /** Daemon nodes. */
+    private final List<ClusterNode> daemonNodes;
+
+    /** All server nodes. */
+    private final List<ClusterNode> srvNodesWithCaches;
+
+    /** All nodes with at least one cache configured. */
+    @GridToStringInclude
+    private final List<ClusterNode> allNodesWithCaches;
+
+    /** All remote nodes with at least one cache configured. */
+    @GridToStringInclude
+    private final List<ClusterNode> rmtNodesWithCaches;
+
+    /** Cache nodes by cache name. */
+    @GridToStringInclude
+    private final Map<Integer, List<ClusterNode>> allCacheNodes;
+
+    /** Affinity cache nodes by cache name. */
+    @GridToStringInclude
+    private final Map<Integer, List<ClusterNode>> affCacheNodes;
+
+    /** Node map. */
+    private final Map<UUID, ClusterNode> nodeMap;
+
+    /** Caches where at least one node has near cache enabled. */
+    @GridToStringInclude
+    private final Set<Integer> nearEnabledCaches;
+
+    /** Alive nodes. */
+    private final Set<UUID> alives = new GridConcurrentHashSet<>();
+
+    /**
+     * @param loc Local node.
+     * @param rmtNodes Remote nodes.
+     * @param allNodes All nodes.
+     * @param srvNodes Server nodes.
+     * @param daemonNodes Daemon nodes.
+     * @param srvNodesWithCaches Server nodes with at least one cache configured.
+     * @param allNodesWithCaches All nodes with at least one cache configured.
+     * @param rmtNodesWithCaches Remote nodes with at least one cache configured.
+     * @param allCacheNodes Cache nodes by cache name.
+     * @param affCacheNodes Affinity cache nodes by cache name.
+     * @param nodeMap Node map.
+     * @param nearEnabledCaches Caches where at least one node has near cache enabled.
+     * @param alives Alive nodes.
+     */
+    DiscoCache(ClusterNode loc,
+        List<ClusterNode> rmtNodes,
+        List<ClusterNode> allNodes,
+        List<ClusterNode> srvNodes,
+        List<ClusterNode> daemonNodes,
+        List<ClusterNode> srvNodesWithCaches,
+        List<ClusterNode> allNodesWithCaches,
+        List<ClusterNode> rmtNodesWithCaches,
+        Map<Integer, List<ClusterNode>> allCacheNodes,
+        Map<Integer, List<ClusterNode>> affCacheNodes,
+        Map<UUID, ClusterNode> nodeMap,
+        Set<Integer> nearEnabledCaches,
+        Set<UUID> alives) {
+        this.loc = loc;
+        this.rmtNodes = rmtNodes;
+        this.allNodes = allNodes;
+        this.srvNodes = srvNodes;
+        this.daemonNodes = daemonNodes;
+        this.srvNodesWithCaches = srvNodesWithCaches;
+        this.allNodesWithCaches = allNodesWithCaches;
+        this.rmtNodesWithCaches = rmtNodesWithCaches;
+        this.allCacheNodes = allCacheNodes;
+        this.affCacheNodes = affCacheNodes;
+        this.nodeMap = nodeMap;
+        this.nearEnabledCaches = nearEnabledCaches;
+        this.alives.addAll(alives);
+    }
+
+    /** @return Local node. */
+    public ClusterNode localNode() {
+        return loc;
+    }
+
+    /** @return Remote nodes. */
+    public List<ClusterNode> remoteNodes() {
+        return rmtNodes;
+    }
+
+    /** @return All nodes. */
+    public List<ClusterNode> allNodes() {
+        return allNodes;
+    }
+
+    /** @return Server nodes. */
+    public List<ClusterNode> serverNodes() {
+        return srvNodes;
+    }
+
+    /** @return Daemon nodes. */
+    public List<ClusterNode> daemonNodes() {
+        return daemonNodes;
+    }
+
+    /** @return Server nodes with at least one cache configured. */
+    public List<ClusterNode> serverNodesWithCaches() {
+        return srvNodesWithCaches;
+    }
+
+    /**
+     * Gets all remote nodes that have at least one cache configured.
+     *
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> remoteNodesWithCaches() {
+        return rmtNodesWithCaches;
+    }
+
+    /**
+     * Gets collection of nodes with at least one cache configured.
+     *
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> allNodesWithCaches() {
+        return allNodesWithCaches;
+    }
+
+    /**
+     * Gets collection of server nodes with at least one cache configured.
+     *
+     * @return Collection of nodes.
+     */
+    public Collection<ClusterNode> aliveServerNodes() {
+        return F.view(serverNodes(), new P1<ClusterNode>() {
+            @Override public boolean apply(ClusterNode node) {
+                return alives.contains(node.id());
+            }
+        });
+    }
+
+    /**
+     * Gets collection of server nodes with at least one cache configured.
+     *
+     * @return Collection of nodes.
+     */
+    public Collection<ClusterNode> aliveServerNodesWithCaches() {
+        return F.view(serverNodesWithCaches(), new P1<ClusterNode>() {
+            @Override public boolean apply(ClusterNode node) {
+                return alives.contains(node.id());
+            }
+        });
+    }
+
+    /**
+     * @return Oldest alive server node.
+     */
+    public @Nullable ClusterNode oldestAliveServerNode(){
+        Iterator<ClusterNode> it = aliveServerNodes().iterator();
+        return it.hasNext() ? it.next() : null;
+    }
+
+    /**
+     * @return Oldest alive server node with at least one cache configured.
+     */
+    public @Nullable ClusterNode oldestAliveServerNodeWithCache(){
+        Iterator<ClusterNode> it = aliveServerNodesWithCaches().iterator();
+        return it.hasNext() ? it.next() : null;
+    }
+
+    /**
+     * Gets all nodes that have cache with given name.
+     *
+     * @param cacheName Cache name.
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> cacheNodes(@Nullable String cacheName) {
+        return cacheNodes(CU.cacheId(cacheName));
+    }
+
+    /**
+     * Gets all nodes that have cache with given ID.
+     *
+     * @param cacheId Cache ID.
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> cacheNodes(Integer cacheId) {
+        return emptyIfNull(allCacheNodes.get(cacheId));
+    }
+
+    /**
+     * Gets all nodes that have cache with given ID and should participate in affinity calculation. With
+     * partitioned cache nodes with near-only cache do not participate in affinity node calculation.
+     *
+     * @param cacheName Cache name.
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> cacheAffinityNodes(@Nullable String cacheName) {
+        return cacheAffinityNodes(CU.cacheId(cacheName));
+    }
+
+    /**
+     * Gets all nodes that have cache with given ID and should participate in affinity calculation. With
+     * partitioned cache nodes with near-only cache do not participate in affinity node calculation.
+     *
+     * @param cacheId Cache ID.
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> cacheAffinityNodes(int cacheId) {
+        return emptyIfNull(affCacheNodes.get(cacheId));
+    }
+
+    /**
+     * Checks if cache with given ID has at least one node with near cache enabled.
+     *
+     * @param cacheId Cache ID.
+     * @return {@code True} if cache with given name has at least one node with near cache enabled.
+     */
+    public boolean hasNearCache(int cacheId) {
+        return nearEnabledCaches.contains(cacheId);
+    }
+
+    /**
+     * @param id Node ID.
+     * @return Node.
+     */
+    public @Nullable ClusterNode node(UUID id) {
+        return nodeMap.get(id);
+    }
+
+    /**
+     * Removes left node from alives lists.
+     *
+     * @param rmvd Removed node.
+     */
+    public void updateAlives(ClusterNode rmvd) {
+        alives.remove(rmvd.id());
+    }
+
+    /**
+     * Removes left nodes from cached alives lists.
+     *
+     * @param discovery Discovery manager.
+     */
+    public void updateAlives(GridDiscoveryManager discovery) {
+        for (UUID alive : alives) {
+            if (!discovery.alive(alive))
+                alives.remove(alive);
+        }
+    }
+
+    /**
+     * @param nodes Cluster nodes.
+     * @return Empty collection if nodes list is {@code null}
+     */
+    private List<ClusterNode> emptyIfNull(List<ClusterNode> nodes) {
+        return nodes == null ? Collections.<ClusterNode>emptyList() : nodes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DiscoCache.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 9aa4db1e0042f..80549dc08228b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -35,14 +35,12 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.NavigableMap;
 import java.util.Set;
-import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -74,24 +72,23 @@
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.jobmetrics.GridJobMetrics;
 import org.apache.ignite.internal.processors.security.SecurityContext;
 import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
-import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.lang.GridTuple5;
+import org.apache.ignite.internal.util.lang.GridTuple6;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.P1;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -99,7 +96,6 @@
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.segmentation.SegmentationPolicy;
@@ -113,6 +109,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
 import org.apache.ignite.thread.IgniteThread;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
@@ -233,7 +230,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
     private long segChkFreq;
 
     /** Local node join to topology event. */
-    private GridFutureAdapter<DiscoveryEvent> locJoinEvt = new GridFutureAdapter<>();
+    private GridFutureAdapter<T2<DiscoveryEvent, DiscoCache>> locJoin = new GridFutureAdapter<>();
 
     /** GC CPU load. */
     private volatile double gcCpuLoad;
@@ -542,20 +539,25 @@ private void updateClientNodes(UUID leftNodeId) {
                     }
                 }
 
+                final DiscoCache discoCache;
+
                 // Put topology snapshot into discovery history.
                 // There is no race possible between history maintenance and concurrent discovery
                 // event notifications, since SPI notifies manager about all events from this listener.
                 if (verChanged) {
-                    DiscoCache cache = new DiscoCache(locNode, F.view(topSnapshot, F.remoteNodes(locNode.id())));
+                    discoCache = createDiscoCache(locNode, topSnapshot);
 
-                    discoCacheHist.put(nextTopVer, cache);
+                    discoCacheHist.put(nextTopVer, discoCache);
 
-                    boolean set = updateTopologyVersionIfGreater(nextTopVer, cache);
+                    boolean set = updateTopologyVersionIfGreater(nextTopVer, discoCache);
 
                     assert set || topVer == 0 : "Topology version has not been updated [this.topVer=" +
                         topSnap + ", topVer=" + topVer + ", node=" + node +
                         ", evt=" + U.gridEventName(type) + ']';
                 }
+                else
+                    // Current version.
+                    discoCache = discoCache();
 
                 // If this is a local join event, just save it and do not notify listeners.
                 if (type == EVT_NODE_JOINED && node.id().equals(locNode.id())) {
@@ -563,7 +565,7 @@ private void updateClientNodes(UUID leftNodeId) {
                         gridStartTime = getSpi().getGridStartTime();
 
                     updateTopologyVersionIfGreater(new AffinityTopologyVersion(locNode.order()),
-                        new DiscoCache(localNode(), F.view(topSnapshot, F.remoteNodes(locNode.id()))));
+                        discoCache);
 
                     startLatch.countDown();
 
@@ -573,14 +575,9 @@ private void updateClientNodes(UUID leftNodeId) {
                     discoEvt.eventNode(node);
                     discoEvt.type(EVT_NODE_JOINED);
 
-                    discoEvt.topologySnapshot(topVer, new ArrayList<>(
-                        F.viewReadOnly(topSnapshot, new C1<ClusterNode, ClusterNode>() {
-                            @Override public ClusterNode apply(ClusterNode e) {
-                                return e;
-                            }
-                        }, FILTER_DAEMON)));
+                    discoEvt.topologySnapshot(topVer, new ArrayList<>(F.view(topSnapshot, FILTER_DAEMON)));
 
-                    locJoinEvt.onDone(discoEvt);
+                    locJoin.onDone(new T2<>(discoEvt, discoCache));
 
                     return;
                 }
@@ -595,7 +592,7 @@ else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
 
                     ((IgniteKernal)ctx.grid()).onDisconnected();
 
-                    locJoinEvt = new GridFutureAdapter<>();
+                    locJoin = new GridFutureAdapter<>();
 
                     registeredCaches.clear();
 
@@ -608,7 +605,7 @@ else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
                     topHist.clear();
 
                     topSnap.set(new Snapshot(AffinityTopologyVersion.ZERO,
-                        new DiscoCache(locNode, Collections.<ClusterNode>emptySet())));
+                        createDiscoCache(locNode, Collections.<ClusterNode>emptySet())));
                 }
                 else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                     assert locNode.isClient() : locNode;
@@ -625,7 +622,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                             try {
                                 fut.get();
 
-                                discoWrk.addEvent(type, nextTopVer, node, topSnapshot, null);
+                                discoWrk.addEvent(type, nextTopVer, node, discoCache, topSnapshot, null);
                             }
                             catch (IgniteException ignore) {
                                 // No-op.
@@ -637,7 +634,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                 }
 
                 if (type == EVT_CLIENT_NODE_DISCONNECTED || type == EVT_NODE_SEGMENTED || !ctx.clientDisconnected())
-                    discoWrk.addEvent(type, nextTopVer, node, topSnapshot, customMsg);
+                    discoWrk.addEvent(type, nextTopVer, node, discoCache, topSnapshot, customMsg);
             }
         });
 
@@ -1333,8 +1330,8 @@ private String topologySnapshotMessage(long topVer, int srvNodesNum, int clientN
             U.join(segChkThread, log);
         }
 
-        if (!locJoinEvt.isDone())
-            locJoinEvt.onDone(
+        if (!locJoin.isDone())
+            locJoin.onDone(
                 new IgniteCheckedException("Failed to wait for local node joined event (grid is stopping)."));
     }
 
@@ -1528,7 +1525,7 @@ public IgniteInternalFuture<Long> topologyFuture(final long awaitVer) {
      *
      * @return Discovery collection cache.
      */
-    private DiscoCache discoCache() {
+    public DiscoCache discoCache() {
         Snapshot cur = topSnap.get();
 
         assert cur != null;
@@ -1577,7 +1574,7 @@ public Collection<ClusterNode> nodes(long topVer) {
      * @return Collection of cache nodes.
      */
     public Collection<ClusterNode> nodes(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).allNodes();
+        return resolveDiscoCache(CU.cacheId(null), topVer).allNodes();
     }
 
     /**
@@ -1585,7 +1582,7 @@ public Collection<ClusterNode> nodes(AffinityTopologyVersion topVer) {
      * @return All server nodes for given topology version.
      */
     public List<ClusterNode> serverNodes(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).srvNodes;
+        return resolveDiscoCache(CU.cacheId(null), topVer).serverNodes();
     }
 
     /**
@@ -1596,7 +1593,7 @@ public List<ClusterNode> serverNodes(AffinityTopologyVersion topVer) {
      * @return Node.
      */
     public ClusterNode node(AffinityTopologyVersion topVer, UUID id) {
-        return resolveDiscoCache(null, topVer).node(id);
+        return resolveDiscoCache(CU.cacheId(null), topVer).node(id);
     }
 
     /**
@@ -1607,49 +1604,38 @@ public ClusterNode node(AffinityTopologyVersion topVer, UUID id) {
      * @return Collection of cache nodes.
      */
     public Collection<ClusterNode> cacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).cacheNodes(cacheName, topVer.topologyVersion());
+        return resolveDiscoCache(CU.cacheId(cacheName), topVer).cacheNodes(cacheName);
     }
 
     /**
-     * Gets all nodes with at least one cache configured.
+     * Gets cache nodes for cache with given ID.
      *
+     * @param cacheId Cache ID.
      * @param topVer Topology version.
      * @return Collection of cache nodes.
      */
-    public Collection<ClusterNode> cacheNodes(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).allNodesWithCaches(topVer.topologyVersion());
+    public Collection<ClusterNode> cacheNodes(int cacheId, AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(cacheId, topVer).cacheNodes(cacheId);
     }
 
     /**
-     * Gets cache remote nodes for cache with given name.
-     *
-     * @param topVer Topology version.
-     * @return Collection of cache nodes.
-     */
-    public Collection<ClusterNode> remoteCacheNodes(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).remoteCacheNodes(topVer.topologyVersion());
-    }
-
-    /**
-     * Gets cache nodes for cache with given name.
+     * Gets all nodes with at least one cache configured.
      *
-     * @param cacheName Cache name.
      * @param topVer Topology version.
      * @return Collection of cache nodes.
      */
-    Collection<ClusterNode> aliveCacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).aliveCacheNodes(cacheName, topVer.topologyVersion());
+    public Collection<ClusterNode> cacheNodes(AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(CU.cacheId(null), topVer).allNodesWithCaches();
     }
 
     /**
      * Gets cache remote nodes for cache with given name.
      *
-     * @param cacheName Cache name.
      * @param topVer Topology version.
      * @return Collection of cache nodes.
      */
-    Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).aliveRemoteCacheNodes(cacheName, topVer.topologyVersion());
+    public Collection<ClusterNode> remoteCacheNodes(AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(CU.cacheId(null), topVer).remoteNodesWithCaches();
     }
 
     /**
@@ -1657,11 +1643,7 @@ Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, Affini
      * @return Oldest alive server nodes with at least one cache configured.
      */
     @Nullable public ClusterNode oldestAliveCacheServerNode(AffinityTopologyVersion topVer) {
-        DiscoCache cache = resolveDiscoCache(null, topVer);
-
-        Map.Entry<ClusterNode, Boolean> e = cache.aliveSrvNodesWithCaches.firstEntry();
-
-        return e != null ? e.getKey() : null;
+        return resolveDiscoCache(CU.cacheId(null), topVer).oldestAliveServerNodeWithCache();
     }
 
     /**
@@ -1672,7 +1654,20 @@ Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, Affini
      * @return Collection of cache affinity nodes.
      */
     public Collection<ClusterNode> cacheAffinityNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).cacheAffinityNodes(cacheName, topVer.topologyVersion());
+        int cacheId = CU.cacheId(cacheName);
+
+        return resolveDiscoCache(cacheId, topVer).cacheAffinityNodes(cacheId);
+    }
+
+    /**
+     * Gets cache nodes for cache with given ID that participate in affinity calculation.
+     *
+     * @param cacheId Cache ID.
+     * @param topVer Topology version.
+     * @return Collection of cache affinity nodes.
+     */
+    public Collection<ClusterNode> cacheAffinityNodes(int cacheId, AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(cacheId, topVer).cacheAffinityNodes(cacheId);
     }
 
     /**
@@ -1742,31 +1737,34 @@ public Map<String, CacheMode> nodeCaches(ClusterNode node) {
     }
 
     /**
-     * Checks if cache with given name has at least one node with near cache enabled.
+     * Checks if cache with given ID has at least one node with near cache enabled.
      *
-     * @param cacheName Cache name.
+     * @param cacheId Cache ID.
      * @param topVer Topology version.
      * @return {@code True} if cache with given name has at least one node with near cache enabled.
      */
-    public boolean hasNearCache(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).hasNearCache(cacheName);
+    public boolean hasNearCache(int cacheId, AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(cacheId, topVer).hasNearCache(cacheId);
     }
 
     /**
      * Gets discovery cache for given topology version.
      *
-     * @param cacheName Cache name (participates in exception message).
+     * @param cacheId Cache ID (participates in exception message).
      * @param topVer Topology version.
      * @return Discovery cache.
      */
-    private DiscoCache resolveDiscoCache(@Nullable String cacheName, AffinityTopologyVersion topVer) {
+    private DiscoCache resolveDiscoCache(int cacheId, AffinityTopologyVersion topVer) {
         Snapshot snap = topSnap.get();
 
         DiscoCache cache = AffinityTopologyVersion.NONE.equals(topVer) || topVer.equals(snap.topVer) ?
             snap.discoCache : discoCacheHist.get(topVer);
 
         if (cache == null) {
-            throw new IgniteException("Failed to resolve nodes topology [cacheName=" + cacheName +
+            DynamicCacheDescriptor desc = ctx.cache().cacheDescriptor(cacheId);
+
+            throw new IgniteException("Failed to resolve nodes topology [" +
+                "cacheName=" + (desc != null ? desc.cacheConfiguration().getName() : "N/A") +
                 ", topVer=" + topVer +
                 ", history=" + discoCacheHist.keySet() +
                 ", snap=" + snap +
@@ -1817,7 +1815,19 @@ public AffinityTopologyVersion topologyVersionEx() {
     /** @return Event that represents a local node joined to topology. */
     public DiscoveryEvent localJoinEvent() {
         try {
-            return locJoinEvt.get();
+            return locJoin.get().get1();
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException(e);
+        }
+    }
+
+    /**
+     * @return Tuple that consists of a local join event and discovery cache at the join time.
+     */
+    public T2<DiscoveryEvent, DiscoCache> localJoin() {
+        try {
+            return locJoin.get();
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException(e);
@@ -1890,6 +1900,114 @@ public void failNode(UUID nodeId, @Nullable String warning) {
         }
     }
 
+    /**
+     * @param loc Local node.
+     * @param topSnapshot Topology snapshot.
+     * @return Newly created discovery cache.
+     */
+    @NotNull private DiscoCache createDiscoCache(ClusterNode loc, Collection<ClusterNode> topSnapshot) {
+        HashSet<UUID> alives = U.newHashSet(topSnapshot.size());
+        HashMap<UUID, ClusterNode> nodeMap = U.newHashMap(topSnapshot.size());
+
+        ArrayList<ClusterNode> daemonNodes = new ArrayList<>(topSnapshot.size());
+        ArrayList<ClusterNode> srvNodes = new ArrayList<>(topSnapshot.size());
+        ArrayList<ClusterNode> rmtNodes = new ArrayList<>(topSnapshot.size());
+        ArrayList<ClusterNode> allNodes = new ArrayList<>(topSnapshot.size());
+
+        for (ClusterNode node : topSnapshot) {
+            if (alive(node))
+                alives.add(node.id());
+
+            if (node.isDaemon())
+                daemonNodes.add(node);
+            else {
+                allNodes.add(node);
+
+                if (!node.isLocal())
+                    rmtNodes.add(node);
+
+                if (!CU.clientNode(node))
+                    srvNodes.add(node);
+            }
+
+            nodeMap.put(node.id(), node);
+        }
+
+        assert !rmtNodes.contains(loc) : "Remote nodes collection shouldn't contain local node" +
+            " [rmtNodes=" + rmtNodes + ", loc=" + loc + ']';
+
+        Map<Integer, List<ClusterNode>> allCacheNodes = U.newHashMap(allNodes.size());
+        Map<Integer, List<ClusterNode>> affCacheNodes = U.newHashMap(allNodes.size());
+
+        Set<ClusterNode> allNodesWithCaches = new TreeSet<>(GridNodeOrderComparator.INSTANCE);
+        Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(GridNodeOrderComparator.INSTANCE);
+        Set<ClusterNode> srvNodesWithCaches = new TreeSet<>(GridNodeOrderComparator.INSTANCE);
+
+        Set<Integer> nearEnabledCaches = new HashSet<>();
+
+        for (ClusterNode node : allNodes) {
+            assert node.order() != 0 : "Invalid node order [locNode=" + loc + ", node=" + node + ']';
+            assert !node.isDaemon();
+
+            for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
+                String cacheName = entry.getKey();
+                CachePredicate filter = entry.getValue();
+
+                if (filter.cacheNode(node)) {
+                    allNodesWithCaches.add(node);
+
+                    if(!CU.clientNode(node))
+                        srvNodesWithCaches.add(node);
+
+                    if (!node.isLocal())
+                        rmtNodesWithCaches.add(node);
+
+                    addToMap(allCacheNodes, cacheName, node);
+
+                    if (filter.dataNode(node))
+                        addToMap(affCacheNodes, cacheName, node);
+
+                    if (filter.nearNode(node))
+                        nearEnabledCaches.add(CU.cacheId(cacheName));
+                }
+            }
+        }
+
+        return new DiscoCache(
+            loc,
+            Collections.unmodifiableList(rmtNodes),
+            Collections.unmodifiableList(allNodes),
+            Collections.unmodifiableList(srvNodes),
+            Collections.unmodifiableList(daemonNodes),
+            U.sealList(srvNodesWithCaches),
+            U.sealList(allNodesWithCaches),
+            U.sealList(rmtNodesWithCaches),
+            Collections.unmodifiableMap(allCacheNodes),
+            Collections.unmodifiableMap(affCacheNodes),
+            Collections.unmodifiableMap(nodeMap),
+            Collections.unmodifiableSet(nearEnabledCaches),
+            alives);
+    }
+
+    /**
+     * Adds node to map.
+     *
+     * @param cacheMap Map to add to.
+     * @param cacheName Cache name.
+     * @param rich Node to add
+     */
+    private void addToMap(Map<Integer, List<ClusterNode>> cacheMap, String cacheName, ClusterNode rich) {
+        List<ClusterNode> cacheNodes = cacheMap.get(CU.cacheId(cacheName));
+
+        if (cacheNodes == null) {
+            cacheNodes = new ArrayList<>();
+
+            cacheMap.put(CU.cacheId(cacheName), cacheNodes);
+        }
+
+        cacheNodes.add(rich);
+    }
+
     /**
      * Updates topology version if current version is smaller than updated.
      *
@@ -1991,8 +2109,16 @@ public void scheduleSegmentCheck() {
                     lastChk = now;
 
                     if (!segValid) {
-                        discoWrk.addEvent(EVT_NODE_SEGMENTED, AffinityTopologyVersion.NONE, getSpi().getLocalNode(),
-                            Collections.<ClusterNode>emptyList(), null);
+                        List<ClusterNode> empty = Collections.emptyList();
+
+                        ClusterNode node = getSpi().getLocalNode();
+
+                        discoWrk.addEvent(EVT_NODE_SEGMENTED,
+                            AffinityTopologyVersion.NONE,
+                            node,
+                            createDiscoCache(node, empty),
+                            empty,
+                            null);
 
                         lastSegChkRes.set(false);
                     }
@@ -2012,8 +2138,8 @@ public void scheduleSegmentCheck() {
     /** Worker for discovery events. */
     private class DiscoveryWorker extends GridWorker {
         /** Event queue. */
-        private final BlockingQueue<GridTuple5<Integer, AffinityTopologyVersion, ClusterNode, Collection<ClusterNode>,
-            DiscoveryCustomMessage>> evts = new LinkedBlockingQueue<>();
+        private final BlockingQueue<GridTuple6<Integer, AffinityTopologyVersion, ClusterNode,
+            DiscoCache, Collection<ClusterNode>, DiscoveryCustomMessage>> evts = new LinkedBlockingQueue<>();
 
         /** Node segmented event fired flag. */
         private boolean nodeSegFired;
@@ -2031,10 +2157,11 @@ private DiscoveryWorker() {
          * @param type Discovery event type. See {@link DiscoveryEvent} for more details.
          * @param topVer Topology version.
          * @param node Remote node this event is connected with.
+         * @param discoCache Discovery cache.
          * @param topSnapshot Topology snapshot.
          */
         @SuppressWarnings("RedundantTypeArguments")
-        private void recordEvent(int type, long topVer, ClusterNode node, Collection<ClusterNode> topSnapshot) {
+        private void recordEvent(int type, long topVer, ClusterNode node, DiscoCache discoCache, Collection<ClusterNode> topSnapshot) {
             assert node != null;
 
             if (ctx.event().isRecordable(type)) {
@@ -2043,7 +2170,6 @@ private void recordEvent(int type, long topVer, ClusterNode node, Collection<Clu
                 evt.node(ctx.discovery().localNode());
                 evt.eventNode(node);
                 evt.type(type);
-
                 evt.topologySnapshot(topVer, U.<ClusterNode, ClusterNode>arrayList(topSnapshot, FILTER_DAEMON));
 
                 if (type == EVT_NODE_METRICS_UPDATED)
@@ -2070,7 +2196,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED)
                 else
                     assert false;
 
-                ctx.event().record(evt);
+                ctx.event().record(evt, discoCache);
             }
         }
 
@@ -2078,6 +2204,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED)
          * @param type Event type.
          * @param topVer Topology version.
          * @param node Node.
+         * @param discoCache Discovery cache.
          * @param topSnapshot Topology snapshot.
          * @param data Custom message.
          */
@@ -2085,12 +2212,13 @@ void addEvent(
             int type,
             AffinityTopologyVersion topVer,
             ClusterNode node,
+            DiscoCache discoCache,
             Collection<ClusterNode> topSnapshot,
             @Nullable DiscoveryCustomMessage data
         ) {
             assert node != null : data;
 
-            evts.add(new GridTuple5<>(type, topVer, node, topSnapshot, data));
+            evts.add(new GridTuple6<>(type, topVer, node, discoCache, topSnapshot, data));
         }
 
         /**
@@ -2127,7 +2255,7 @@ private String quietNode(ClusterNode node) {
         /** @throws InterruptedException If interrupted. */
         @SuppressWarnings("DuplicateCondition")
         private void body0() throws InterruptedException {
-            GridTuple5<Integer, AffinityTopologyVersion, ClusterNode, Collection<ClusterNode>,
+            GridTuple6<Integer, AffinityTopologyVersion, ClusterNode, DiscoCache, Collection<ClusterNode>,
                 DiscoveryCustomMessage> evt = evts.take();
 
             int type = evt.get1();
@@ -2260,11 +2388,11 @@ else if (log.isDebugEnabled())
                         customEvt.node(ctx.discovery().localNode());
                         customEvt.eventNode(node);
                         customEvt.type(type);
-                        customEvt.topologySnapshot(topVer.topologyVersion(), evt.get4());
+                        customEvt.topologySnapshot(topVer.topologyVersion(), evt.get5());
                         customEvt.affinityTopologyVersion(topVer);
-                        customEvt.customMessage(evt.get5());
+                        customEvt.customMessage(evt.get6());
 
-                        ctx.event().record(customEvt);
+                        ctx.event().record(customEvt, evt.get4());
                     }
 
                     return;
@@ -2278,7 +2406,7 @@ else if (log.isDebugEnabled())
                     assert false : "Invalid discovery event: " + type;
             }
 
-            recordEvent(type, topVer.topologyVersion(), node, evt.get4());
+            recordEvent(type, topVer.topologyVersion(), node, evt.get4(), evt.get5());
 
             if (segmented)
                 onSegmentation();
@@ -2488,432 +2616,6 @@ private Snapshot(AffinityTopologyVersion topVer, DiscoCache discoCache) {
         }
     }
 
-    /** Cache for discovery collections. */
-    private class DiscoCache {
-        /** Remote nodes. */
-        private final List<ClusterNode> rmtNodes;
-
-        /** All nodes. */
-        private final List<ClusterNode> allNodes;
-
-        /** All server nodes. */
-        private final List<ClusterNode> srvNodes;
-
-        /** All nodes with at least one cache configured. */
-        @GridToStringInclude
-        private final Collection<ClusterNode> allNodesWithCaches;
-
-        /** All nodes with at least one cache configured. */
-        @GridToStringInclude
-        private final Collection<ClusterNode> rmtNodesWithCaches;
-
-        /** Cache nodes by cache name. */
-        @GridToStringInclude
-        private final Map<String, Collection<ClusterNode>> allCacheNodes;
-
-        /** Remote cache nodes by cache name. */
-        @GridToStringInclude
-        private final Map<String, Collection<ClusterNode>> rmtCacheNodes;
-
-        /** Cache nodes by cache name. */
-        @GridToStringInclude
-        private final Map<String, Collection<ClusterNode>> affCacheNodes;
-
-        /** Caches where at least one node has near cache enabled. */
-        @GridToStringInclude
-        private final Set<String> nearEnabledCaches;
-
-        /** Nodes grouped by version. */
-        private final NavigableMap<IgniteProductVersion, Collection<ClusterNode>> nodesByVer;
-
-        /** Daemon nodes. */
-        private final List<ClusterNode> daemonNodes;
-
-        /** Node map. */
-        private final Map<UUID, ClusterNode> nodeMap;
-
-        /** Local node. */
-        private final ClusterNode loc;
-
-        /** Highest node order. */
-        private final long maxOrder;
-
-        /**
-         * Cached alive nodes list. As long as this collection doesn't accept {@code null}s use {@link
-         * #maskNull(String)} before passing raw cache names to it.
-         */
-        private final ConcurrentMap<String, Collection<ClusterNode>> aliveCacheNodes;
-
-        /**
-         * Cached alive remote nodes list. As long as this collection doesn't accept {@code null}s use {@link
-         * #maskNull(String)} before passing raw cache names to it.
-         */
-        private final ConcurrentMap<String, Collection<ClusterNode>> aliveRmtCacheNodes;
-
-        /**
-         * Cached alive server remote nodes with caches.
-         */
-        private final ConcurrentSkipListMap<ClusterNode, Boolean> aliveSrvNodesWithCaches;
-
-        /**
-         * @param loc Local node.
-         * @param rmts Remote nodes.
-         */
-        private DiscoCache(ClusterNode loc, Collection<ClusterNode> rmts) {
-            this.loc = loc;
-
-            rmtNodes = Collections.unmodifiableList(new ArrayList<>(F.view(rmts, FILTER_DAEMON)));
-
-            assert !rmtNodes.contains(loc) : "Remote nodes collection shouldn't contain local node" +
-                " [rmtNodes=" + rmtNodes + ", loc=" + loc + ']';
-
-            List<ClusterNode> all = new ArrayList<>(rmtNodes.size() + 1);
-
-            if (!loc.isDaemon())
-                all.add(loc);
-
-            all.addAll(rmtNodes);
-
-            Collections.sort(all, GridNodeOrderComparator.INSTANCE);
-
-            allNodes = Collections.unmodifiableList(all);
-
-            Map<String, Collection<ClusterNode>> cacheMap = new HashMap<>(allNodes.size(), 1.0f);
-            Map<String, Collection<ClusterNode>> rmtCacheMap = new HashMap<>(allNodes.size(), 1.0f);
-            Map<String, Collection<ClusterNode>> dhtNodesMap = new HashMap<>(allNodes.size(), 1.0f);
-            Collection<ClusterNode> nodesWithCaches = new HashSet<>(allNodes.size());
-            Collection<ClusterNode> rmtNodesWithCaches = new HashSet<>(allNodes.size());
-
-            aliveCacheNodes = new ConcurrentHashMap8<>(allNodes.size(), 1.0f);
-            aliveRmtCacheNodes = new ConcurrentHashMap8<>(allNodes.size(), 1.0f);
-            aliveSrvNodesWithCaches = new ConcurrentSkipListMap<>(GridNodeOrderComparator.INSTANCE);
-            nodesByVer = new TreeMap<>();
-
-            long maxOrder0 = 0;
-
-            Set<String> nearEnabledSet = new HashSet<>();
-
-            List<ClusterNode> srvNodes = new ArrayList<>();
-
-            for (ClusterNode node : allNodes) {
-                assert node.order() != 0 : "Invalid node order [locNode=" + loc + ", node=" + node + ']';
-                assert !node.isDaemon();
-
-                if (!CU.clientNode(node))
-                    srvNodes.add(node);
-
-                if (node.order() > maxOrder0)
-                    maxOrder0 = node.order();
-
-                boolean hasCaches = false;
-
-                for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
-                    String cacheName = entry.getKey();
-
-                    CachePredicate filter = entry.getValue();
-
-                    if (filter.cacheNode(node)) {
-                        nodesWithCaches.add(node);
-
-                        if (!loc.id().equals(node.id()))
-                            rmtNodesWithCaches.add(node);
-
-                        addToMap(cacheMap, cacheName, node);
-
-                        if (alive(node.id()))
-                            addToMap(aliveCacheNodes, maskNull(cacheName), node);
-
-                        if (filter.dataNode(node))
-                            addToMap(dhtNodesMap, cacheName, node);
-
-                        if (filter.nearNode(node))
-                            nearEnabledSet.add(cacheName);
-
-                        if (!loc.id().equals(node.id())) {
-                            addToMap(rmtCacheMap, cacheName, node);
-
-                            if (alive(node.id()))
-                                addToMap(aliveRmtCacheNodes, maskNull(cacheName), node);
-                        }
-
-                        hasCaches = true;
-                    }
-                }
-
-                if (hasCaches && alive(node.id()) && !CU.clientNode(node))
-                    aliveSrvNodesWithCaches.put(node, Boolean.TRUE);
-
-                IgniteProductVersion nodeVer = U.productVersion(node);
-
-                // Create collection for this version if it does not exist.
-                Collection<ClusterNode> nodes = nodesByVer.get(nodeVer);
-
-                if (nodes == null) {
-                    nodes = new ArrayList<>(allNodes.size());
-
-                    nodesByVer.put(nodeVer, nodes);
-                }
-
-                nodes.add(node);
-            }
-
-            Collections.sort(srvNodes, CU.nodeComparator(true));
-
-            // Need second iteration to add this node to all previous node versions.
-            for (ClusterNode node : allNodes) {
-                IgniteProductVersion nodeVer = U.productVersion(node);
-
-                // Get all versions lower or equal node's version.
-                NavigableMap<IgniteProductVersion, Collection<ClusterNode>> updateView =
-                    nodesByVer.headMap(nodeVer, false);
-
-                for (Collection<ClusterNode> prevVersions : updateView.values())
-                    prevVersions.add(node);
-            }
-
-            maxOrder = maxOrder0;
-
-            allCacheNodes = Collections.unmodifiableMap(cacheMap);
-            rmtCacheNodes = Collections.unmodifiableMap(rmtCacheMap);
-            affCacheNodes = Collections.unmodifiableMap(dhtNodesMap);
-            allNodesWithCaches = Collections.unmodifiableCollection(nodesWithCaches);
-            this.rmtNodesWithCaches = Collections.unmodifiableCollection(rmtNodesWithCaches);
-            nearEnabledCaches = Collections.unmodifiableSet(nearEnabledSet);
-            this.srvNodes = Collections.unmodifiableList(srvNodes);
-
-            daemonNodes = Collections.unmodifiableList(new ArrayList<>(
-                F.view(F.concat(false, loc, rmts), F0.not(FILTER_DAEMON))));
-
-            Map<UUID, ClusterNode> nodeMap = new HashMap<>(allNodes().size() + daemonNodes.size(), 1.0f);
-
-            for (ClusterNode n : F.concat(false, allNodes(), daemonNodes()))
-                nodeMap.put(n.id(), n);
-
-            this.nodeMap = nodeMap;
-        }
-
-        /**
-         * Adds node to map.
-         *
-         * @param cacheMap Map to add to.
-         * @param cacheName Cache name.
-         * @param rich Node to add
-         */
-        private void addToMap(Map<String, Collection<ClusterNode>> cacheMap, String cacheName, ClusterNode rich) {
-            Collection<ClusterNode> cacheNodes = cacheMap.get(cacheName);
-
-            if (cacheNodes == null) {
-                cacheNodes = new ArrayList<>(allNodes.size());
-
-                cacheMap.put(cacheName, cacheNodes);
-            }
-
-            cacheNodes.add(rich);
-        }
-
-        /** @return Local node. */
-        ClusterNode localNode() {
-            return loc;
-        }
-
-        /** @return Remote nodes. */
-        Collection<ClusterNode> remoteNodes() {
-            return rmtNodes;
-        }
-
-        /** @return All nodes. */
-        Collection<ClusterNode> allNodes() {
-            return allNodes;
-        }
-
-        /**
-         * Gets collection of nodes which have version equal or greater than {@code ver}.
-         *
-         * @param ver Version to check.
-         * @return Collection of nodes with version equal or greater than {@code ver}.
-         */
-        Collection<ClusterNode> elderNodes(IgniteProductVersion ver) {
-            Map.Entry<IgniteProductVersion, Collection<ClusterNode>> entry = nodesByVer.ceilingEntry(ver);
-
-            if (entry == null)
-                return Collections.emptyList();
-
-            return entry.getValue();
-        }
-
-        /**
-         * @return Versions map.
-         */
-        NavigableMap<IgniteProductVersion, Collection<ClusterNode>> versionsMap() {
-            return nodesByVer;
-        }
-
-        /**
-         * Gets collection of nodes with at least one cache configured.
-         *
-         * @param topVer Topology version (maximum allowed node order).
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> allNodesWithCaches(final long topVer) {
-            return filter(topVer, allNodesWithCaches);
-        }
-
-        /**
-         * Gets all nodes that have cache with given name.
-         *
-         * @param cacheName Cache name.
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> cacheNodes(@Nullable String cacheName, final long topVer) {
-            return filter(topVer, allCacheNodes.get(cacheName));
-        }
-
-        /**
-         * Gets all remote nodes that have at least one cache configured.
-         *
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> remoteCacheNodes(final long topVer) {
-            return filter(topVer, rmtNodesWithCaches);
-        }
-
-        /**
-         * Gets all nodes that have cache with given name and should participate in affinity calculation. With
-         * partitioned cache nodes with near-only cache do not participate in affinity node calculation.
-         *
-         * @param cacheName Cache name.
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> cacheAffinityNodes(@Nullable String cacheName, final long topVer) {
-            return filter(topVer, affCacheNodes.get(cacheName));
-        }
-
-        /**
-         * Gets all alive nodes that have cache with given name.
-         *
-         * @param cacheName Cache name.
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> aliveCacheNodes(@Nullable String cacheName, final long topVer) {
-            return filter(topVer, aliveCacheNodes.get(maskNull(cacheName)));
-        }
-
-        /**
-         * Gets all alive remote nodes that have cache with given name.
-         *
-         * @param cacheName Cache name.
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, final long topVer) {
-            return filter(topVer, aliveRmtCacheNodes.get(maskNull(cacheName)));
-        }
-
-        /**
-         * Checks if cache with given name has at least one node with near cache enabled.
-         *
-         * @param cacheName Cache name.
-         * @return {@code True} if cache with given name has at least one node with near cache enabled.
-         */
-        boolean hasNearCache(@Nullable String cacheName) {
-            return nearEnabledCaches.contains(cacheName);
-        }
-
-        /**
-         * Removes left node from cached alives lists.
-         *
-         * @param leftNode Left node.
-         */
-        void updateAlives(ClusterNode leftNode) {
-            if (leftNode.order() > maxOrder)
-                return;
-
-            filterNodeMap(aliveCacheNodes, leftNode);
-
-            filterNodeMap(aliveRmtCacheNodes, leftNode);
-
-            aliveSrvNodesWithCaches.remove(leftNode);
-        }
-
-        /**
-         * Creates a copy of nodes map without the given node.
-         *
-         * @param map Map to copy.
-         * @param exclNode Node to exclude.
-         */
-        private void filterNodeMap(ConcurrentMap<String, Collection<ClusterNode>> map, final ClusterNode exclNode) {
-            for (String cacheName : registeredCaches.keySet()) {
-                String maskedName = maskNull(cacheName);
-
-                while (true) {
-                    Collection<ClusterNode> oldNodes = map.get(maskedName);
-
-                    if (oldNodes == null || oldNodes.isEmpty())
-                        break;
-
-                    Collection<ClusterNode> newNodes = new ArrayList<>(oldNodes);
-
-                    if (!newNodes.remove(exclNode))
-                        break;
-
-                    if (map.replace(maskedName, oldNodes, newNodes))
-                        break;
-                }
-            }
-        }
-
-        /**
-         * Replaces {@code null} with {@code NULL_CACHE_NAME}.
-         *
-         * @param cacheName Cache name.
-         * @return Masked name.
-         */
-        private String maskNull(@Nullable String cacheName) {
-            return cacheName == null ? NULL_CACHE_NAME : cacheName;
-        }
-
-        /**
-         * @param topVer Topology version.
-         * @param nodes Nodes.
-         * @return Filtered collection (potentially empty, but never {@code null}).
-         */
-        private Collection<ClusterNode> filter(final long topVer, @Nullable Collection<ClusterNode> nodes) {
-            if (nodes == null)
-                return Collections.emptyList();
-
-            // If no filtering needed, return original collection.
-            return nodes.isEmpty() || topVer < 0 || topVer >= maxOrder ?
-                nodes :
-                F.view(nodes, new P1<ClusterNode>() {
-                    @Override public boolean apply(ClusterNode node) {
-                        return node.order() <= topVer;
-                    }
-                });
-        }
-
-        /** @return Daemon nodes. */
-        Collection<ClusterNode> daemonNodes() {
-            return daemonNodes;
-        }
-
-        /**
-         * @param id Node ID.
-         * @return Node.
-         */
-        @Nullable ClusterNode node(UUID id) {
-            return nodeMap.get(id);
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return S.toString(DiscoCache.class, this, "allNodesWithDaemons", U.toShortString(allNodes));
-        }
-    }
-
     /**
      * Cache predicate.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/DiscoveryEventListener.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/DiscoveryEventListener.java
new file mode 100644
index 0000000000000..963d97eefee84
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/DiscoveryEventListener.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.eventstorage;
+
+import java.util.EventListener;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+
+/**
+ * Internal listener for discovery events.
+ */
+public interface DiscoveryEventListener extends EventListener {
+    /**
+     * @param evt Discovery event.
+     * @param discoCache Discovery cache.
+     */
+    public void onEvent(DiscoveryEvent evt, DiscoCache discoCache);
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
index 607bb9688aaf1..a2c64ba33d8b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
@@ -47,6 +47,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoManager;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.deployment.GridDeployment;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.platform.PlatformEventFilterListener;
 import org.apache.ignite.internal.util.GridConcurrentLinkedHashSet;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -79,6 +80,9 @@ public class GridEventStorageManager extends GridManagerAdapter<EventStorageSpi>
     /** Local event listeners. */
     private final ConcurrentMap<Integer, Set<GridLocalEventListener>> lsnrs = new ConcurrentHashMap8<>();
 
+    /** Internal discovery listeners. */
+    private final ConcurrentMap<Integer, Set<DiscoveryEventListener>> discoLsnrs = new ConcurrentHashMap8<>();
+
     /** Busy lock to control activity of threads. */
     private final ReadWriteLock busyLock = new ReentrantReadWriteLock();
 
@@ -234,6 +238,7 @@ private void leaveBusy() {
         msgLsnr = null;
 
         lsnrs.clear();
+        discoLsnrs.clear();
     }
 
     /** {@inheritDoc} */
@@ -299,6 +304,30 @@ public void record(Event evt) {
         }
     }
 
+    /**
+     * Records discovery events.
+     *
+     * @param evt Event to record.
+     * @param discoCache Discovery cache.
+     */
+    public void record(DiscoveryEvent evt, DiscoCache discoCache) {
+        assert evt != null;
+
+        if (!enterBusy())
+            return;
+
+        try {
+            // Notify internal discovery listeners first.
+            notifyDiscoveryListeners(evt, discoCache);
+
+            // Notify all other registered listeners.
+            record(evt);
+        }
+        finally {
+            leaveBusy();
+        }
+    }
+
     /**
      * Gets types of enabled user-recordable events.
      *
@@ -570,7 +599,7 @@ public void addLocalEventListener(GridLocalEventListener lsnr, int[] types) {
 
         try {
             for (int t : types) {
-                getOrCreate(t).add(lsnr);
+                getOrCreate(lsnrs, t).add(lsnr);
 
                 if (!isRecordable(t))
                     U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
@@ -595,14 +624,14 @@ public void addLocalEventListener(GridLocalEventListener lsnr, int type, @Nullab
             return;
 
         try {
-            getOrCreate(type).add(lsnr);
+            getOrCreate(lsnrs, type).add(lsnr);
 
             if (!isRecordable(type))
                 U.warn(log, "Added listener for disabled event type: " + U.gridEventName(type));
 
             if (types != null) {
                 for (int t : types) {
-                    getOrCreate(t).add(lsnr);
+                    getOrCreate(lsnrs, t).add(lsnr);
 
                     if (!isRecordable(t))
                         U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
@@ -615,16 +644,70 @@ public void addLocalEventListener(GridLocalEventListener lsnr, int type, @Nullab
     }
 
     /**
+     * Adds discovery event listener. Note that this method specifically disallow an empty
+     * array of event type to prevent accidental subscription for all system event that
+     * may lead to a drastic performance decrease.
+     *
+     * @param lsnr Listener to add.
+     * @param types Event types to subscribe listener for.
+     */
+    public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int[] types) {
+        assert lsnr != null;
+        assert types != null;
+        assert types.length > 0;
+
+        if (!enterBusy())
+            return;
+
+        try {
+            for (int t : types) {
+                getOrCreate(discoLsnrs, t).add(lsnr);
+            }
+        }
+        finally {
+            leaveBusy();
+        }
+    }
+
+    /**
+     * Adds discovery event listener.
+     *
+     * @param lsnr Listener to add.
+     * @param type Event type to subscribe listener for.
+     * @param types Additional event types to subscribe listener for.
+     */
+    public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int type, @Nullable int... types) {
+        assert lsnr != null;
+
+        if (!enterBusy())
+            return;
+
+        try {
+            getOrCreate(discoLsnrs, type).add(lsnr);
+
+            if (types != null) {
+                for (int t : types) {
+                    getOrCreate(discoLsnrs, t).add(lsnr);
+                }
+            }
+        }
+        finally {
+            leaveBusy();
+        }
+    }
+
+    /**
+     * @param lsnrs Listeners map.
      * @param type Event type.
      * @return Listeners for given event type.
      */
-    private Collection<GridLocalEventListener> getOrCreate(Integer type) {
-        Set<GridLocalEventListener> set = lsnrs.get(type);
+    private <T> Collection<T> getOrCreate(ConcurrentMap<Integer, Set<T>> lsnrs, Integer type) {
+        Set<T> set = lsnrs.get(type);
 
         if (set == null) {
             set = new GridConcurrentLinkedHashSet<>();
 
-            Set<GridLocalEventListener> prev = lsnrs.putIfAbsent(type, set);
+            Set<T> prev = lsnrs.putIfAbsent(type, set);
 
             if (prev != null)
                 set = prev;
@@ -687,6 +770,38 @@ public boolean removeLocalEventListener(GridLocalEventListener lsnr, @Nullable i
         return found;
     }
 
+    /**
+     * Removes listener for specified events, if any. If no event types provided - it
+     * remove the listener for all its registered events.
+     *
+     * @param lsnr Listener.
+     * @param types Event types.
+     * @return Returns {@code true} if removed.
+     */
+    public boolean removeDiscoveryEventListener(DiscoveryEventListener lsnr, @Nullable int... types) {
+        assert lsnr != null;
+
+        boolean found = false;
+
+        if (F.isEmpty(types)) {
+            for (Set<DiscoveryEventListener> set : discoLsnrs.values())
+                if (set.remove(lsnr))
+                    found = true;
+        }
+        else {
+            assert types != null;
+
+            for (int type : types) {
+                Set<DiscoveryEventListener> set = discoLsnrs.get(type);
+
+                if (set != null && set.remove(lsnr))
+                    found = true;
+            }
+        }
+
+        return found;
+    }
+
     /**
      *
      * @param p Optional predicate.
@@ -779,6 +894,41 @@ private void notifyListeners(@Nullable Collection<GridLocalEventListener> set, E
         }
     }
 
+    /**
+     * @param evt Discovery event
+     * @param cache Discovery cache.
+     */
+    private void notifyDiscoveryListeners(DiscoveryEvent evt, DiscoCache cache) {
+        assert evt != null;
+
+        notifyDiscoveryListeners(discoLsnrs.get(evt.type()), evt, cache);
+    }
+
+    /**
+     * @param set Set of listeners.
+     * @param evt Discovery event.
+     * @param cache Discovery cache.
+     */
+    private void notifyDiscoveryListeners(@Nullable Collection<DiscoveryEventListener> set, DiscoveryEvent evt, DiscoCache cache) {
+        assert evt != null;
+
+        if (!F.isEmpty(set)) {
+            assert set != null;
+
+            for (DiscoveryEventListener lsnr : set) {
+                try {
+                    lsnr.onEvent(evt, cache);
+                }
+                catch (Throwable e) {
+                    U.error(log, "Unexpected exception in listener notification for event: " + evt, e);
+
+                    if (e instanceof Error)
+                        throw (Error)e;
+                }
+            }
+        }
+    }
+
     /**
      * @param p Grid event predicate.
      * @return Collection of grid events.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index a388c7affb65a..50704622017df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -252,10 +253,12 @@ public void onReconnected() {
      *
      * @param topVer Topology version to calculate affinity cache for.
      * @param discoEvt Discovery event that caused this topology version change.
+     * @param discoCache Discovery cache.
      * @return Affinity assignments.
      */
     @SuppressWarnings("IfMayBeConditional")
-    public List<List<ClusterNode>> calculate(AffinityTopologyVersion topVer, DiscoveryEvent discoEvt) {
+    public List<List<ClusterNode>> calculate(AffinityTopologyVersion topVer, DiscoveryEvent discoEvt,
+        DiscoCache discoCache) {
         if (log.isDebugEnabled())
             log.debug("Calculating affinity [topVer=" + topVer + ", locNodeId=" + ctx.localNodeId() +
                 ", discoEvt=" + discoEvt + ']');
@@ -266,7 +269,7 @@ public List<List<ClusterNode>> calculate(AffinityTopologyVersion topVer, Discove
         List<ClusterNode> sorted;
 
         if (!locCache) {
-            sorted = new ArrayList<>(ctx.discovery().cacheAffinityNodes(cacheName, topVer));
+            sorted = new ArrayList<>(discoCache.cacheAffinityNodes(cacheId()));
 
             Collections.sort(sorted, GridNodeOrderComparator.INSTANCE);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 2890887cca416..2642d16c56fc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -382,7 +382,7 @@ else if (req.start() && !req.clientStartOnly()) {
                 cctx.cache().prepareCacheStart(req, fut.topologyVersion());
 
                 if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
-                    if (cctx.discovery().cacheAffinityNodes(req.cacheName(), fut.topologyVersion()).isEmpty())
+                    if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
                         U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
                 }
 
@@ -403,7 +403,7 @@ else if (!req.clientStartOnly()) {
                             assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
 
                             List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
-                                fut.discoveryEvent());
+                                fut.discoveryEvent(), fut.discoCache());
 
                             aff.initialize(fut.topologyVersion(), assignment);
                         }
@@ -753,7 +753,7 @@ private void initStartedCacheOnCoordinator(GridDhtPartitionsExchangeFuture fut,
 
             assert old == null : old;
 
-            List<List<ClusterNode>> newAff = cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent());
+            List<List<ClusterNode>> newAff = cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
             cache.affinity().initialize(fut.topologyVersion(), newAff);
         }
@@ -791,7 +791,7 @@ public void initStartedCaches(boolean crd,
 
                     if (cache.affinity().lastVersion().equals(AffinityTopologyVersion.NONE)) {
                         List<List<ClusterNode>> assignment =
-                            cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent());
+                            cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
                         cache.affinity().initialize(fut.topologyVersion(), assignment);
                     }
@@ -817,14 +817,15 @@ public void initStartedCaches(boolean crd,
     private void initAffinity(GridAffinityAssignmentCache aff, GridDhtPartitionsExchangeFuture fut, boolean fetch)
         throws IgniteCheckedException {
         if (!fetch && canCalculateAffinity(aff, fut)) {
-            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(), fut.discoveryEvent());
+            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
             aff.initialize(fut.topologyVersion(), assignment);
         }
         else {
             GridDhtAssignmentFetchFuture fetchFut = new GridDhtAssignmentFetchFuture(cctx,
                 aff.cacheName(),
-                fut.topologyVersion());
+                fut.topologyVersion(),
+                fut.discoCache());
 
             fetchFut.init();
 
@@ -878,7 +879,7 @@ public void onServerJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd)
 
                             CacheHolder cache = cache(fut, cacheDesc);
 
-                            List<List<ClusterNode>> newAff = cache.affinity().calculate(topVer, fut.discoveryEvent());
+                            List<List<ClusterNode>> newAff = cache.affinity().calculate(topVer, fut.discoveryEvent(), fut.discoCache());
 
                             cache.affinity().initialize(topVer, newAff);
                         }
@@ -945,14 +946,15 @@ private void fetchAffinityOnJoin(GridDhtPartitionsExchangeFuture fut) throws Ign
 
             if (cctx.localNodeId().equals(cacheDesc.receivedFrom())) {
                 List<List<ClusterNode>> assignment =
-                    cacheCtx.affinity().affinityCache().calculate(fut.topologyVersion(), fut.discoveryEvent());
+                    cacheCtx.affinity().affinityCache().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
                 cacheCtx.affinity().affinityCache().initialize(fut.topologyVersion(), assignment);
             }
             else {
                 GridDhtAssignmentFetchFuture fetchFut = new GridDhtAssignmentFetchFuture(cctx,
                     cacheCtx.name(),
-                    topVer);
+                    topVer,
+                    fut.discoCache());
 
                 fetchFut.init();
 
@@ -986,7 +988,7 @@ private void fetchAffinity(GridDhtPartitionsExchangeFuture fut,
         GridDhtAffinityAssignmentResponse res = fetchFut.get();
 
         if (res == null) {
-            List<List<ClusterNode>> aff = affCache.calculate(topVer, fut.discoveryEvent());
+            List<List<ClusterNode>> aff = affCache.calculate(topVer, fut.discoveryEvent(), fut.discoCache());
 
             affCache.initialize(topVer, aff);
         }
@@ -998,7 +1000,7 @@ private void fetchAffinity(GridDhtPartitionsExchangeFuture fut,
             else {
                 assert !affCache.centralizedAffinityFunction() || !lateAffAssign;
 
-                affCache.calculate(topVer, fut.discoveryEvent());
+                affCache.calculate(topVer, fut.discoveryEvent(), fut.discoCache());
             }
 
             List<List<ClusterNode>> aff = res.affinityAssignment(cctx.discovery());
@@ -1028,7 +1030,7 @@ public boolean onServerLeft(final GridDhtPartitionsExchangeFuture fut) throws Ig
                 if (cacheCtx.isLocal())
                     continue;
 
-                cacheCtx.affinity().affinityCache().calculate(fut.topologyVersion(), fut.discoveryEvent());
+                cacheCtx.affinity().affinityCache().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
             }
 
             centralizedAff = true;
@@ -1078,7 +1080,7 @@ private IgniteInternalFuture<?> initCoordinatorCaches(final GridDhtPartitionsExc
 
                 if (cache != null) {
                     if (cache.client())
-                        cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent());
+                        cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
                     return;
                 }
@@ -1118,7 +1120,8 @@ private IgniteInternalFuture<?> initCoordinatorCaches(final GridDhtPartitionsExc
 
                     GridDhtAssignmentFetchFuture fetchFut = new GridDhtAssignmentFetchFuture(cctx,
                         aff.cacheName(),
-                        prev.topologyVersion());
+                        prev.topologyVersion(),
+                        prev.discoCache());
 
                     fetchFut.init();
 
@@ -1129,7 +1132,7 @@ private IgniteInternalFuture<?> initCoordinatorCaches(final GridDhtPartitionsExc
                             throws IgniteCheckedException {
                             fetchAffinity(prev, aff, (GridDhtAssignmentFetchFuture)fetchFut);
 
-                            aff.calculate(fut.topologyVersion(), fut.discoveryEvent());
+                            aff.calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
                             affFut.onDone(fut.topologyVersion());
                         }
@@ -1269,7 +1272,7 @@ private void initAffinityOnNodeJoin(GridDhtPartitionsExchangeFuture fut,
 
         assert aff.idealAssignment() != null : "Previous assignment is not available.";
 
-        List<List<ClusterNode>> idealAssignment = aff.calculate(topVer, fut.discoveryEvent());
+        List<List<ClusterNode>> idealAssignment = aff.calculate(topVer, fut.discoveryEvent(), fut.discoCache());
         List<List<ClusterNode>> newAssignment = null;
 
         if (latePrimary) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
index 8b7be1b3288e2..d9ff61684e7d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
@@ -77,7 +77,7 @@ public class GridCacheAffinityManager extends GridCacheManagerAdapter {
     @Override protected void onKernalStart0() throws IgniteCheckedException {
         if (cctx.isLocal())
             // No discovery event needed for local affinity.
-            aff.calculate(LOC_CACHE_TOP_VER, null);
+            aff.calculate(LOC_CACHE_TOP_VER, null, null);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7cf75fee86be3..81f21a8405fb3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -48,14 +48,14 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
-import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
-import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
@@ -173,35 +173,33 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     private DateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS");
 
     /** Discovery listener. */
-    private final GridLocalEventListener discoLsnr = new GridLocalEventListener() {
-        @Override public void onEvent(Event evt) {
+    private final DiscoveryEventListener discoLsnr = new DiscoveryEventListener() {
+        @Override public void onEvent(DiscoveryEvent evt, DiscoCache cache) {
             if (!enterBusy())
                 return;
 
             try {
-                DiscoveryEvent e = (DiscoveryEvent)evt;
-
                 ClusterNode loc = cctx.localNode();
 
-                assert e.type() == EVT_NODE_JOINED || e.type() == EVT_NODE_LEFT || e.type() == EVT_NODE_FAILED ||
-                    e.type() == EVT_DISCOVERY_CUSTOM_EVT;
+                assert evt.type() == EVT_NODE_JOINED || evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED ||
+                    evt.type() == EVT_DISCOVERY_CUSTOM_EVT;
 
-                final ClusterNode n = e.eventNode();
+                final ClusterNode n = evt.eventNode();
 
                 GridDhtPartitionExchangeId exchId = null;
                 GridDhtPartitionsExchangeFuture exchFut = null;
 
-                if (e.type() != EVT_DISCOVERY_CUSTOM_EVT) {
+                if (evt.type() != EVT_DISCOVERY_CUSTOM_EVT) {
                     assert !loc.id().equals(n.id());
 
-                    if (e.type() == EVT_NODE_LEFT || e.type() == EVT_NODE_FAILED) {
+                    if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) {
                         assert cctx.discovery().node(n.id()) == null;
 
                         // Avoid race b/w initial future add and discovery event.
                         GridDhtPartitionsExchangeFuture initFut = null;
 
                         if (readyTopVer.get().equals(AffinityTopologyVersion.NONE)) {
-                            initFut = exchangeFuture(initialExchangeId(), null, null, null);
+                            initFut = exchangeFuture(initialExchangeId(), null, null, null, null);
 
                             initFut.onNodeLeft(n);
                         }
@@ -213,18 +211,18 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                     }
 
                     assert
-                        e.type() != EVT_NODE_JOINED || n.order() > loc.order() :
+                        evt.type() != EVT_NODE_JOINED || n.order() > loc.order() :
                         "Node joined with smaller-than-local " +
                             "order [newOrder=" + n.order() + ", locOrder=" + loc.order() + ']';
 
                     exchId = exchangeId(n.id(),
-                        affinityTopologyVersion(e),
-                        e.type());
+                        affinityTopologyVersion(evt),
+                        evt.type());
 
-                    exchFut = exchangeFuture(exchId, e, null, null);
+                    exchFut = exchangeFuture(exchId, evt, cache,null, null);
                 }
                 else {
-                    DiscoveryCustomEvent customEvt = (DiscoveryCustomEvent)e;
+                    DiscoveryCustomEvent customEvt = (DiscoveryCustomEvent)evt;
 
                     if (customEvt.customMessage() instanceof DynamicCacheChangeBatch) {
                         DynamicCacheChangeBatch batch = (DynamicCacheChangeBatch)customEvt.customMessage();
@@ -254,9 +252,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                         }
 
                         if (!F.isEmpty(valid)) {
-                            exchId = exchangeId(n.id(), affinityTopologyVersion(e), e.type());
+                            exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type());
 
-                            exchFut = exchangeFuture(exchId, e, valid, null);
+                            exchFut = exchangeFuture(exchId, evt, cache, valid, null);
                         }
                     }
                     else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
@@ -264,13 +262,13 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
 
                         if (msg.exchangeId() == null) {
                             if (msg.exchangeNeeded()) {
-                                exchId = exchangeId(n.id(), affinityTopologyVersion(e), e.type());
+                                exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type());
 
-                                exchFut = exchangeFuture(exchId, e, null, msg);
+                                exchFut = exchangeFuture(exchId, evt, cache, null, msg);
                             }
                         }
                         else
-                            exchangeFuture(msg.exchangeId(), null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
+                            exchangeFuture(msg.exchangeId(), null, null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
                     }
                 }
 
@@ -279,7 +277,7 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                         log.debug("Discovery event (will start exchange): " + exchId);
 
                     // Event callback - without this callback future will never complete.
-                    exchFut.onEvent(exchId, e);
+                    exchFut.onEvent(exchId, evt, cache);
 
                     // Start exchange process.
                     addFuture(exchFut);
@@ -301,7 +299,7 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
 
         exchWorker = new ExchangeWorker();
 
-        cctx.gridEvents().addLocalEventListener(discoLsnr, EVT_NODE_JOINED, EVT_NODE_LEFT, EVT_NODE_FAILED,
+        cctx.gridEvents().addDiscoveryEventListener(discoLsnr, EVT_NODE_JOINED, EVT_NODE_LEFT, EVT_NODE_FAILED,
             EVT_DISCOVERY_CUSTOM_EVT);
 
         cctx.io().addHandler(0, GridDhtPartitionsSingleMessage.class,
@@ -359,11 +357,14 @@ private GridDhtPartitionExchangeId initialExchangeId() {
         assert startTime > 0;
 
         // Generate dummy discovery event for local node joining.
-        DiscoveryEvent discoEvt = cctx.discovery().localJoinEvent();
+        T2<DiscoveryEvent, DiscoCache> localJoin = cctx.discovery().localJoin();
+
+        DiscoveryEvent discoEvt = localJoin.get1();
+        DiscoCache discoCache = localJoin.get2();
 
         GridDhtPartitionExchangeId exchId = initialExchangeId();
 
-        GridDhtPartitionsExchangeFuture fut = exchangeFuture(exchId, discoEvt, null, null);
+        GridDhtPartitionsExchangeFuture fut = exchangeFuture(exchId, discoEvt, discoCache, null, null);
 
         if (reconnect)
             reconnectExchangeFut = new GridFutureAdapter<>();
@@ -470,7 +471,7 @@ public static Object rebalanceTopic(int idx) {
 
     /** {@inheritDoc} */
     @Override protected void onKernalStop0(boolean cancel) {
-        cctx.gridEvents().removeLocalEventListener(discoLsnr);
+        cctx.gridEvents().removeDiscoveryEventListener(discoLsnr);
 
         cctx.io().removeHandler(0, GridDhtPartitionsSingleMessage.class);
         cctx.io().removeHandler(0, GridDhtPartitionsFullMessage.class);
@@ -1063,12 +1064,14 @@ private GridDhtPartitionExchangeId exchangeId(UUID nodeId, AffinityTopologyVersi
     /**
      * @param exchId Exchange ID.
      * @param discoEvt Discovery event.
+     * @param cache Discovery data cache.
      * @param reqs Cache change requests.
      * @param affChangeMsg Affinity change message.
      * @return Exchange future.
      */
     private GridDhtPartitionsExchangeFuture exchangeFuture(GridDhtPartitionExchangeId exchId,
         @Nullable DiscoveryEvent discoEvt,
+        @Nullable DiscoCache cache,
         @Nullable Collection<DynamicCacheChangeRequest> reqs,
         @Nullable CacheAffinityChangeMessage affChangeMsg) {
         GridDhtPartitionsExchangeFuture fut;
@@ -1087,7 +1090,7 @@ private GridDhtPartitionsExchangeFuture exchangeFuture(GridDhtPartitionExchangeI
         }
 
         if (discoEvt != null)
-            fut.onEvent(exchId, discoEvt);
+            fut.onEvent(exchId, discoEvt, cache);
 
         if (stopErr != null)
             fut.onDone(stopErr);
@@ -1231,7 +1234,7 @@ else if (!cacheCtx.isLocal())
                     refreshPartitions();
             }
             else
-                exchangeFuture(msg.exchangeId(), null, null, null).onReceive(node, msg);
+                exchangeFuture(msg.exchangeId(), null, null, null, null).onReceive(node, msg);
         }
         finally {
             leaveBusy();
@@ -1285,6 +1288,7 @@ else if (!cacheCtx.isLocal())
             else {
                 if (msg.client()) {
                     final GridDhtPartitionsExchangeFuture exchFut = exchangeFuture(msg.exchangeId(),
+                        null,
                         null,
                         null,
                         null);
@@ -1297,7 +1301,7 @@ else if (!cacheCtx.isLocal())
                     });
                 }
                 else
-                    exchangeFuture(msg.exchangeId(), null, null, null).onReceive(node, msg);
+                    exchangeFuture(msg.exchangeId(), null, null, null, null).onReceive(node, msg);
             }
         }
         finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 816132d351139..9366d0ca127c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
@@ -42,7 +43,6 @@
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
@@ -103,6 +103,9 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
     /** */
     private final Object similarAffKey;
 
+    /** */
+    private volatile DiscoCache discoCache;
+
     /**
      * @param cctx Context.
      * @param cacheId Cache ID.
@@ -121,6 +124,8 @@ public GridClientPartitionTopology(
 
         topVer = exchFut.topologyVersion();
 
+        discoCache = exchFut.discoCache();
+
         log = cctx.logger(getClass());
 
         lock.writeLock().lock();
@@ -191,6 +196,7 @@ public int cacheId() {
             this.stopping = stopping;
 
             topVer = exchId.topologyVersion();
+            discoCache = exchFut.discoCache();
 
             updateSeq.setIfGreater(updSeq);
 
@@ -271,7 +277,7 @@ private void beforeExchange0(ClusterNode loc, GridDhtPartitionsExchangeFuture ex
             removeNode(exchId.nodeId());
 
         // In case if node joins, get topology at the time of joining node.
-        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
+        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
         assert oldest != null;
 
@@ -424,7 +430,7 @@ else if (!node2part.nodeId().equals(loc.id())) {
 
             if (!F.isEmpty(nodeIds)) {
                 for (UUID nodeId : nodeIds) {
-                    ClusterNode n = cctx.discovery().node(nodeId);
+                    ClusterNode n = discoCache.node(nodeId);
 
                     if (n != null && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion())) {
                         if (nodes == null)
@@ -450,7 +456,7 @@ else if (!node2part.nodeId().equals(loc.id())) {
      * @return List of nodes for the partition.
      */
     private List<ClusterNode> nodes(int p, AffinityTopologyVersion topVer, GridDhtPartitionState state, GridDhtPartitionState... states) {
-        Collection<UUID> allIds = topVer.topologyVersion() > 0 ? F.nodeIds(CU.allNodes(cctx, topVer)) : null;
+        Collection<UUID> allIds = topVer.topologyVersion() > 0 ? F.nodeIds(discoCache.allNodesWithCaches()) : null;
 
         lock.readLock().lock();
 
@@ -473,7 +479,7 @@ private List<ClusterNode> nodes(int p, AffinityTopologyVersion topVer, GridDhtPa
                     continue;
 
                 if (hasState(p, id, state, states)) {
-                    ClusterNode n = cctx.discovery().node(id);
+                    ClusterNode n = discoCache.node(id);
 
                     if (n != null && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion()))
                         nodes.add(n);
@@ -766,7 +772,7 @@ private void updateLocal(int p, UUID nodeId, GridDhtPartitionState state, long u
         assert nodeId.equals(cctx.localNodeId());
 
         // In case if node joins, get topology at the time of joining node.
-        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
+        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
         // If this node became the oldest node.
         if (oldest.id().equals(cctx.localNodeId())) {
@@ -816,7 +822,7 @@ private void removeNode(UUID nodeId) {
         assert nodeId != null;
         assert lock.writeLock().isHeldByCurrentThread();
 
-        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
+        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
         ClusterNode loc = cctx.localNode();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index ab8e863e90e7d..2c3d7ecc54482 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -72,16 +73,18 @@ public class GridDhtAssignmentFetchFuture extends GridFutureAdapter<GridDhtAffin
      * @param ctx Context.
      * @param cacheName Cache name.
      * @param topVer Topology version.
+     * @param discoCache Discovery cache.
      */
     public GridDhtAssignmentFetchFuture(
         GridCacheSharedContext ctx,
         String cacheName,
-        AffinityTopologyVersion topVer
+        AffinityTopologyVersion topVer,
+        DiscoCache discoCache
     ) {
         this.ctx = ctx;
         this.key = new T2<>(CU.cacheId(cacheName), topVer);
 
-        Collection<ClusterNode> availableNodes = ctx.discovery().cacheAffinityNodes(cacheName, topVer);
+        Collection<ClusterNode> availableNodes = discoCache.cacheAffinityNodes(CU.cacheId(cacheName));
 
         LinkedList<ClusterNode> tmp = new LinkedList<>();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 1b4dcc9293dee..5c3fba08fcf7d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -95,6 +96,9 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
     /** */
     private volatile AffinityTopologyVersion topVer = AffinityTopologyVersion.NONE;
 
+    /** Discovery cache. */
+    private volatile DiscoCache discoCache;
+
     /** */
     private volatile boolean stopping;
 
@@ -151,6 +155,8 @@ public void onReconnected() {
             rebalancedTopVer = AffinityTopologyVersion.NONE;
 
             topVer = AffinityTopologyVersion.NONE;
+
+            discoCache = cctx.discovery().discoCache();
         }
         finally {
             lock.writeLock().unlock();
@@ -293,6 +299,8 @@ private boolean waitForRent() throws IgniteCheckedException {
             rebalancedTopVer = AffinityTopologyVersion.NONE;
 
             topVer = exchId.topologyVersion();
+
+            discoCache = exchFut.discoCache();
         }
         finally {
             lock.writeLock().unlock();
@@ -356,7 +364,7 @@ private boolean waitForRent() throws IgniteCheckedException {
     private void initPartitions0(GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
         ClusterNode loc = cctx.localNode();
 
-        ClusterNode oldest = currentCoordinator();
+        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
         GridDhtPartitionExchangeId exchId = exchFut.exchangeId();
 
@@ -481,7 +489,7 @@ else if (localNode(p, aff))
             if (exchId.isLeft())
                 removeNode(exchId.nodeId());
 
-            ClusterNode oldest = currentCoordinator();
+            ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
             if (log.isDebugEnabled())
                 log.debug("Partition map beforeExchange [exchId=" + exchId + ", fullMap=" + fullMapString() + ']');
@@ -882,7 +890,7 @@ private List<ClusterNode> nodes(int p,
         AffinityTopologyVersion topVer,
         GridDhtPartitionState state,
         GridDhtPartitionState... states) {
-        Collection<UUID> allIds = topVer.topologyVersion() > 0 ? F.nodeIds(CU.affinityNodes(cctx, topVer)) : null;
+        Collection<UUID> allIds = topVer.topologyVersion() > 0 ? F.nodeIds(discoCache.cacheAffinityNodes(cctx.cacheId())) : null;
 
         lock.readLock().lock();
 
@@ -979,7 +987,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
 
     /** {@inheritDoc} */
     @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-    @Nullable @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
+    @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         GridDhtPartitionFullMap partMap,
         @Nullable Map<Integer, Long> cntrMap) {
         if (log.isDebugEnabled())
@@ -1112,7 +1120,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
+    @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         GridDhtPartitionMap2 parts,
         @Nullable Map<Integer, Long> cntrMap,
         boolean checkEvictions) {
@@ -1284,7 +1292,8 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
                 List<ClusterNode> affNodes = aff.get(p);
 
                 if (!affNodes.contains(cctx.localNode())) {
-                    Collection<UUID> nodeIds = F.nodeIds(nodes(p, topVer, OWNING));
+                    List<ClusterNode> nodes = nodes(p, topVer, OWNING);
+                    Collection<UUID> nodeIds = F.nodeIds(nodes);
 
                     // If all affinity nodes are owners, then evict partition from local node.
                     if (nodeIds.containsAll(F.nodeIds(affNodes))) {
@@ -1302,15 +1311,13 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
                         int affCnt = affNodes.size();
 
                         if (ownerCnt > affCnt) {
-                            List<ClusterNode> sorted = new ArrayList<>(cctx.discovery().nodes(nodeIds));
-
                             // Sort by node orders in ascending order.
-                            Collections.sort(sorted, CU.nodeComparator(true));
+                            Collections.sort(nodes, CU.nodeComparator(true));
 
-                            int diff = sorted.size() - affCnt;
+                            int diff = nodes.size() - affCnt;
 
                             for (int i = 0; i < diff; i++) {
-                                ClusterNode n = sorted.get(i);
+                                ClusterNode n = nodes.get(i);
 
                                 if (locId.equals(n.id())) {
                                     part.rent(false);
@@ -1335,17 +1342,6 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
         return changed;
     }
 
-    /**
-     * @return Current coordinator node.
-     */
-    @Nullable private ClusterNode currentCoordinator() {
-        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
-
-        assert oldest != null || cctx.kernalContext().clientNode();
-
-        return oldest;
-    }
-
     /**
      * Updates value for single partition.
      *
@@ -1356,7 +1352,7 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
      */
     @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
     private long updateLocal(int p, GridDhtPartitionState state, long updateSeq) {
-        ClusterNode oldest = currentCoordinator();
+        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
         assert oldest != null || cctx.kernalContext().clientNode();
 
@@ -1421,7 +1417,7 @@ private long updateLocal(int p, GridDhtPartitionState state, long updateSeq) {
     private void removeNode(UUID nodeId) {
         assert nodeId != null;
 
-        ClusterNode oldest = CU.oldest(cctx.discovery().serverNodes(topVer));
+        ClusterNode oldest = discoCache.oldestAliveServerNode();
 
         assert oldest != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 44780f18a78eb..c91f881b60378 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -2441,7 +2441,7 @@ private UpdateSingleResult updateSingle(
 
         AffinityTopologyVersion topVer = req.topologyVersion();
 
-        boolean checkReaders = hasNear || ctx.discovery().hasNearCache(name(), topVer);
+        boolean checkReaders = hasNear || ctx.discovery().hasNearCache(ctx.cacheId(), topVer);
 
         boolean readersOnly = false;
 
@@ -2676,7 +2676,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
 
         AffinityTopologyVersion topVer = req.topologyVersion();
 
-        boolean checkReaders = hasNear || ctx.discovery().hasNearCache(name(), topVer);
+        boolean checkReaders = hasNear || ctx.discovery().hasNearCache(ctx.cacheId(), topVer);
 
         CacheStorePartialUpdateException storeErr = null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index e945de958a339..7b5d09b844c31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -44,7 +44,7 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
-import org.apache.ignite.internal.managers.discovery.GridDiscoveryTopologySnapshot;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -103,6 +103,10 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** Dummy reassign flag. */
     private final boolean reassign;
 
+    /** */
+    @GridToStringExclude
+    private volatile DiscoCache discoCache;
+
     /** Discovery event. */
     private volatile DiscoveryEvent discoEvt;
 
@@ -147,9 +151,6 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** */
     private boolean init;
 
-    /** Topology snapshot. */
-    private AtomicReference<GridDiscoveryTopologySnapshot> topSnapshot = new AtomicReference<>();
-
     /** Last committed cache version before next topology version use. */
     private AtomicReference<GridCacheVersion> lastVer = new AtomicReference<>();
 
@@ -331,6 +332,13 @@ public boolean dummyReassign() {
         return (dummy() || forcePreload()) && reassign();
     }
 
+    /**
+     * @return Discovery cache.
+     */
+    public DiscoCache discoCache() {
+        return discoCache;
+    }
+
     /**
      * @param cacheId Cache ID to check.
      * @param topVer Topology version.
@@ -374,11 +382,13 @@ public boolean onAdded() {
      *
      * @param exchId Exchange ID.
      * @param discoEvt Discovery event.
+     * @param discoCache Discovery data cache.
      */
-    public void onEvent(GridDhtPartitionExchangeId exchId, DiscoveryEvent discoEvt) {
+    public void onEvent(GridDhtPartitionExchangeId exchId, DiscoveryEvent discoEvt, DiscoCache discoCache) {
         assert exchId.equals(this.exchId);
 
         this.discoEvt = discoEvt;
+        this.discoCache = discoCache;
 
         evtLatch.countDown();
     }
@@ -435,7 +445,9 @@ public void init() throws IgniteInterruptedCheckedException {
         assert !dummy && !forcePreload : this;
 
         try {
-            srvNodes = new ArrayList<>(cctx.discovery().serverNodes(topologyVersion()));
+            discoCache.updateAlives(cctx.discovery());
+
+            srvNodes = new ArrayList<>(discoCache.serverNodes());
 
             remaining.addAll(F.nodeIds(F.view(srvNodes, F.remoteNodes(cctx.localNodeId()))));
 
@@ -550,7 +562,7 @@ private void updateTopologies(boolean crd) throws IgniteCheckedException {
                     exchId.topologyVersion().equals(cacheCtx.startTopologyVersion());
 
                 if (updateTop && clientTop != null)
-                    cacheCtx.topology().update(exchId, clientTop.partitionMap(true), clientTop.updateCounters(false));
+                    top.update(exchId, clientTop.partitionMap(true), clientTop.updateCounters(false));
             }
 
             top.updateTopologyVersion(exchId, this, updSeq, stopping(cacheCtx.cacheId()));
@@ -842,7 +854,7 @@ private void warnNoAffinityNodes() {
         List<String> cachesWithoutNodes = null;
 
         for (String name : cctx.cache().cacheNames()) {
-            if (cctx.discovery().cacheAffinityNodes(name, topologyVersion()).isEmpty()) {
+            if (discoCache.cacheAffinityNodes(name).isEmpty()) {
                 if (cachesWithoutNodes == null)
                     cachesWithoutNodes = new ArrayList<>();
 
@@ -1096,7 +1108,6 @@ private void sendPartitions(ClusterNode oldestNode) {
      * Cleans up resources to avoid excessive memory usage.
      */
     public void cleanUp() {
-        topSnapshot.set(null);
         singleMsgs.clear();
         fullMsgs.clear();
         crd = null;
@@ -1252,7 +1263,7 @@ private void onAllReceived(boolean discoThread) {
         try {
             assert crd.isLocal();
 
-            if (!crd.equals(cctx.discovery().serverNodes(topologyVersion()).get(0))) {
+            if (!crd.equals(discoCache.serverNodes().get(0))) {
                 for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
                     if (!cacheCtx.isLocal())
                         cacheCtx.topology().beforeExchange(GridDhtPartitionsExchangeFuture.this, !centralizedAff);
@@ -1559,6 +1570,8 @@ public void onNodeLeft(final ClusterNode node) {
 
                         ClusterNode crd0;
 
+                        discoCache.updateAlives(node);
+
                         synchronized (mux) {
                             if (!srvNodes.remove(node))
                                 return;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index d26242db68f0d..99146aab1d88e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -48,7 +48,6 @@
 import org.apache.ignite.configuration.DeploymentMode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
-import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.GridClosureCallMode;
 import org.apache.ignite.internal.GridKernalContext;
@@ -58,8 +57,9 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
-import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -167,7 +167,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
     private IgniteInternalCache<Object, Object> cache;
 
     /** Topology listener. */
-    private GridLocalEventListener topLsnr = new TopologyListener();
+    private DiscoveryEventListener topLsnr = new TopologyListener();
 
     static {
         Set<IgniteProductVersion> versions = new TreeSet<>(new Comparator<IgniteProductVersion>() {
@@ -251,7 +251,7 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         cache = ctx.cache().utilityCache();
 
         if (!ctx.clientNode())
-            ctx.event().addLocalEventListener(topLsnr, EVTS);
+            ctx.event().addDiscoveryEventListener(topLsnr, EVTS);
 
         try {
             if (ctx.deploy().enabled())
@@ -314,7 +314,7 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         busyLock.block();
 
         if (!ctx.clientNode())
-            ctx.event().removeLocalEventListener(topLsnr);
+            ctx.event().removeDiscoveryEventListener(topLsnr);
 
         Collection<ServiceContextImpl> ctxs = new ArrayList<>();
 
@@ -1568,9 +1568,9 @@ private void onDeployment(final GridServiceDeployment dep, final AffinityTopolog
     /**
      * Topology listener.
      */
-    private class TopologyListener implements GridLocalEventListener {
+    private class TopologyListener implements DiscoveryEventListener {
         /** {@inheritDoc} */
-        @Override public void onEvent(Event evt) {
+        @Override public void onEvent(DiscoveryEvent evt, final DiscoCache discoCache) {
             if (!busyLock.enterBusy())
                 return;
 
@@ -1588,11 +1588,14 @@ private class TopologyListener implements GridLocalEventListener {
                     }
                 }
                 else
-                    topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
+                    topVer = new AffinityTopologyVersion((evt).topologyVersion(), 0);
 
                 depExe.execute(new BusyRunnable() {
                     @Override public void run0() {
-                        ClusterNode oldest = ctx.discovery().oldestAliveCacheServerNode(topVer);
+                        // In case the cache instance isn't tracked by DiscoveryManager anymore.
+                        discoCache.updateAlives(ctx.discovery());
+
+                        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
                         if (oldest != null && oldest.isLocal()) {
                             final Collection<GridServiceDeployment> retries = new ConcurrentLinkedQueue<>();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
index 31b4bc7ca6682..f0c50ebeb3bf1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
@@ -23,16 +23,11 @@
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.GridCacheUtils;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -90,8 +85,8 @@ public class GridDiscoveryManagerAliveCacheSelfTest extends GridCommonAbstractTe
     }
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
         CacheConfiguration cCfg = defaultCacheConfiguration();
 
@@ -103,7 +98,7 @@ public class GridDiscoveryManagerAliveCacheSelfTest extends GridCommonAbstractTe
 
         TcpDiscoverySpi disc = new TcpDiscoverySpi();
 
-        if (clientMode && ((gridName.charAt(gridName.length() - 1) - '0') & 1) != 0)
+        if (clientMode && ((igniteInstanceName.charAt(igniteInstanceName.length() - 1) - '0') & 1) != 0)
             cfg.setClientMode(true);
         else
             disc.setMaxMissedClientHeartbeats(50);
@@ -158,8 +153,6 @@ private void doTestAlive() throws Exception {
             stopTempNodes();
 
             latch.await();
-
-            validateAlives();
         }
     }
 
@@ -199,55 +192,6 @@ private void awaitDiscovery(long nodesCnt) throws InterruptedException {
         }
     }
 
-    /**
-     * Validates that all node collections contain actual information.
-     */
-    @SuppressWarnings("SuspiciousMethodCalls")
-    private void validateAlives() {
-        for (Ignite g : alive) {
-            log.info("Validate node: " + g.name());
-
-            assertEquals("Unexpected nodes number for node: " + g.name(), PERM_NODES_CNT, g.cluster().nodes().size());
-        }
-
-        for (final Ignite g : alive) {
-            IgniteKernal k = (IgniteKernal)g;
-
-            GridDiscoveryManager discoMgr = k.context().discovery();
-
-            final Collection<ClusterNode> currTop = g.cluster().nodes();
-
-            long currVer = discoMgr.topologyVersion();
-
-            long startVer = discoMgr.localNode().order();
-
-            for (long v = currVer; v > currVer - GridDiscoveryManager.DISCOVERY_HISTORY_SIZE && v >= startVer; v--) {
-                F.forAll(discoMgr.aliveCacheNodes(null, new AffinityTopologyVersion(v)),
-                    new IgnitePredicate<ClusterNode>() {
-                        @Override public boolean apply(ClusterNode e) {
-                            return currTop.contains(e);
-                        }
-                    });
-
-                F.forAll(discoMgr.aliveRemoteCacheNodes(null, new AffinityTopologyVersion(v)),
-                    new IgnitePredicate<ClusterNode>() {
-                        @Override public boolean apply(ClusterNode e) {
-                            return currTop.contains(e) || g.cluster().localNode().equals(e);
-                        }
-                    });
-
-                GridCacheSharedContext<?, ?> ctx = k.context().cache().context();
-
-                ClusterNode oldest =
-                    ctx.discovery().oldestAliveCacheServerNode(new AffinityTopologyVersion(currVer));
-
-                assertNotNull(oldest);
-
-                assertTrue(currTop.contains(oldest));
-            }
-        }
-    }
-
     /**
      * Starts temporary nodes.
      *
@@ -293,4 +237,4 @@ private void stopTempNodes() {
             G.stop(g.name(), false);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
index ba8fa5b6e15a3..5de29106f0ee3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
@@ -54,10 +54,10 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA
     private static boolean binaryMarshallerEnabled;
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
-        if (gridName.equals(getTestGridName(1)))
+        if (igniteInstanceName.equals(getTestGridName(1)))
             cfg.setClientMode(true);
 
         if (binaryMarshallerEnabled)
@@ -160,7 +160,7 @@ private void doTestUseDefaultSuid(String first, String second, boolean fail) thr
                 if (fail)
                     fail("Node should not join");
             }
-            catch (Exception e) {
+            catch (Exception ignored) {
                 if (!fail)
                     fail("Node should join");
             }
@@ -215,7 +215,7 @@ private void doTestUseStrSerVer2(String first, String second, boolean fail) thro
                 if (fail)
                     fail("Node should not join");
             }
-            catch (Exception e) {
+            catch (Exception ignored) {
                 if (!fail)
                     fail("Node should join");
             }
@@ -346,8 +346,8 @@ private void testPreferIpV4Stack(boolean preferIpV4) throws Exception {
      */
     public static class RegularDiscovery extends GridDiscoveryManagerAttributesSelfTest {
         /** {@inheritDoc} */
-        @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-            IgniteConfiguration cfg = super.getConfiguration(gridName);
+        @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+            IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
             ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setForceServerMode(true);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerSelfTest.java
deleted file mode 100644
index c9179d44c0060..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerSelfTest.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.managers.discovery;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.NearCacheConfiguration;
-import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
-
-/**
- *
- */
-public abstract class GridDiscoveryManagerSelfTest extends GridCommonAbstractTest {
-    /** */
-    private static final String CACHE_NAME = "cache";
-
-    /** */
-    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        stopAllGrids();
-    }
-
-    /** {@inheritDoc} */
-    @SuppressWarnings("IfMayBeConditional")
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        CacheConfiguration ccfg1 = defaultCacheConfiguration();
-
-        ccfg1.setName(CACHE_NAME);
-
-        CacheConfiguration ccfg2 = defaultCacheConfiguration();
-
-        ccfg2.setName(null);
-
-        if (gridName.equals(getTestGridName(1)))
-            cfg.setClientMode(true);
-        else {
-            ccfg1.setNearConfiguration(null);
-            ccfg2.setNearConfiguration(null);
-
-            ccfg1.setCacheMode(PARTITIONED);
-            ccfg2.setCacheMode(PARTITIONED);
-
-            cfg.setCacheConfiguration(ccfg1, ccfg2);
-        }
-
-        TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi();
-
-        discoverySpi.setIpFinder(IP_FINDER);
-
-        cfg.setDiscoverySpi(discoverySpi);
-
-        return cfg;
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testHasNearCache() throws Exception {
-        IgniteKernal g0 = (IgniteKernal)startGrid(0); // PARTITIONED_ONLY cache.
-
-        AffinityTopologyVersion none = new AffinityTopologyVersion(-1);
-        AffinityTopologyVersion one = new AffinityTopologyVersion(1);
-        AffinityTopologyVersion two = new AffinityTopologyVersion(2, 2);
-        AffinityTopologyVersion three = new AffinityTopologyVersion(3);
-        AffinityTopologyVersion four = new AffinityTopologyVersion(4);
-        AffinityTopologyVersion five = new AffinityTopologyVersion(5);
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, none));
-        assertFalse(g0.context().discovery().hasNearCache(null, none));
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, one));
-        assertFalse(g0.context().discovery().hasNearCache(null, one));
-
-        IgniteKernal g1 = (IgniteKernal)startGrid(1); // NEAR_ONLY cache.
-
-        grid(1).createNearCache(null, new NearCacheConfiguration());
-
-        grid(1).createNearCache(CACHE_NAME, new NearCacheConfiguration());
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, one));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertFalse(g0.context().discovery().hasNearCache(null, one));
-        assertTrue(g0.context().discovery().hasNearCache(null, two));
-
-        assertTrue(g1.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertTrue(g1.context().discovery().hasNearCache(null, two));
-
-        IgniteKernal g2 = (IgniteKernal)startGrid(2); // PARTITIONED_ONLY cache.
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, one));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertFalse(g0.context().discovery().hasNearCache(null, one));
-        assertTrue(g0.context().discovery().hasNearCache(null, two));
-        assertTrue(g0.context().discovery().hasNearCache(null, three));
-
-        assertTrue(g1.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertTrue(g1.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertTrue(g1.context().discovery().hasNearCache(null, two));
-        assertTrue(g1.context().discovery().hasNearCache(null, three));
-
-        assertTrue(g2.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertTrue(g2.context().discovery().hasNearCache(null, three));
-
-        stopGrid(2);
-
-        // Wait all nodes are on version 4.
-        for (;;) {
-            if (F.forAll(
-                Ignition.allGrids(),
-                new IgnitePredicate<Ignite>() {
-                    @Override public boolean apply(Ignite ignite) {
-                        return ignite.cluster().topologyVersion() == 4;
-                    }
-                }))
-                break;
-
-            Thread.sleep(1000);
-        }
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, one));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, four));
-        assertFalse(g0.context().discovery().hasNearCache(null, one));
-        assertTrue(g0.context().discovery().hasNearCache(null, two));
-        assertTrue(g0.context().discovery().hasNearCache(null, three));
-        assertTrue(g0.context().discovery().hasNearCache(null, four));
-
-        assertTrue(g1.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertTrue(g1.context().discovery().hasNearCache(CACHE_NAME, four));
-        assertTrue(g1.context().discovery().hasNearCache(null, three));
-        assertTrue(g1.context().discovery().hasNearCache(null, four));
-
-        stopGrid(1);
-
-        // Wait all nodes are on version 5.
-        for (;;) {
-            if (F.forAll(
-                Ignition.allGrids(),
-                new IgnitePredicate<Ignite>() {
-                    @Override public boolean apply(Ignite ignite) {
-                        return ignite.cluster().topologyVersion() == 5;
-                    }
-                }))
-                break;
-
-            Thread.sleep(1000);
-        }
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, one));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, four));
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, five));
-
-        assertFalse(g0.context().discovery().hasNearCache(null, one));
-        assertTrue(g0.context().discovery().hasNearCache(null, two));
-        assertTrue(g0.context().discovery().hasNearCache(null, three));
-        assertTrue(g0.context().discovery().hasNearCache(null, four));
-        assertFalse(g0.context().discovery().hasNearCache(null, five));
-    }
-
-    /**
-     *
-     */
-    public static class RegularDiscovery extends GridDiscoveryManagerSelfTest {
-        /** {@inheritDoc} */
-        @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-            IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-            ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setForceServerMode(true);
-
-            return cfg;
-        }
-    }
-
-    /**
-     *
-     */
-    public static class ClientDiscovery extends GridDiscoveryManagerSelfTest {
-        // No-op.
-    }
-}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/IgniteTopologyPrintFormatSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/IgniteTopologyPrintFormatSelfTest.java
index 58992afbbd1a7..86ad4589eaa85 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/IgniteTopologyPrintFormatSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/IgniteTopologyPrintFormatSelfTest.java
@@ -50,16 +50,16 @@ public class IgniteTopologyPrintFormatSelfTest extends GridCommonAbstractTest {
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
         TcpDiscoverySpi disc = new TcpDiscoverySpi();
         disc.setIpFinder(IP_FINDER);
 
-        if (gridName.endsWith("client"))
+        if (igniteInstanceName.endsWith("client"))
             cfg.setClientMode(true);
 
-        if (gridName.endsWith("client_force_server")) {
+        if (igniteInstanceName.endsWith("client_force_server")) {
             cfg.setClientMode(true);
             disc.setForceServerMode(true);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index b28619cd91ed5..985dddb93bf4d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -42,7 +42,6 @@
 import org.apache.ignite.internal.managers.deployment.GridDeploymentManagerStopSelfTest;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerAliveCacheSelfTest;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerAttributesSelfTest;
-import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerSelfTest;
 import org.apache.ignite.internal.managers.discovery.IgniteTopologyPrintFormatSelfTest;
 import org.apache.ignite.internal.managers.events.GridEventStorageManagerSelfTest;
 import org.apache.ignite.internal.managers.swapspace.GridSwapSpaceManagerSelfTest;
@@ -111,8 +110,6 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridDiscoveryManagerAttributesSelfTest.RegularDiscovery.class);
         suite.addTestSuite(GridDiscoveryManagerAttributesSelfTest.ClientDiscovery.class);
         suite.addTestSuite(GridDiscoveryManagerAliveCacheSelfTest.class);
-        suite.addTestSuite(GridDiscoveryManagerSelfTest.RegularDiscovery.class);
-        suite.addTestSuite(GridDiscoveryManagerSelfTest.ClientDiscovery.class);
         suite.addTestSuite(GridDiscoveryEventSelfTest.class);
         suite.addTestSuite(GridPortProcessorSelfTest.class);
         suite.addTestSuite(GridHomePathSelfTest.class);

From 6775f646df127f5cdbabf7a154b65856f38afa1e Mon Sep 17 00:00:00 2001
From: Dmitriy Shabalin <dshabalin@gridgain.com>
Date: Wed, 22 Mar 2017 10:37:05 +0700
Subject: [PATCH 202/446] IGNITE-4686 Added ability to group registered users
 in admin panel.

(cherry picked from commit 827befb)
---
 modules/web-console/frontend/app/app.js       |   1 +
 .../form-field-datepicker.pug                 |  12 +-
 .../list-of-registered-users/index.js         |   2 +
 .../list-of-registered-users.column-defs.js   |  48 ++---
 .../list-of-registered-users.controller.js    | 193 +++++++++++++-----
 .../list-of-registered-users.scss             |  28 +++
 .../list-of-registered-users.tpl.pug          |  50 +++--
 .../ui-grid-header/ui-grid-header.scss        |   6 +
 .../ui-grid-header/ui-grid-header.tpl.pug     |   4 +-
 .../ui-grid-settings/ui-grid-settings.scss    |  10 +
 .../frontend/app/primitives/badge/index.scss  |  36 ++++
 .../frontend/app/primitives/index.js          |  19 ++
 .../frontend/app/primitives/tabs/index.scss   |  73 +++++++
 .../frontend/public/stylesheets/style.scss    |   2 +
 14 files changed, 389 insertions(+), 95 deletions(-)
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.scss
 create mode 100644 modules/web-console/frontend/app/primitives/badge/index.scss
 create mode 100644 modules/web-console/frontend/app/primitives/index.js
 create mode 100644 modules/web-console/frontend/app/primitives/tabs/index.scss

diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 1e21d247c060a..26d3ad586c2e7 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -16,6 +16,7 @@
  */
 
 import '../public/stylesheets/style.scss';
+import '../app/primitives';
 import './components/ui-grid-header/ui-grid-header.scss';
 import './components/ui-grid-settings/ui-grid-settings.scss';
 import './components/form-field-datepicker/form-field-datepicker.scss';
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
index c9d382cb36cc4..d70476f373709 100644
--- a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
+++ b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin ignite-form-field-datepicker(label, model, name, disabled, required, placeholder, tip)
+mixin ignite-form-field-datepicker(label, model, name, mindate, maxdate, disabled, required, placeholder, tip)
     mixin form-field-input()
         input.form-control(
             id=`{{ ${name} }}Input`
@@ -30,8 +30,10 @@ mixin ignite-form-field-datepicker(label, model, name, disabled, required, place
             bs-datepicker
             data-date-format='MMM yyyy'
             data-start-view='1'
-            data-min-view='1' 
-            data-max-date='today'
+            data-min-view='1'
+
+            data-min-date=mindate ? `{{ ${mindate} }}` : false
+            data-max-date=maxdate ? `{{ ${maxdate} }}` : `today`
 
             data-container='body > .wrapper'
 
@@ -43,7 +45,9 @@ mixin ignite-form-field-datepicker(label, model, name, disabled, required, place
         )&attributes(attributes.attributes)
 
     .ignite-form-field
-        +ignite-form-field__label(label, name, required)
+        if name
+            +ignite-form-field__label(label, name, required)
+
         .ignite-form-field__control
             if tip
                 i.tipField.icon-help(bs-tooltip='' data-title=tip)
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/index.js b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
index 22a89da3b6a24..4e5061f352e0a 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/index.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import './list-of-registered-users.scss';
+
 import templateUrl from './list-of-registered-users.tpl.pug';
 import controller from './list-of-registered-users.controller';
 
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
index e6ba842aa055b..e859acf7d0ec6 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
@@ -49,32 +49,32 @@ const ACTIONS_TEMPLATE = `
 const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
 
 export default [
-    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 65, width: 65, enableFiltering: false, enableSorting: false, pinnedLeft: true},
-    {displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by name...' }, pinnedLeft: true},
-    {displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
-    {displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 160, enableFiltering: true},
-    {displayName: 'Country', categoryDisplayName: 'Country', field: 'countryCode', minWidth: 80, enableFiltering: true},
-    {displayName: 'Last login', categoryDisplayName: 'Last login', field: 'lastLogin', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 105, width: 105, enableFiltering: false, visible: false},
-    {displayName: 'Last activity', categoryDisplayName: 'Last activity', field: 'lastActivity', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 105, width: 105, enableFiltering: false, visible: true, sort: { direction: 'desc', priority: 0 }},
+    {name: 'actions', displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false},
+    {name: 'user', displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by name...' }},
+    {name: 'email', displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
+    {name: 'company', displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by company...' }},
+    {name: 'country', displayName: 'Country', categoryDisplayName: 'Country', field: 'countryCode', minWidth: 80, enableFiltering: true, filter: { placeholder: 'Filter by country...' }},
+    {name: 'lastlogin', displayName: 'Last login', categoryDisplayName: 'Last login', field: 'lastLogin', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 105, width: 105, enableFiltering: false, visible: false},
+    {name: 'lastactivity', displayName: 'Last activity', categoryDisplayName: 'Last activity', field: 'lastActivity', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 115, width: 115, enableFiltering: false, visible: true, sort: { direction: 'desc', priority: 0 }},
     // Configurations
-    {displayName: 'Clusters count', categoryDisplayName: 'Configurations', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: 'counters.clusters', type: 'number', headerTooltip: 'Clusters count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
-    {displayName: 'Models count', categoryDisplayName: 'Configurations', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: 'counters.models', type: 'number', headerTooltip: 'Models count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
-    {displayName: 'Caches count', categoryDisplayName: 'Configurations', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: 'counters.caches', type: 'number', headerTooltip: 'Caches count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
-    {displayName: 'IGFS count', categoryDisplayName: 'Configurations', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: 'counters.igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
+    {name: 'cfg_clusters', displayName: 'Clusters count', categoryDisplayName: 'Configurations', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: 'counters.clusters', type: 'number', headerTooltip: 'Clusters count', minWidth: 55, width: 55, enableFiltering: false, visible: false},
+    {name: 'cfg_models', displayName: 'Models count', categoryDisplayName: 'Configurations', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: 'counters.models', type: 'number', headerTooltip: 'Models count', minWidth: 55, width: 55, enableFiltering: false, visible: false},
+    {name: 'cfg_caches', displayName: 'Caches count', categoryDisplayName: 'Configurations', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: 'counters.caches', type: 'number', headerTooltip: 'Caches count', minWidth: 55, width: 55, enableFiltering: false, visible: false},
+    {name: 'cfg_igfs', displayName: 'IGFS count', categoryDisplayName: 'Configurations', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: 'counters.igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 55, width: 55, enableFiltering: false, visible: false},
     // Activities Total
-    {displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Total count of configuration usages', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Total count of queries usages', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Total count of demo startup', minWidth: 60, width: 60, enableFiltering: false},
-    {displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Total count of agent downloads', minWidth: 55, width: 55, enableFiltering: false},
-    {displayName: 'Starts', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Total count of agent startup', minWidth: 60, width: 60, enableFiltering: false},
+    {name: 'cfg', displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Total count of configuration usages', minWidth: 55, width: 55, enableFiltering: false},
+    {name: 'qry', displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Total count of queries usages', minWidth: 55, width: 55, enableFiltering: false},
+    {name: 'demo', displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Total count of demo startup', minWidth: 65, width: 65, enableFiltering: false},
+    {name: 'dnld', displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Total count of agent downloads', minWidth: 55, width: 55, enableFiltering: false},
+    {name: 'starts', displayName: 'Starts', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Total count of agent startup', minWidth: 65, width: 65, enableFiltering: false},
     // Activities Configuration
-    {displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration clusters', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration model', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', headerTooltip: 'Configuration caches', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'IGFS', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/igfs"] || 0', type: 'number', headerTooltip: 'Configuration IGFS', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', headerTooltip: 'Configuration summary', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {name: 'clusters', displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration clusters', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'model', displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration model', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'caches', displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', headerTooltip: 'Configuration caches', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'igfs', displayName: 'IGFS', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/igfs"] || 0', type: 'number', headerTooltip: 'Configuration IGFS', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'summary', displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', headerTooltip: 'Configuration summary', minWidth: 55, width: 80, enableFiltering: false, visible: false},
     // Activities Queries
-    {displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', headerTooltip: 'Query executions', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', headerTooltip: 'Query explain executions', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', headerTooltip: 'Scan query executions', minWidth: 50, width: 80, enableFiltering: false, visible: false}
+    {name: 'execute', displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', headerTooltip: 'Query executions', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'explain', displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', headerTooltip: 'Query explain executions', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'scan', displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', headerTooltip: 'Scan query executions', minWidth: 55, width: 80, enableFiltering: false, visible: false}
 ];
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 54971b142b014..acf76faf44918 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -30,34 +30,21 @@ const rowTemplate = `<div
   ui-grid-cell/>`;
 
 export default class IgniteListOfRegisteredUsersCtrl {
-    static $inject = ['$scope', '$state', '$filter', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
+    static $inject = ['$scope', '$state', '$filter', 'User', 'uiGridGroupingConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
 
-    constructor($scope, $state, $filter, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
+    constructor($scope, $state, $filter, User, uiGridGroupingConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
         const $ctrl = this;
 
-        const companySelectOptions = [];
-        const countrySelectOptions = [];
-
         const dtFilter = $filter('date');
 
+        $ctrl.groupBy = 'user';
+
         $ctrl.params = {
-            startDate: new Date()
+            startDate: new Date(),
+            endDate: new Date()
         };
 
-        const columnCompany = _.find(columnDefs, { displayName: 'Company' });
-        const columnCountry = _.find(columnDefs, { displayName: 'Country' });
-
-        columnCompany.filter = {
-            selectOptions: companySelectOptions,
-            type: uiGridConstants.filter.SELECT,
-            condition: uiGridConstants.filter.EXACT
-        };
-
-        columnCountry.filter = {
-            selectOptions: countrySelectOptions,
-            type: uiGridConstants.filter.SELECT,
-            condition: uiGridConstants.filter.EXACT
-        };
+        $ctrl.uiGridGroupingConstants = uiGridGroupingConstants;
 
         const becomeUser = (user) => {
             AdminData.becomeUser(user._id)
@@ -105,6 +92,11 @@ export default class IgniteListOfRegisteredUsersCtrl {
             return renderableRows;
         };
 
+        $ctrl._userGridOptions = {
+            columnDefs,
+            categories
+        };
+
         $ctrl.gridOptions = {
             data: [],
             columnVirtualizationThreshold: 30,
@@ -134,19 +126,6 @@ export default class IgniteListOfRegisteredUsersCtrl {
             }
         };
 
-        const usersToFilterOptions = (column) => {
-            return _.sortBy(
-                _.map(
-                    _.groupBy($ctrl.gridOptions.data, (usr) => {
-                        const fld = usr[column];
-
-                        return _.isNil(fld) ? fld : fld.toUpperCase();
-                    }),
-                    (arr, value) => ({label: `${_.head(arr)[column] || 'Not set'} (${arr.length})`, value})
-                ),
-                'value');
-        };
-
         /**
          * @param {{startDate: number, endDate: number}} params
          */
@@ -154,31 +133,32 @@ export default class IgniteListOfRegisteredUsersCtrl {
             AdminData.loadUsers(params)
                 .then((data) => $ctrl.gridOptions.data = data)
                 .then((data) => {
-                    companySelectOptions.length = 0;
-                    countrySelectOptions.length = 0;
-
-                    companySelectOptions.push(...usersToFilterOptions('company'));
-                    countrySelectOptions.push(...usersToFilterOptions('countryCode'));
-
                     this.gridApi.grid.refresh();
 
+                    this.companies = _.values(_.groupBy(data, (b) => b.company.toLowerCase()));
+                    this.countries = _.values(_.groupBy(data, (b) => b.countryCode));
+
                     return data;
-                })
-                .then((data) => $ctrl.adjustHeight(data.length));
+                });
+        };
+
+        const fitlerDates = (sdt, edt) => {
+            $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(sdt, 'yyyy_MM')}.csv`;
+
+            const startDate = Date.UTC(sdt.getFullYear(), sdt.getMonth(), 1);
+            const endDate = Date.UTC(edt.getFullYear(), edt.getMonth() + 1, 1);
+
+            reloadUsers({ startDate, endDate });
         };
 
         $scope.$watch(() => $ctrl.params.companiesExclude, () => {
             $ctrl.gridApi.grid.refreshRows();
         });
 
-        $scope.$watch(() => $ctrl.params.startDate, (dt) => {
-            $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(dt, 'yyyy_MM')}.csv`;
+        $scope.$watch(() => $ctrl.params.startDate, (sdt) => fitlerDates(sdt, $ctrl.params.endDate));
+        $scope.$watch(() => $ctrl.params.endDate, (edt) => fitlerDates($ctrl.params.startDate, edt));
 
-            const startDate = Date.UTC(dt.getFullYear(), dt.getMonth(), 1);
-            const endDate = Date.UTC(dt.getFullYear(), dt.getMonth() + 1, 1);
-
-            reloadUsers({ startDate, endDate });
-        });
+        $scope.$watch(() => $ctrl.gridApi.grid.getVisibleRows().length, (length) => $ctrl.adjustHeight(length >= 20 ? 20 : length));
     }
 
     adjustHeight(rows) {
@@ -235,4 +215,121 @@ export default class IgniteListOfRegisteredUsersCtrl {
     exportCsv() {
         this.gridApi.exporter.csvExport('visible', 'visible');
     }
+
+    groupByUser() {
+        this.groupBy = 'user';
+
+        this.gridApi.grouping.clearGrouping();
+
+        this.gridOptions.categories = this._userGridOptions.categories;
+        this.gridOptions.columnDefs = this._userGridOptions.columnDefs;
+    }
+
+    groupByCompany() {
+        this.groupBy = 'company';
+
+        this.gridApi.grouping.clearGrouping();
+        this.gridApi.grouping.groupColumn('company');
+        this.gridApi.grouping.aggregateColumn('user', this.uiGridGroupingConstants.aggregation.COUNT);
+
+        if (this._companyGridOptions) {
+            this.gridOptions.categories = this._companyGridOptions.categories;
+            this.gridOptions.columnDefs = this._companyGridOptions.columnDefs;
+
+            return;
+        }
+
+        const _categories = _.cloneDeep(categories);
+        const _columnDefs = _.cloneDeep(columnDefs);
+
+        // Cut company category;
+        const company = _categories.splice(3, 1)[0];
+
+        // Hide Actions category;
+        _categories.splice(0, 1);
+
+        _.forEach(_.filter(_columnDefs, {displayName: 'Actions'}), (col) => {
+            col.visible = false;
+        });
+
+        // Add company as first column;
+        _categories.unshift(company);
+
+        _.forEach(_columnDefs, (col) => {
+            col.enableSorting = true;
+
+            if (col.type !== 'number')
+                return;
+
+            col.treeAggregationType = this.uiGridGroupingConstants.aggregation.SUM;
+            col.customTreeAggregationFinalizerFn = (agg) => agg.rendered = agg.value;
+        });
+
+        // Set grouping to last activity column
+        const lastactivity = _.find(_columnDefs, { name: 'lastactivity' });
+
+        if (_.nonNil(lastactivity)) {
+            lastactivity.treeAggregationType = this.uiGridGroupingConstants.aggregation.MAX;
+            lastactivity.customTreeAggregationFinalizerFn = (agg) => agg.rendered = agg.value;
+        }
+
+        this._companyGridOptions = {
+            categories: this.gridOptions.categories = _categories,
+            columnDefs: this.gridOptions.columnDefs = _columnDefs
+        };
+    }
+
+    groupByCountry() {
+        this.groupBy = 'country';
+
+        this.gridApi.grouping.clearGrouping();
+        this.gridApi.grouping.groupColumn('country');
+        this.gridApi.grouping.aggregateColumn('user', this.uiGridGroupingConstants.aggregation.COUNT);
+
+        if (this._countryGridOptions) {
+            this.gridOptions.categories = this._countryGridOptions.categories;
+            this.gridOptions.columnDefs = this._countryGridOptions.columnDefs;
+
+            return;
+        }
+
+        const _categories = _.cloneDeep(categories);
+        const _columnDefs = _.cloneDeep(columnDefs);
+
+        // Cut country category;
+        const country = _categories.splice(4, 1)[0];
+
+        // Hide Actions category;
+        _categories.splice(0, 1);
+
+        _.forEach(_.filter(_columnDefs, {displayName: 'Actions'}), (col) => {
+            col.visible = false;
+        });
+
+        // Add company as first column;
+        _categories.unshift(country);
+
+        _.forEach(_columnDefs, (col) => {
+            col.enableSorting = true;
+
+            if (col.type !== 'number')
+                return;
+
+            col.treeAggregationType = this.uiGridGroupingConstants.aggregation.SUM;
+            col.customTreeAggregationFinalizerFn = (agg) => agg.rendered = agg.value;
+        });
+
+        // Set grouping to last activity column
+        const lastactivity = _.find(_columnDefs, { name: 'lastactivity' });
+
+        if (_.nonNil(lastactivity)) {
+            lastactivity.treeAggregationType = this.uiGridGroupingConstants.aggregation.MAX;
+            lastactivity.customTreeAggregationFinalizerFn = (agg) => agg.rendered = agg.value;
+        }
+
+        this._countryGridOptions = {
+            categories: this.gridOptions.categories = _categories,
+            columnDefs: this.gridOptions.columnDefs = _columnDefs
+        };
+    }
 }
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.scss b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.scss
new file mode 100644
index 0000000000000..8059d7084102c
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.scss
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.list-of-registered-users {
+  & > a {
+    display: inline-block;
+    margin: 10px;
+    margin-left: 0;
+
+    &.active {
+      font-weight: bold;
+    }
+  }
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
index 52975b98dc0ee..ec4b4fda0543d 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
@@ -34,25 +34,39 @@ mixin grid-settings()
         li
             a(ng-click='$hide()') Close
 
-.panel.panel-default
-    .panel-heading.ui-grid-settings
-        +grid-settings
-        label Total users: 
-            strong {{ $ctrl.gridOptions.data.length }}&nbsp;&nbsp;&nbsp;
-        label Showing users:
-            strong {{ $ctrl.gridApi.grid.getVisibleRows().length }}
-            sub(ng-show='users.length === $ctrl.gridApi.grid.getVisibleRows().length') all
-
-        form.pull-right(ng-form=form novalidate)
-            -var form = 'admin'
 
-            button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
+.list-of-registered-users
+    ul.tabs
+        li(role='presentation' ng-class='{ active: $ctrl.groupBy === "user" }') 
+            a(ng-click='$ctrl.groupByUser()') 
+                span Users
+                span.badge {{ $ctrl.gridOptions.data.length }}
+        li(role='presentation' ng-class='{ active: $ctrl.groupBy === "company" }')
+            a(ng-click='$ctrl.groupByCompany()') 
+                span Companies
+                span.badge {{ $ctrl.companies.length }}
+        li(role='presentation' ng-class='{ active: $ctrl.groupBy === "country" }')
+            a(ng-click='$ctrl.groupByCountry()')
+                span Countries
+                span.badge {{ $ctrl.countries.length }}
+
+    .panel.panel-default
+        .panel-heading.ui-grid-settings
+            +grid-settings
+            label(ng-show='$ctrl.groupBy === "user"') Showing users:&nbsp;
+                strong {{ $ctrl.gridApi.grid.getVisibleRows().length }}
+                sub(ng-show='users.length === $ctrl.gridApi.grid.getVisibleRows().length') all
+
+            -var form = 'admin'
+            form.pull-right(name=form novalidate)
+                button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
 
-            .ui-grid-settings-dateperiod
-                +ignite-form-field-datepicker('Period:', '$ctrl.params.startDate', '"period"')
+                .ui-grid-settings-dateperiod
+                    +ignite-form-field-datepicker('Period:', '$ctrl.params.startDate', '"period"', null, '$ctrl.params.endDate')
+                    +ignite-form-field-datepicker('Period:', '$ctrl.params.endDate', null, '$ctrl.params.startDate', null)
 
-            .ui-grid-settings-filter
-                +ignite-form-field-text('Exclude:', '$ctrl.params.companiesExclude', '"exclude"', false, false, 'Exclude by company name...')
+                .ui-grid-settings-filter
+                    +ignite-form-field-text('Exclude:', '$ctrl.params.companiesExclude', '"exclude"', false, false, 'Exclude by company name...')
 
-    .panel-collapse
-        .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-exporter ui-grid-pinning)
+        .panel-collapse
+            .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-exporter ui-grid-pinning ui-grid-grouping)
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
index c6e7bdf241e98..4530c026bb0f2 100644
--- a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
@@ -28,6 +28,12 @@
         height: 30px;
     }
 
+    .ui-grid-header-cell {
+        .ui-grid-cell-contents > span:not(.ui-grid-header-cell-label) {
+            right: 3px;
+        }
+    }
+
     .ui-grid-header-cell [role="columnheader"] {
         display: flex;
         
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug
index 7e44d94671452..9b14fca7f41d6 100644
--- a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug
+++ b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug
@@ -20,8 +20,10 @@
             .ui-grid-header-canvas
                 .ui-grid-header-cell-wrapper(ng-style='colContainer.headerCellWrapperStyle()')
                     .ui-grid-header-cell-row(role='row')
-                        .ui-grid-header-span.ui-grid-header-cell.ui-grid-clearfix(ng-repeat='cat in grid.options.categories')
+                        .ui-grid-header-span.ui-grid-header-cell.ui-grid-clearfix.ui-grid-category(ng-repeat='cat in grid.options.categories', ng-if='cat.visible && \
+                        (colContainer.renderedColumns | uiGridSubcategories: cat.name).length > 0')
                             div(ng-show='(colContainer.renderedColumns|uiGridSubcategories:cat.name).length > 1')
                                 .ui-grid-cell-contents {{ cat.name }}
                             .ui-grid-header-cell-row
                                 .ui-grid-header-cell.ui-grid-clearfix(ng-repeat='col in (colContainer.renderedColumns|uiGridSubcategories:cat.name) track by col.uid' ui-grid-header-cell='' col='col' render-index='$index')
+                        .ui-grid-header-cell.ui-grid-clearfix(ng-if='col.colDef.name === "treeBaseRowHeaderCol"' ng-repeat='col in colContainer.renderedColumns track by col.uid' ui-grid-header-cell='' col='col' render-index='$index')
diff --git a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
index 24f4d9b7efb7f..d0a31f05f464c 100644
--- a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
@@ -129,6 +129,16 @@
                     width: 60%;
                 }
             }
+
+            &:nth-child(2) {
+                float: left;
+
+                width: 100px;
+
+                .ignite-form-field__control {
+                    width: 100%;
+                }
+            }
         }
     }
 }
diff --git a/modules/web-console/frontend/app/primitives/badge/index.scss b/modules/web-console/frontend/app/primitives/badge/index.scss
new file mode 100644
index 0000000000000..837ab5bc9cad6
--- /dev/null
+++ b/modules/web-console/frontend/app/primitives/badge/index.scss
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@import '../../../public/stylesheets/variables';
+
+.badge {
+  display: inline-block;
+  min-width: 26px;
+  height: 18px;
+
+  padding: 2px 9px;
+
+  border-radius: 9px;
+
+  color: white;
+  font-family: Roboto;
+  font-size: 12px;
+  text-align: center;
+  line-height: 12px;
+
+  background-color: $brand-primary;
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/primitives/index.js b/modules/web-console/frontend/app/primitives/index.js
new file mode 100644
index 0000000000000..7940f7a788249
--- /dev/null
+++ b/modules/web-console/frontend/app/primitives/index.js
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './badge/index.scss';
+import './tabs/index.scss';
diff --git a/modules/web-console/frontend/app/primitives/tabs/index.scss b/modules/web-console/frontend/app/primitives/tabs/index.scss
new file mode 100644
index 0000000000000..eed88cb0480c4
--- /dev/null
+++ b/modules/web-console/frontend/app/primitives/tabs/index.scss
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@import '../../../public/stylesheets/variables';
+
+ul.tabs {
+    $width: auto;
+    $height: 40px;
+    $offset-vertical: 11px;
+    $offset-horizontal: 25px;
+    $font-size: 14px;
+
+    list-style: none;
+
+    padding-left: 0;
+    border-bottom: 1px solid $nav-tabs-border-color;
+
+    li {
+        position: relative;
+        top: 1px;
+
+        display: inline-block;
+
+        border-bottom: 5px solid transparent;
+
+        a {
+            display: inline-block;
+            width: $width;
+            height: $height;
+
+            padding: $offset-vertical $offset-horizontal;
+
+            color: $text-color;
+            font-size: $font-size;
+            text-align: center;
+            line-height: $height - 2*$offset-vertical;
+
+            &:hover {
+              text-decoration: none;
+            }
+
+            .badge {
+              margin-left: $offset-vertical;
+            }
+        }
+
+        &.active {
+            border-color: $brand-primary;
+        }
+
+        &:not(.active):hover {
+            border-color: lighten($brand-primary, 25%);
+        }
+
+        & + li {
+            margin-left: 45px;
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index a07472e8840a4..2f4966fc8170c 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -2302,6 +2302,8 @@ html,body,.splash-screen {
 
 .admin-page {
     .panel-heading {
+        height: 38px;
+
         border-bottom: 0;
         padding-bottom: 0;
 

From 7d94963251cc66823883d760668c2e2b31574bf1 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 22 Mar 2017 18:20:32 +0700
Subject: [PATCH 203/446] IGNITE-4659 Fixed error page.

(cherry picked from commit 50de012)
---
 .../web-console/frontend/app/modules/states/errors.state.js   | 4 ++--
 modules/web-console/frontend/views/{403.pug => 403.tpl.pug}   | 0
 modules/web-console/frontend/views/{404.pug => 404.tpl.pug}   | 0
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename modules/web-console/frontend/views/{403.pug => 403.tpl.pug} (100%)
 rename modules/web-console/frontend/views/{404.pug => 404.tpl.pug} (100%)

diff --git a/modules/web-console/frontend/app/modules/states/errors.state.js b/modules/web-console/frontend/app/modules/states/errors.state.js
index e219132beb34e..e816ff80224ae 100644
--- a/modules/web-console/frontend/app/modules/states/errors.state.js
+++ b/modules/web-console/frontend/app/modules/states/errors.state.js
@@ -16,8 +16,8 @@
  */
 
 import angular from 'angular';
-import templateNotFoundPage from 'views/404.pug';
-import templateNotAuthorizedPage from 'views/403.pug';
+import templateNotFoundPage from 'views/404.tpl.pug';
+import templateNotAuthorizedPage from 'views/403.tpl.pug';
 
 angular
     .module('ignite-console.states.errors', [
diff --git a/modules/web-console/frontend/views/403.pug b/modules/web-console/frontend/views/403.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/403.pug
rename to modules/web-console/frontend/views/403.tpl.pug
diff --git a/modules/web-console/frontend/views/404.pug b/modules/web-console/frontend/views/404.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/404.pug
rename to modules/web-console/frontend/views/404.tpl.pug

From eb837c7853ffc68e5a3f690e985407e603e3f955 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 22 Mar 2017 18:21:03 +0700
Subject: [PATCH 204/446] IGNITE-4854 Fixed project structure button on summary
 page under firefox.

(cherry picked from commit 117e18e)
---
 .../web-console/frontend/views/configuration/summary.tpl.pug  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/frontend/views/configuration/summary.tpl.pug b/modules/web-console/frontend/views/configuration/summary.tpl.pug
index 9fdd0de3f3097..13d6e3a2c79da 100644
--- a/modules/web-console/frontend/views/configuration/summary.tpl.pug
+++ b/modules/web-console/frontend/views/configuration/summary.tpl.pug
@@ -43,8 +43,8 @@ mixin hard-link(ref, txt)
                             i.fa.fa-fw.fa-download(ng-hide='isPrepareDownloading')
                             i.fa.fa-fw.fa-refresh.fa-spin(ng-show='isPrepareDownloading')
                             span.tipLabel Download project
-                    button.btn.btn-primary(bs-tooltip='' data-title='Preview generated project structure' data-placement='bottom')
-                        div(bs-popover data-template-url='{{ ctrl.summaryProjectStructureTemplateUrl }}', data-placement='bottom', data-trigger='click' data-auto-close='true')
+                    span(bs-tooltip='' data-title='Preview generated project structure' data-placement='bottom')
+                        button.btn.btn-primary(bs-popover data-template-url='{{ ctrl.summaryProjectStructureTemplateUrl }}', data-placement='bottom', data-trigger='click' data-auto-close='true')
                             i.fa.fa-sitemap
                             label.tipLabel Project structure
                     button.btn.btn-primary(id='proprietary-jdbc-drivers' ng-if='downloadJdbcDriversVisible()' ng-click='downloadJdbcDrivers()' bs-tooltip='' data-title='Open proprietary JDBC drivers download pages' data-placement='bottom') Download JDBC drivers

From ec7b9d848274b229b69dc7b8a20902654f719c44 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 23 Mar 2017 09:32:38 +0700
Subject: [PATCH 205/446] IGNITE-4855 Fixed error on switching between
 notebooks.

(cherry picked from commit 6a148e2)
---
 .../frontend/app/modules/sql/sql.module.js           | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/sql/sql.module.js b/modules/web-console/frontend/app/modules/sql/sql.module.js
index 670e4f45090d9..79614c2c9c73d 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.module.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.module.js
@@ -34,9 +34,7 @@ angular.module('ignite-console.sql', [
                 .state('base.sql', {
                     url: '/queries',
                     abstract: true,
-                    template: '<ui-view></ui-view>',
-                    controller,
-                    controllerAs: '$ctrl'
+                    template: '<ui-view></ui-view>'
                 })
                 .state('base.sql.notebook', {
                     url: '/notebook/{noteId}',
@@ -44,7 +42,9 @@ angular.module('ignite-console.sql', [
                     onEnter: AclRoute.checkAccess('query'),
                     metaTags: {
                         title: 'Query notebook'
-                    }
+                    },
+                    controller,
+                    controllerAs: '$ctrl'
                 })
                 .state('base.sql.demo', {
                     url: '/demo',
@@ -52,7 +52,9 @@ angular.module('ignite-console.sql', [
                     onEnter: AclRoute.checkAccess('query'),
                     metaTags: {
                         title: 'SQL demo'
-                    }
+                    },
+                    controller,
+                    controllerAs: '$ctrl'
                 });
         }]
     )

From 36f7621b6eaa710d3b1eba7fddd0dfe92e11133e Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 23 Mar 2017 10:57:03 +0700
Subject: [PATCH 206/446] IGNITE-4830 Fixed error ui.

(cherry picked from commit 48e78a9)
---
 .../frontend/public/stylesheets/style.scss      |  3 +--
 .../web-console/frontend/views/sql/sql.tpl.pug  | 17 +++++++++--------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index 2f4966fc8170c..8dbf123f17688 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -752,8 +752,7 @@ button.form-control {
         .error {
             padding: 10px 10px;
 
-            text-align: center;
-            color: $brand-primary;
+            text-align: left;
         }
 
         .empty {
diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug
index 701ee143af0ca..dcfc53137f616 100644
--- a/modules/web-console/frontend/views/sql/sql.tpl.pug
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -110,10 +110,9 @@ mixin query-settings
                     span Allow non-collocated joins
             .row(ng-if='enforceJoinOrderAvailable(paragraph)')
                 label.tipLabel(bs-tooltip data-placement='bottom' data-title='Enforce join order of tables in the query.<br/>\
-                    If <b>set</b> then query optimizer will not reorder tables in join.<br/>\
-                    <b>NOTE:</b> It is not recommended to enable this property until you are sure that\
-                    your indexes and the query itself are correct and tuned as much as possible but\
-                    query optimizer still produces wrong join order.' data-trigger='hover')
+                    If <b>set</b>, then query optimizer will not reorder tables within join.<br/>\
+                    <b>NOTE:</b> It is not recommended to enable this property unless you have verified that\
+                    indexes are not selected in optimal order.' data-trigger='hover')
                     input(type='checkbox' ng-model='paragraph.enforceJoinOrder')
                     span Enforce join order
 
@@ -247,16 +246,18 @@ mixin paragraph-query
             .pull-right
                 +query-settings
         .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
-            .error(ng-switch-when='error') Error: {{paragraph.error.message}}
+            .error(ng-switch-when='error')
+                label Error: {{paragraph.error.message}}
+                br
+                a(ng-show='paragraph.resultType() === "error"' ng-click='showStackTrace(paragraph)') Show more
             .empty(ng-switch-when='empty') Result set is empty
             .table(ng-switch-when='table')
                 +table-result-heading-query
                 +table-result-body
             .chart(ng-switch-when='chart')
                 +chart-result
-            .footer.clearfix
-                a.pull-left(ng-show='paragraph.resultType() === "error"' ng-click='showStackTrace(paragraph)') Show error details
-                a.pull-left(ng-show='paragraph.resultType() !== "error"' ng-click='showResultQuery(paragraph)') Show query
+            .footer.clearfix(ng-show='paragraph.resultType() !== "error"')
+                a.pull-left(ng-click='showResultQuery(paragraph)') Show query
 
                 -var nextVisibleCondition = 'paragraph.resultType() !== "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
 

From 1f3b2fcd003c1f084874d5c421953da0a7cd02cb Mon Sep 17 00:00:00 2001
From: Valentin Kulichenko <valentin.lulichenko@gmail.com>
Date: Mon, 27 Mar 2017 18:12:17 -0700
Subject: [PATCH 207/446] IGNITE-4802 - Separate thread pool for managed
 services

---
 .../configuration/IgniteConfiguration.java    |  30 ++++
 .../ignite/internal/GridKernalContext.java    |   7 +
 .../internal/GridKernalContextImpl.java       |  11 ++
 .../apache/ignite/internal/IgniteKernal.java  |   2 +
 .../apache/ignite/internal/IgnitionEx.java    |  16 +++
 .../managers/communication/GridIoManager.java |   2 +
 .../managers/communication/GridIoPolicy.java  |   5 +-
 .../processors/pool/PoolProcessor.java        |  10 +-
 .../processors/service/GridServiceProxy.java  |  10 ++
 .../task/GridTaskThreadContextKey.java        |   7 +-
 .../processors/task/GridTaskWorker.java       |  16 ++-
 .../services/ServiceThreadPoolSelfTest.java   | 133 ++++++++++++++++++
 .../junits/GridTestKernalContext.java         |   1 +
 .../testsuites/IgniteKernalSelfTestSuite.java |   2 +
 14 files changed, 245 insertions(+), 7 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/services/ServiceThreadPoolSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index dcd8a801aa808..b240446cd5ae0 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -53,6 +53,7 @@
 import org.apache.ignite.plugin.PluginProvider;
 import org.apache.ignite.plugin.segmentation.SegmentationPolicy;
 import org.apache.ignite.plugin.segmentation.SegmentationResolver;
+import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.spi.checkpoint.CheckpointSpi;
 import org.apache.ignite.spi.checkpoint.noop.NoopCheckpointSpi;
@@ -233,6 +234,9 @@ public class IgniteConfiguration {
     /** Public pool size. */
     private int pubPoolSize = DFLT_PUBLIC_THREAD_CNT;
 
+    /** Service pool size. */
+    private Integer svcPoolSize;
+
     /** Async Callback pool size. */
     private int callbackPoolSize = DFLT_PUBLIC_THREAD_CNT;
 
@@ -561,6 +565,7 @@ public IgniteConfiguration(IgniteConfiguration cfg) {
         storeSesLsnrs = cfg.getCacheStoreSessionListenerFactories();
         stripedPoolSize = cfg.getStripedPoolSize();
         svcCfgs = cfg.getServiceConfiguration();
+        svcPoolSize = cfg.getServiceThreadPoolSize();
         sysPoolSize = cfg.getSystemThreadPoolSize();
         timeSrvPortBase = cfg.getTimeServerPortBase();
         timeSrvPortRange = cfg.getTimeServerPortRange();
@@ -773,6 +778,18 @@ public int getPublicThreadPoolSize() {
         return pubPoolSize;
     }
 
+    /**
+     * Should return a thread pool size to be used in grid.
+     * This executor service will be in charge of processing {@link Service} proxy invocations.
+     * <p>
+     * If not provided, executor service will have size {@link #DFLT_PUBLIC_THREAD_CNT}.
+     *
+     * @return Thread pool size to be used in grid to process service proxy invocations.
+     */
+    public int getServiceThreadPoolSize() {
+        return svcPoolSize != null ? svcPoolSize : getPublicThreadPoolSize();
+    }
+
     /**
      * Size of thread pool that is in charge of processing internal system messages.
      * <p>
@@ -893,6 +910,19 @@ public IgniteConfiguration setPublicThreadPoolSize(int poolSize) {
         return this;
     }
 
+    /**
+     * Sets thread pool size to use within grid.
+     *
+     * @param poolSize Thread pool size to use within grid.
+     * @see IgniteConfiguration#getServiceThreadPoolSize()
+     * @return {@code this} for chaining.
+     */
+    public IgniteConfiguration setServiceThreadPoolSize(int poolSize) {
+        svcPoolSize = poolSize;
+
+        return this;
+    }
+
     /**
      * Sets system thread pool size to use within grid.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 927944fd908c5..d95f09cce3e17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -504,6 +504,13 @@ public interface GridKernalContext extends Iterable<GridComponent> {
      */
     public ExecutorService getExecutorService();
 
+    /**
+     * Executor service that is in charge of processing service proxy invocations.
+     *
+     * @return Thread pool implementation to be used in grid for service proxy invocations.
+     */
+    public ExecutorService getServiceExecutorService();
+
     /**
      * Executor service that is in charge of processing internal system messages.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index a2ad1b2043368..8e1fd7e9eb049 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -295,6 +295,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     @GridToStringExclude
     protected ExecutorService execSvc;
 
+    /** */
+    @GridToStringExclude
+    protected ExecutorService svcExecSvc;
+
     /** */
     @GridToStringExclude
     protected ExecutorService sysExecSvc;
@@ -405,6 +409,7 @@ protected GridKernalContextImpl(
         ExecutorService utilityCachePool,
         ExecutorService marshCachePool,
         ExecutorService execSvc,
+        ExecutorService svcExecSvc,
         ExecutorService sysExecSvc,
         StripedExecutor stripedExecSvc,
         ExecutorService p2pExecSvc,
@@ -426,6 +431,7 @@ protected GridKernalContextImpl(
         this.utilityCachePool = utilityCachePool;
         this.marshCachePool = marshCachePool;
         this.execSvc = execSvc;
+        this.svcExecSvc = svcExecSvc;
         this.sysExecSvc = sysExecSvc;
         this.stripedExecSvc = stripedExecSvc;
         this.p2pExecSvc = p2pExecSvc;
@@ -951,6 +957,11 @@ protected Object readResolve() throws ObjectStreamException {
         return execSvc;
     }
 
+    /** {@inheritDoc} */
+    @Override public ExecutorService getServiceExecutorService() {
+        return svcExecSvc;
+    }
+
     /** {@inheritDoc} */
     @Override public ExecutorService getSystemExecutorService() {
         return sysExecSvc;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 063fe255af95f..716116625bc6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -686,6 +686,7 @@ public void start(
         ExecutorService utilityCachePool,
         ExecutorService marshCachePool,
         final ExecutorService execSvc,
+        final ExecutorService svcExecSvc,
         final ExecutorService sysExecSvc,
         final StripedExecutor stripedExecSvc,
         ExecutorService p2pExecSvc,
@@ -795,6 +796,7 @@ public void start(
                 utilityCachePool,
                 marshCachePool,
                 execSvc,
+                svcExecSvc,
                 sysExecSvc,
                 stripedExecSvc,
                 p2pExecSvc,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index f32a75309166c..8212dd0512de7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -1457,6 +1457,9 @@ private static final class IgniteNamedInstance {
         /** Executor service. */
         private ThreadPoolExecutor execSvc;
 
+        /** Executor service for services. */
+        private ThreadPoolExecutor svcExecSvc;
+
         /** System executor service. */
         private ThreadPoolExecutor sysExecSvc;
 
@@ -1656,6 +1659,18 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
 
             execSvc.allowCoreThreadTimeOut(true);
 
+            validateThreadPoolSize(cfg.getServiceThreadPoolSize(), "service");
+
+            svcExecSvc = new IgniteThreadPoolExecutor(
+                "svc",
+                cfg.getGridName(),
+                cfg.getServiceThreadPoolSize(),
+                cfg.getServiceThreadPoolSize(),
+                DFLT_THREAD_KEEP_ALIVE_TIME,
+                new LinkedBlockingQueue<Runnable>());
+
+            svcExecSvc.allowCoreThreadTimeOut(true);
+
             validateThreadPoolSize(cfg.getSystemThreadPoolSize(), "system");
 
             sysExecSvc = new IgniteThreadPoolExecutor(
@@ -1801,6 +1816,7 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                     utilityCacheExecSvc,
                     marshCacheExecSvc,
                     execSvc,
+                    svcExecSvc,
                     sysExecSvc,
                     stripedExecSvc,
                     p2pExecSvc,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 7ef7bc080fe98..2eda4b715ee01 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -96,6 +96,7 @@
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.MARSH_CACHE_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.P2P_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.PUBLIC_POOL;
+import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SERVICE_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.UTILITY_CACHE_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.isReservedGridIoPolicy;
@@ -686,6 +687,7 @@ private void onMessage0(UUID nodeId, GridIoMessage msg, IgniteRunnable msgC) {
                 case MARSH_CACHE_POOL:
                 case IDX_POOL:
                 case IGFS_POOL:
+                case SERVICE_POOL:
                 {
                     if (msg.isOrdered())
                         processOrderedMessage(nodeId, msg, plc, msgC);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java
index 70a73541d2ffa..bb64f6a083acf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java
@@ -49,6 +49,9 @@ public class GridIoPolicy {
     /** Pool for handling distributed index range requests. */
     public static final byte IDX_POOL = 8;
 
+    /** Pool for service proxy executions. */
+    public static final byte SERVICE_POOL = 9;
+
     /**
      * Defines the range of reserved pools that are not available for plugins.
      * @param key The key.
@@ -57,4 +60,4 @@ public class GridIoPolicy {
     public static boolean isReservedGridIoPolicy(byte key) {
         return key >= 0 && key <= 31;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
index 59e5e7dffb7e9..66efc8c2084b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.pool;
 
+import java.util.Arrays;
+import java.util.concurrent.Executor;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.GridKernalContext;
@@ -25,9 +27,6 @@
 import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor;
 import org.apache.ignite.plugin.extensions.communication.IoPool;
 
-import java.util.Arrays;
-import java.util.concurrent.Executor;
-
 /**
  * Processor which abstracts out thread pool management.
  */
@@ -128,6 +127,11 @@ public Executor poolForPolicy(byte plc) throws IgniteCheckedException {
 
                 return ctx.getIgfsExecutorService();
 
+            case GridIoPolicy.SERVICE_POOL:
+                assert ctx.getServiceExecutorService() != null : "Service pool is not configured.";
+
+                return ctx.getServiceExecutorService();
+
             default: {
                 if (plc < 0)
                     throw new IgniteCheckedException("Policy cannot be negative: " + plc);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index aa609340b3bfb..d2e96bac52c0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -42,15 +42,19 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.services.Service;
 import org.jsr166.ThreadLocalRandom8;
 
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_IO_POLICY;
+
 /**
  * Wrapper for making {@link org.apache.ignite.services.Service} class proxies.
  */
@@ -58,6 +62,9 @@ public class GridServiceProxy<T> implements Serializable {
     /** */
     private static final long serialVersionUID = 0L;
 
+    /** */
+    private static final IgniteProductVersion SVC_POOL_SINCE_VER = IgniteProductVersion.fromString("1.8.5");
+
     /** Grid logger. */
     @GridToStringExclude
     private final IgniteLogger log;
@@ -176,6 +183,9 @@ else if (U.isToStringMethod(mtd))
                         }
                     }
                     else {
+                        if (node.version().compareTo(SVC_POOL_SINCE_VER) >= 0)
+                            ctx.task().setThreadContext(TC_IO_POLICY, GridIoPolicy.SERVICE_POOL);
+
                         // Execute service remotely.
                         return ctx.closure().callAsyncNoFailover(
                             GridClosureCallMode.BROADCAST,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
index 3bb19241a4ebf..2ec63df4ed83c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
@@ -34,5 +34,8 @@ public enum GridTaskThreadContextKey {
     TC_TIMEOUT,
 
     /** Security subject ID. */
-    TC_SUBJ_ID
-}
\ No newline at end of file
+    TC_SUBJ_ID,
+
+    /** IO manager policy. */
+    TC_IO_POLICY
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
index d89e80bc99825..c426008f5fb11 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
@@ -101,6 +101,7 @@
 import static org.apache.ignite.internal.GridTopic.TOPIC_JOB_CANCEL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.MANAGEMENT_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.PUBLIC_POOL;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_IO_POLICY;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
 
 /**
@@ -1381,8 +1382,21 @@ private void sendRequest(ComputeJobResult res) {
                     if (loc)
                         ctx.job().processJobExecuteRequest(ctx.discovery().localNode(), req);
                     else {
+                        byte plc;
+
+                        if (internal)
+                            plc = MANAGEMENT_POOL;
+                        else {
+                            Byte ctxPlc = getThreadContext(TC_IO_POLICY);
+
+                            if (ctxPlc != null)
+                                plc = ctxPlc;
+                            else
+                                plc = PUBLIC_POOL;
+                        }
+
                         // Send job execution request.
-                        ctx.io().send(node, TOPIC_JOB, req, internal ? MANAGEMENT_POOL : PUBLIC_POOL);
+                        ctx.io().send(node, TOPIC_JOB, req, plc);
 
                         if (log.isDebugEnabled())
                             log.debug("Sent job request [req=" + req + ", node=" + node + ']');
diff --git a/modules/core/src/test/java/org/apache/ignite/services/ServiceThreadPoolSelfTest.java b/modules/core/src/test/java/org/apache/ignite/services/ServiceThreadPoolSelfTest.java
new file mode 100644
index 0000000000000..40efdfaa91d4f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/services/ServiceThreadPoolSelfTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.services;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test verifying that services thread pool is properly used.
+ */
+public class ServiceThreadPoolSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDefaultPoolSize() throws Exception {
+        Ignite ignite = startGrid("grid", new IgniteConfiguration());
+
+        IgniteConfiguration cfg = ignite.configuration();
+
+        assertEquals(IgniteConfiguration.DFLT_PUBLIC_THREAD_CNT, cfg.getPublicThreadPoolSize());
+        assertEquals(IgniteConfiguration.DFLT_PUBLIC_THREAD_CNT, cfg.getServiceThreadPoolSize());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testInheritedPoolSize() throws Exception {
+        Ignite ignite = startGrid("grid", new IgniteConfiguration().setPublicThreadPoolSize(42));
+
+        IgniteConfiguration cfg = ignite.configuration();
+
+        assertEquals(42, cfg.getPublicThreadPoolSize());
+        assertEquals(42, cfg.getServiceThreadPoolSize());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCustomPoolSize() throws Exception {
+        Ignite ignite = startGrid("grid", new IgniteConfiguration().setServiceThreadPoolSize(42));
+
+        IgniteConfiguration cfg = ignite.configuration();
+
+        assertEquals(IgniteConfiguration.DFLT_PUBLIC_THREAD_CNT, cfg.getPublicThreadPoolSize());
+        assertEquals(42, cfg.getServiceThreadPoolSize());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testExecution() throws Exception {
+        startGrid(0); // Server.
+
+        Ignition.setClientMode(true);
+
+        Ignite ignite = startGrid(); // Client.
+
+        ignite.services().deployClusterSingleton("my-service", new MyServiceImpl());
+
+        MyService svc = ignite.services().serviceProxy("my-service", MyService.class, false);
+
+        svc.hello();
+    }
+
+    /**
+     */
+    private interface MyService extends Service{
+        /**
+         * Hello!
+         */
+        void hello();
+    }
+
+    /**
+     */
+    private static class MyServiceImpl implements MyService {
+        /** {@inheritDoc} */
+        @Override public void hello() {
+            String thread = Thread.currentThread().getName();
+
+            assertTrue("Service is executed in wrong thread: " + thread, thread.startsWith("svc-#"));
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(ServiceContext ctx) throws Exception {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
index 143159dd4c57f..49cb206ee3fb3 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
@@ -64,6 +64,7 @@ public GridTestKernalContext(IgniteLogger log, IgniteConfiguration cfg) throws I
                 null,
                 null,
                 null,
+                null,
                 U.allPluginProviders()
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 4f771d6529b85..9b41fbcf96873 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -72,6 +72,7 @@
 import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
 import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
 import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
+import org.apache.ignite.services.ServiceThreadPoolSelfTest;
 import org.apache.ignite.spi.communication.GridCacheMessageSelfTest;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -147,6 +148,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(IgniteServiceProxyTimeoutInitializedTest.class);
         suite.addTestSuite(IgniteServiceDynamicCachesSelfTest.class);
         suite.addTestSuite(GridServiceContinuousQueryRedeploy.class);
+        suite.addTestSuite(ServiceThreadPoolSelfTest.class);
 
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class);

From 39edcc7890ce497771f532a83a57ecae318d8c88 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 28 Mar 2017 18:56:17 +0300
Subject: [PATCH 208/446] IGNITE-4815: Fixed confusing javadoc. This closes
 #1613.

---
 .../org/apache/ignite/spi/collision/CollisionSpi.java | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/collision/CollisionSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/collision/CollisionSpi.java
index d82d9a1dccb34..9b191f6a5d220 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/collision/CollisionSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/collision/CollisionSpi.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.spi.collision;
 
+import org.apache.ignite.events.EventType;
 import org.apache.ignite.spi.IgniteSpi;
 import org.jetbrains.annotations.Nullable;
 
@@ -48,8 +49,14 @@
  */
 public interface CollisionSpi extends IgniteSpi {
     /**
-     * This is a callback called when either new grid job arrived or executing job finished its
-     * execution. When new job arrives it is added to the end of the wait list and this
+     * This is a callback called:
+     * <ul>
+     *      <li>new grid job arrived</li>
+     *      <li>executing job finished its execution</li>
+     *      <li>topology changed</li>
+     *     <li>periodically (on {@link EventType#EVT_NODE_METRICS_UPDATED})</li>
+     * </ul>
+     * When new job arrives it is added to the end of the wait list and this
      * method is called. When job finished its execution, it is removed from the active list and
      * this method is called (i.e., when grid job is finished it will not appear in any list
      * in collision resolution).

From e47bf27b5582cd695a7d3de3c39d54b3df4c59cc Mon Sep 17 00:00:00 2001
From: Valentin Kulichenko <valentin.lulichenko@gmail.com>
Date: Tue, 28 Mar 2017 16:51:44 -0700
Subject: [PATCH 209/446] IGNITE-4762 - transactionsAllowed flag for JDBC
 driver

---
 .../jdbc2/JdbcConnectionSelfTest.java         | 34 +++++++++++++
 .../org/apache/ignite/IgniteJdbcDriver.java   |  9 +++-
 .../ignite/internal/jdbc2/JdbcConnection.java | 50 ++++++++++++-------
 3 files changed, 75 insertions(+), 18 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java
index 8c05df3adf700..0bbdca86c67f9 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java
@@ -267,4 +267,38 @@ public void testClose() throws Exception {
             );
         }
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxAllowedCommit() throws Exception {
+        String url = CFG_URL_PREFIX + "transactionsAllowed=true@" + CFG_URL;
+
+        try (final Connection conn = DriverManager.getConnection(url)) {
+            conn.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
+
+            assertEquals(Connection.TRANSACTION_SERIALIZABLE, conn.getTransactionIsolation());
+
+            conn.setAutoCommit(false);
+
+            conn.commit();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxAllowedRollback() throws Exception {
+        String url = CFG_URL_PREFIX + "transactionsAllowed=true@" + CFG_URL;
+
+        try (final Connection conn = DriverManager.getConnection(url)) {
+            conn.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
+
+            assertEquals(Connection.TRANSACTION_SERIALIZABLE, conn.getTransactionIsolation());
+
+            conn.setAutoCommit(false);
+
+            conn.rollback();
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
index d432c1e4f9fee..c511d5f8b7c02 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
@@ -292,6 +292,9 @@ public class IgniteJdbcDriver implements Driver {
     /** Distributed joins parameter name. */
     private static final String PARAM_DISTRIBUTED_JOINS = "distributedJoins";
 
+    /** Transactions allowed parameter name. */
+    private static final String PARAM_TX_ALLOWED = "transactionsAllowed";
+
     /** Hostname property name. */
     public static final String PROP_HOST = PROP_PREFIX + "host";
 
@@ -313,6 +316,9 @@ public class IgniteJdbcDriver implements Driver {
     /** Distributed joins property name. */
     public static final String PROP_DISTRIBUTED_JOINS = PROP_PREFIX + PARAM_DISTRIBUTED_JOINS;
 
+    /** Transactions allowed property name. */
+    public static final String PROP_TX_ALLOWED = PROP_PREFIX + PARAM_TX_ALLOWED;
+
     /** Cache name property name. */
     public static final String PROP_CFG = PROP_PREFIX + "cfg";
 
@@ -378,7 +384,8 @@ public class IgniteJdbcDriver implements Driver {
             new PropertyInfo("Node ID", info.getProperty(PROP_NODE_ID), ""),
             new PropertyInfo("Local", info.getProperty(PROP_LOCAL), ""),
             new PropertyInfo("Collocated", info.getProperty(PROP_COLLOCATED), ""),
-            new PropertyInfo("Distributed Joins", info.getProperty(PROP_DISTRIBUTED_JOINS), "")
+            new PropertyInfo("Distributed Joins", info.getProperty(PROP_DISTRIBUTED_JOINS), ""),
+            new PropertyInfo("Transactions Allowed", info.getProperty(PROP_TX_ALLOWED), "")
         );
 
         if (info.getProperty(PROP_CFG) != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
index 5c4a147174b45..dc3fe7f454624 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
@@ -73,6 +73,7 @@
 import static org.apache.ignite.IgniteJdbcDriver.PROP_DISTRIBUTED_JOINS;
 import static org.apache.ignite.IgniteJdbcDriver.PROP_LOCAL;
 import static org.apache.ignite.IgniteJdbcDriver.PROP_NODE_ID;
+import static org.apache.ignite.IgniteJdbcDriver.PROP_TX_ALLOWED;
 
 /**
  * JDBC connection implementation.
@@ -118,6 +119,12 @@ public class JdbcConnection implements Connection {
     /** Distributed joins flag. */
     private boolean distributedJoins;
 
+    /** Transactions allowed flag. */
+    private boolean txAllowed;
+
+    /** Current transaction isolation. */
+    private int txIsolation;
+
     /** Statements. */
     final Set<JdbcStatement> statements = new HashSet<>();
 
@@ -134,15 +141,16 @@ public JdbcConnection(String url, Properties props) throws SQLException {
 
         this.url = url;
 
-        this.cacheName = props.getProperty(PROP_CACHE);
-        this.locQry = Boolean.parseBoolean(props.getProperty(PROP_LOCAL));
-        this.collocatedQry = Boolean.parseBoolean(props.getProperty(PROP_COLLOCATED));
-        this.distributedJoins = Boolean.parseBoolean(props.getProperty(PROP_DISTRIBUTED_JOINS));
+        cacheName = props.getProperty(PROP_CACHE);
+        locQry = Boolean.parseBoolean(props.getProperty(PROP_LOCAL));
+        collocatedQry = Boolean.parseBoolean(props.getProperty(PROP_COLLOCATED));
+        distributedJoins = Boolean.parseBoolean(props.getProperty(PROP_DISTRIBUTED_JOINS));
+        txAllowed = Boolean.parseBoolean(props.getProperty(PROP_TX_ALLOWED));
 
         String nodeIdProp = props.getProperty(PROP_NODE_ID);
 
         if (nodeIdProp != null)
-            this.nodeId = UUID.fromString(nodeIdProp);
+            nodeId = UUID.fromString(nodeIdProp);
 
         try {
             String cfgUrl = props.getProperty(PROP_CFG);
@@ -259,7 +267,7 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public void setAutoCommit(boolean autoCommit) throws SQLException {
         ensureNotClosed();
 
-        if (!autoCommit)
+        if (!txAllowed && !autoCommit)
             throw new SQLFeatureNotSupportedException("Transactions are not supported.");
     }
 
@@ -274,14 +282,16 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public void commit() throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        if (!txAllowed)
+            throw new SQLFeatureNotSupportedException("Transactions are not supported.");
     }
 
     /** {@inheritDoc} */
     @Override public void rollback() throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        if (!txAllowed)
+            throw new SQLFeatureNotSupportedException("Transactions are not supported.");
     }
 
     /** {@inheritDoc} */
@@ -354,14 +364,20 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public void setTransactionIsolation(int level) throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        if (txAllowed)
+            txIsolation = level;
+        else
+            throw new SQLFeatureNotSupportedException("Transactions are not supported.");
     }
 
     /** {@inheritDoc} */
     @Override public int getTransactionIsolation() throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        if (txAllowed)
+            return txIsolation;
+        else
+            throw new SQLFeatureNotSupportedException("Transactions are not supported.");
     }
 
     /** {@inheritDoc} */
@@ -413,7 +429,7 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public void setHoldability(int holdability) throws SQLException {
         ensureNotClosed();
 
-        if (holdability != HOLD_CURSORS_OVER_COMMIT)
+        if (!txAllowed && holdability != HOLD_CURSORS_OVER_COMMIT)
             throw new SQLFeatureNotSupportedException("Invalid holdability (transactions are not supported).");
     }
 
@@ -428,28 +444,28 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public Savepoint setSavepoint() throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        throw new SQLFeatureNotSupportedException("Savepoints are not supported.");
     }
 
     /** {@inheritDoc} */
     @Override public Savepoint setSavepoint(String name) throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        throw new SQLFeatureNotSupportedException("Savepoints are not supported.");
     }
 
     /** {@inheritDoc} */
     @Override public void rollback(Savepoint savepoint) throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        throw new SQLFeatureNotSupportedException("Savepoints are not supported.");
     }
 
     /** {@inheritDoc} */
     @Override public void releaseSavepoint(Savepoint savepoint) throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        throw new SQLFeatureNotSupportedException("Savepoints are not supported.");
     }
 
     /** {@inheritDoc} */
@@ -463,7 +479,7 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
         if (resSetConcurrency != CONCUR_READ_ONLY)
             throw new SQLFeatureNotSupportedException("Invalid concurrency (updates are not supported).");
 
-        if (resSetHoldability != HOLD_CURSORS_OVER_COMMIT)
+        if (!txAllowed && resSetHoldability != HOLD_CURSORS_OVER_COMMIT)
             throw new SQLFeatureNotSupportedException("Invalid holdability (transactions are not supported).");
 
         JdbcStatement stmt = new JdbcStatement(this);
@@ -484,7 +500,7 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
         if (resSetConcurrency != CONCUR_READ_ONLY)
             throw new SQLFeatureNotSupportedException("Invalid concurrency (updates are not supported).");
 
-        if (resSetHoldability != HOLD_CURSORS_OVER_COMMIT)
+        if (!txAllowed && resSetHoldability != HOLD_CURSORS_OVER_COMMIT)
             throw new SQLFeatureNotSupportedException("Invalid holdability (transactions are not supported).");
 
         JdbcPreparedStatement stmt = new JdbcPreparedStatement(this, sql);

From 336672432486830c31cb4b6f8bb1b401c276f3e5 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 29 Mar 2017 10:53:25 +0700
Subject: [PATCH 210/446] IGNITE-4659 Fixed typo. (cherry picked from commit
 6f1e970)

---
 .../states/configuration/clusters/general/discovery/s3.pug      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
index ed6e20e4c3c1a..853c32cc5394f 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
@@ -22,6 +22,6 @@ include /app/helpers/jade/mixins
 
 div
     .details-row
-        +text('Bucket name:', `${model}.bucketName`, `${discoveryKind}BucketName'`, required, 'Input bucket name', 'Bucket name for IP finder')
+        +text('Bucket name:', `${model}.bucketName`, `'${discoveryKind}BucketName'`, required, 'Input bucket name', 'Bucket name for IP finder')
     .details-row
         label Note, AWS credentials will be generated as stub

From b689624e2984ec3cf42ca5b01966937e236abcdc Mon Sep 17 00:00:00 2001
From: Alexander Paschenko <alexander.a.paschenko@gmail.com>
Date: Wed, 29 Mar 2017 18:15:20 +0300
Subject: [PATCH 211/446] IGNITE-4868 Raw Object keys and values indexing fix

---
 .../processors/query/GridQueryProcessor.java  |   8 +-
 .../IgniteCacheObjectKeyIndexingSelfTest.java | 124 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite2.java       |   4 +-
 3 files changed, 131 insertions(+), 5 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheObjectKeyIndexingSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 10bf75a01113d..0bde688ded248 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -17,12 +17,11 @@
 
 package org.apache.ignite.internal.processors.query;
 
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.util.concurrent.TimeUnit;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.math.BigDecimal;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
 import java.sql.Time;
 import java.sql.Timestamp;
 import java.util.ArrayList;
@@ -40,6 +39,7 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 import javax.cache.Cache;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
@@ -457,7 +457,7 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
      * @return {@code True} if will be deserialized.
      */
     private boolean mustDeserializeBinary(Class cls) {
-        if (cls != null && ctx.config().getMarshaller() instanceof BinaryMarshaller) {
+        if (cls != null && cls != Object.class && ctx.config().getMarshaller() instanceof BinaryMarshaller) {
             CacheObjectBinaryProcessorImpl proc0 = (CacheObjectBinaryProcessorImpl)ctx.cacheObjects();
 
             return proc0.binaryContext().mustDeserialize(cls);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheObjectKeyIndexingSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheObjectKeyIndexingSelfTest.java
new file mode 100644
index 0000000000000..9e3a0bc160844
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheObjectKeyIndexingSelfTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test index behavior when key is of plain Object type per indexing settings.
+ */
+public class IgniteCacheObjectKeyIndexingSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        Ignition.stopAll(true);
+    }
+
+    /** */
+    protected static CacheConfiguration<Object, TestObject> cacheCfg() {
+        return new CacheConfiguration<Object, TestObject>()
+            .setIndexedTypes(Object.class, TestObject.class);
+    }
+
+    /** */
+    public void testObjectKeyHandling() throws Exception {
+        Ignite ignite = grid();
+
+        IgniteCache<Object, TestObject> cache = ignite.getOrCreateCache(cacheCfg());
+
+        UUID uid = UUID.randomUUID();
+
+        cache.put(uid, new TestObject("A"));
+
+        assertItemsNumber(1);
+
+        cache.put(1, new TestObject("B"));
+
+        assertItemsNumber(2);
+
+        cache.put(uid, new TestObject("C"));
+
+        // Key should have been replaced
+        assertItemsNumber(2);
+
+        List<List<?>> res = cache.query(new SqlFieldsQuery("select _key, name from TestObject order by name")).getAll();
+
+        assertEquals(res,
+            Arrays.asList(
+                Arrays.asList(1, "B"),
+                Arrays.asList(uid, "C")
+            )
+        );
+
+        cache.remove(1);
+
+        assertItemsNumber(1);
+
+        res = cache.query(new SqlFieldsQuery("select _key, name from TestObject")).getAll();
+
+        assertEquals(res,
+            Collections.singletonList(
+                Arrays.asList(uid, "C")
+            )
+        );
+        
+        cache.remove(uid);
+
+        // Removal has worked for both keys although the table was the same and keys were of different type
+        assertItemsNumber(0);
+    }
+
+    /** */
+    @SuppressWarnings("ConstantConditions")
+    private void assertItemsNumber(long num) {
+        assertEquals(num, grid().cachex().size());
+
+        assertEquals(num, grid().cache(null).query(new SqlFieldsQuery("select count(*) from TestObject")).getAll()
+            .get(0).get(0));
+    }
+    
+    /** */
+    private static class TestObject {
+        /** */
+        @QuerySqlField
+        public final String name;
+
+        /** */
+        private TestObject(String name) {
+            this.name = name;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index 8ac219f89f199..923a79c3bd189 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -36,14 +36,15 @@
 import org.apache.ignite.internal.processors.cache.GridCacheQueryIndexingDisabledSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheFieldsQueryNoDataSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheNoClassQuerySelfTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheObjectKeyIndexingSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheP2pUnmarshallingQueryErrorTest;
 import org.apache.ignite.internal.processors.cache.SqlFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheAtomicFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheAtomicNearEnabledFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheDistributedQueryCancelSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheDistributedQueryStopOnCancelOrTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCachePartitionedFieldsQueryP2PEnabledSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCachePartitionedFieldsQuerySelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheDistributedQueryStopOnCancelOrTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryStopOnCancelOrTimeoutDistributedJoinSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedFieldsQueryP2PEnabledSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedFieldsQuerySelfTest;
@@ -120,6 +121,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheQueryNewClientSelfTest.class);
         suite.addTestSuite(CacheOffheapBatchIndexingSingleTypeTest.class);
         suite.addTestSuite(CacheSqlQueryValueCopySelfTest.class);
+        suite.addTestSuite(IgniteCacheObjectKeyIndexingSelfTest.class);
 
         return suite;
     }

From 357c20ab9593390fb7af25f8638188595c5f6cd4 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 30 Mar 2017 12:50:42 +0300
Subject: [PATCH 212/446] IGNITE-4284 - Fix. Failed second client node join
 with continuous query and peer class loading enabled.

---
 .../continuous/GridContinuousProcessor.java   |  39 +++--
 .../custom/DummyEventFilterFactory.java       |  47 ++++++
 .../ContinuousQueryPeerClassLoadingTest.java  | 142 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite3.java       |   2 +
 4 files changed, 214 insertions(+), 16 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 9fd9b6d44b89e..6a4f57d1e3ae8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -67,6 +67,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
@@ -495,29 +496,34 @@ public void unlockStopping() {
             for (Map.Entry<UUID, Map<UUID, LocalRoutineInfo>> entry : data.clientInfos.entrySet()) {
                 UUID clientNodeId = entry.getKey();
 
-                if (!ctx.localNodeId().equals(clientNodeId)) {
+                if (!ctx.clientNode()) {
                     Map<UUID, LocalRoutineInfo> clientRoutineMap = entry.getValue();
 
                     for (Map.Entry<UUID, LocalRoutineInfo> e : clientRoutineMap.entrySet()) {
                         UUID routineId = e.getKey();
                         LocalRoutineInfo info = e.getValue();
 
-                        try {
-                            if (info.prjPred != null)
-                                ctx.resource().injectGeneric(info.prjPred);
-
-                            if (info.prjPred == null || info.prjPred.apply(ctx.discovery().localNode())) {
-                                registerHandler(clientNodeId,
-                                    routineId,
-                                    info.hnd,
-                                    info.bufSize,
-                                    info.interval,
-                                    info.autoUnsubscribe,
-                                    false);
+                        GridCacheContext cctx = ctx.cache().context().cacheContext(CU.cacheId(info.hnd.cacheName()));
+
+                        // Do not register handler if it's not affinity node.
+                        if (cctx == null || cctx.affinityNode()) {
+                            try {
+                                if (info.prjPred != null)
+                                    ctx.resource().injectGeneric(info.prjPred);
+
+                                if (info.prjPred == null || info.prjPred.apply(ctx.discovery().localNode())) {
+                                    registerHandler(clientNodeId,
+                                        routineId,
+                                        info.hnd,
+                                        info.bufSize,
+                                        info.interval,
+                                        info.autoUnsubscribe,
+                                        false);
+                                }
+                            }
+                            catch (IgniteCheckedException err) {
+                                U.error(log, "Failed to register continuous handler.", err);
                             }
-                        }
-                        catch (IgniteCheckedException err) {
-                            U.error(log, "Failed to register continuous handler.", err);
                         }
                     }
                 }
@@ -583,6 +589,7 @@ public void onCacheStop(GridCacheContext ctx) {
      * @return Routine ID.
      * @throws IgniteCheckedException If failed.
      */
+    @SuppressWarnings("unchecked")
     public UUID registerStaticRoutine(
         String cacheName,
         CacheEntryUpdatedListener<?, ?> locLsnr,
diff --git a/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java b/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java
new file mode 100644
index 0000000000000..e0688bccd934d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.custom;
+
+import javax.cache.configuration.Factory;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryEventFilter;
+import javax.cache.event.CacheEntryListenerException;
+
+/**
+ * Must be not in org.apache.ignite.internal
+ */
+public class DummyEventFilterFactory implements Factory<CacheEntryEventFilter<Integer, String>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override public CacheEntryEventFilter<Integer, String> create() {
+        return new DummyEventFilter();
+    }
+
+    /**
+     *
+     */
+    private static class DummyEventFilter implements CacheEntryEventFilter<Integer, String> {
+        /** {@inheritDoc} */
+        @Override public boolean evaluate(
+            final CacheEntryEvent<? extends Integer, ? extends String> evt) throws CacheEntryListenerException {
+            return true;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java
new file mode 100644
index 0000000000000..e5d1d609d7052
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryListenerException;
+import javax.cache.event.CacheEntryUpdatedListener;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.custom.DummyEventFilterFactory;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Checks if filter factory correctly deployed on all nodes.
+ */
+public class ContinuousQueryPeerClassLoadingTest extends GridCommonAbstractTest {
+    /** */
+    public static final String CACHE_NAME = "test-cache";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setPeerClassLoadingEnabled(true);
+        cfg.setClientMode(gridName.contains("client"));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRemoteFilterFactoryClient() throws Exception {
+        check("server", "client1", "client2");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRemoteFilterFactoryServer1() throws Exception {
+        check("server1", "server2", "client");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRemoteFilterFactoryServer2() throws Exception {
+        check("server1", "server2", "server3");
+    }
+
+    /**
+     * @param node1Name Node 1 name.
+     * @param node2Name Node 2 name.
+     * @param node3Name Node 3 name.
+     */
+    private void check(String node1Name, String node2Name, String node3Name) throws Exception {
+        final Ignite node1 = startGrid(node1Name);
+
+        final IgniteCache<Integer, String> cache = node1.getOrCreateCache(CACHE_NAME);
+
+        for (int i = 0; i < 10; i++)
+            cache.put(i, String.valueOf(i));
+
+        final Ignite node2 = startGrid(node2Name);
+
+        final ContinuousQuery<Integer, String> qry1 = new ContinuousQuery<>();
+        final ContinuousQuery<Integer, String> qry2 = new ContinuousQuery<>();
+
+        qry1.setRemoteFilterFactory(new DummyEventFilterFactory());
+        qry2.setRemoteFilterFactory(new DummyEventFilterFactory());
+
+        final AtomicInteger client1Evts = new AtomicInteger(0);
+        final AtomicInteger client2Evts = new AtomicInteger(0);
+
+        final CountDownLatch latch1 = new CountDownLatch(20);
+        final CountDownLatch latch2 = new CountDownLatch(10);
+
+        qry1.setLocalListener(new CacheEntryUpdatedListener<Integer, String>() {
+            @Override public void onUpdated(
+                final Iterable<CacheEntryEvent<? extends Integer, ? extends String>> evts) throws CacheEntryListenerException {
+                System.out.println(">> Client 1 events " + evts);
+                for (CacheEntryEvent<? extends Integer, ? extends String> evt : evts)
+                    latch1.countDown();
+            }
+        });
+
+        qry2.setLocalListener(new CacheEntryUpdatedListener<Integer, String>() {
+            @Override public void onUpdated(
+                final Iterable<CacheEntryEvent<? extends Integer, ? extends String>> evts) throws CacheEntryListenerException {
+                System.out.println(">> Client 2 events " + evts);
+                for (CacheEntryEvent<? extends Integer, ? extends String> evt : evts)
+                    latch2.countDown();
+            }
+        });
+
+        final IgniteCache<Integer, String> cache1 = node2.cache(CACHE_NAME);
+
+        cache1.query(qry1);
+
+        for (int i = 10; i < 20; i++)
+            cache.put(i, String.valueOf(i));
+
+        // Fail on start second client.
+        final Ignite node3 = startGrid(node3Name);
+
+        final IgniteCache<Integer, String> cache2 = node3.cache(CACHE_NAME);
+
+        cache2.query(qry2);
+
+        for (int i = 20; i < 30; i++)
+            cache.put(i, String.valueOf(i));
+
+        assert latch1.await(5, TimeUnit.SECONDS) : latch1.getCount();
+        assert latch2.await(5, TimeUnit.SECONDS) : latch2.getCount();
+    }
+
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
index a865788cbdfc4..6b2fea0a5ab55 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationStoreEnabledTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationSwapEnabledTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryPeerClassLoadingTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryRemoteFilterMissingInClassPathSelfTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicNearEnabledSelfTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicOffheapTieredTest;
@@ -123,6 +124,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheKeepBinaryIterationNearEnabledTest.class);
         suite.addTestSuite(IgniteCacheContinuousQueryBackupQueueTest.class);
         suite.addTestSuite(IgniteCacheContinuousQueryNoUnsubscribeTest.class);
+        suite.addTestSuite(ContinuousQueryPeerClassLoadingTest.class);
 
         return suite;
     }

From 45a4cfe7ced27aa9d726b36326bfae8b006e84da Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 30 Mar 2017 01:28:20 +0300
Subject: [PATCH 213/446] IGNITE-4141 - JDBC driver should always set
 withKeepBinary flag when querying cache. This fixes #1617.

(Backport from master)
(cherry picked from commit 8dd88d8)
---
 .../jdbc/AbstractJdbcPojoQuerySelfTest.java   | 169 ++++++++++++++++++
 .../jdbc/JdbcPojoLegacyQuerySelfTest.java     |  44 +++++
 .../ignite/jdbc/JdbcPojoQuerySelfTest.java    |  56 ++++++
 .../jdbc/suite/IgniteJdbcDriverTestSuite.java |   4 +
 .../ignite/internal/jdbc2/JdbcQueryTask.java  |   2 +-
 .../internal/jdbc2/JdbcQueryTaskV2.java       |   2 +-
 .../query/jdbc/GridCacheQueryJdbcTask.java    |   4 +-
 7 files changed, 277 insertions(+), 4 deletions(-)
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/AbstractJdbcPojoQuerySelfTest.java
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoLegacyQuerySelfTest.java
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoQuerySelfTest.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/AbstractJdbcPojoQuerySelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/AbstractJdbcPojoQuerySelfTest.java
new file mode 100644
index 0000000000000..a150574893cb2
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/AbstractJdbcPojoQuerySelfTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc;
+
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Collections;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.config.GridTestProperties;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * Test for Jdbc driver query without class on client
+ */
+public abstract class AbstractJdbcPojoQuerySelfTest extends GridCommonAbstractTest {
+    /** IP finder. */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** TestObject class name. */
+    protected static final String TEST_OBJECT = "org.apache.ignite.internal.JdbcTestObject";
+
+    /** TestObject class name. */
+    protected static final String TEST_OBJECT_2 = "org.apache.ignite.internal.JdbcTestObject2";
+
+    /** Statement. */
+    protected Statement stmt;
+
+    /** */
+    private String marshallerBackup = GridTestProperties.getProperty(GridTestProperties.MARSH_CLASS_NAME);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, BinaryMarshaller.class.getName());
+
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration<?,?> cache = defaultCacheConfiguration();
+
+        cache.setWriteSynchronizationMode(FULL_SYNC);
+        cache.setAtomicityMode(TRANSACTIONAL);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(disco);
+        cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+        QueryEntity queryEntity = new QueryEntity();
+        queryEntity.setKeyType("java.lang.String");
+        queryEntity.setValueType("org.apache.ignite.internal.JdbcTestObject");
+        queryEntity.addQueryField("id", "java.lang.Integer", null);
+        queryEntity.addQueryField("testObject", "org.apache.ignite.internal.JdbcTestObject2", null);
+        queryEntity.setIndexes(Collections.singletonList(new QueryIndex("id")));
+
+        cache.setQueryEntities(Collections.singletonList(queryEntity));
+
+        cfg.setCacheConfiguration(cache);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        Ignite ignite = startGrid(0);
+
+        BinaryObjectBuilder builder = ignite.binary().builder(TEST_OBJECT);
+        BinaryObjectBuilder builder2 = ignite.binary().builder(TEST_OBJECT_2);
+
+        builder2.setField("id", 1);
+        builder2.setField("boolVal", true);
+
+        BinaryObject testObject = builder2.build();
+
+        builder.setField("id", 1);
+        builder.setField("testObject", testObject);
+
+        BinaryObject binObj = builder.build();
+
+        IgniteCache<String, BinaryObject> cache = grid(0).cache(null);
+
+        cache.put("0", binObj);
+
+        Class.forName("org.apache.ignite.IgniteJdbcDriver");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, marshallerBackup);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stmt = DriverManager.getConnection(getURL()).createStatement();
+
+        assertNotNull(stmt);
+        assertFalse(stmt.isClosed());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        if (stmt != null) {
+            stmt.getConnection().close();
+            stmt.close();
+
+            assertTrue(stmt.isClosed());
+        }
+    }
+
+    /**
+     * @param rs Result set.
+     * @throws Exception In case of error.
+     */
+    protected void assertResultSet(ResultSet rs) throws Exception {
+        assertNotNull(rs);
+
+        int cnt = 0;
+
+        while (rs.next()) {
+            assertNotNull(rs.getString("id"));
+            assertNotNull(rs.getString("testObject"));
+
+            assertTrue(rs.getObject("testObject").toString().contains("id=1"));
+            assertTrue(rs.getObject("testObject").toString().contains("boolVal=true"));
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+    }
+
+    /**
+     * @return URL.
+     */
+    protected abstract String getURL();
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoLegacyQuerySelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoLegacyQuerySelfTest.java
new file mode 100644
index 0000000000000..4fa7ba5443870
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoLegacyQuerySelfTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc;
+
+import java.sql.ResultSet;
+
+/**
+ * Test for Jdbc driver query without class on client
+ */
+public class JdbcPojoLegacyQuerySelfTest extends AbstractJdbcPojoQuerySelfTest {
+    /** URL. */
+    private static final String URL = "jdbc:ignite://127.0.0.1/";
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testJdbcQuery() throws Exception {
+        stmt.execute("select * from JdbcTestObject");
+
+        ResultSet rs = stmt.getResultSet();
+
+        assertResultSet(rs);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected String getURL() {
+        return URL;
+    }
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoQuerySelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoQuerySelfTest.java
new file mode 100644
index 0000000000000..c2af8a1a5583f
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoQuerySelfTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc;
+
+import java.sql.ResultSet;
+
+import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX;
+
+/**
+ * Test for Jdbc driver query without class on client
+ */
+public class JdbcPojoQuerySelfTest extends AbstractJdbcPojoQuerySelfTest {
+    /** URL. */
+    private static final String URL = CFG_URL_PREFIX + "modules/clients/src/test/config/jdbc-bin-config.xml";
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testJdbcQueryTask2() throws Exception {
+        stmt.execute("select * from JdbcTestObject");
+
+        ResultSet rs = stmt.getResultSet();
+
+        assertResultSet(rs);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testJdbcQueryTask1() throws Exception {
+        ResultSet rs = stmt.executeQuery("select * from JdbcTestObject");
+
+        assertResultSet(rs);
+
+    }
+
+    /** {@inheritDoc} */
+    @Override protected String getURL() {
+        return URL;
+    }
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index 048643b75dd04..a28d29e00c52b 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -25,6 +25,8 @@
 import org.apache.ignite.jdbc.JdbcLocalCachesSelfTest;
 import org.apache.ignite.jdbc.JdbcMetadataSelfTest;
 import org.apache.ignite.jdbc.JdbcNoDefaultCacheTest;
+import org.apache.ignite.jdbc.JdbcPojoLegacyQuerySelfTest;
+import org.apache.ignite.jdbc.JdbcPojoQuerySelfTest;
 import org.apache.ignite.jdbc.JdbcPreparedStatementSelfTest;
 import org.apache.ignite.jdbc.JdbcResultSetSelfTest;
 import org.apache.ignite.jdbc.JdbcStatementSelfTest;
@@ -50,6 +52,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(JdbcEmptyCacheSelfTest.class));
         suite.addTest(new TestSuite(JdbcLocalCachesSelfTest.class));
         suite.addTest(new TestSuite(JdbcNoDefaultCacheTest.class));
+        suite.addTest(new TestSuite(JdbcPojoQuerySelfTest.class));
+        suite.addTest(new TestSuite(JdbcPojoLegacyQuerySelfTest.class));
 
         // Ignite client node based driver tests
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcConnectionSelfTest.class));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java
index 0b23f9b18d0e7..bd6b0f2c74c98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java
@@ -158,7 +158,7 @@ public JdbcQueryTask(Ignite ignite, String cacheName, String sql,
             qry.setCollocated(collocatedQry);
             qry.setDistributedJoins(distributedJoins);
 
-            QueryCursor<List<?>> qryCursor = cache.query(qry);
+            QueryCursor<List<?>> qryCursor = cache.withKeepBinary().query(qry);
 
             Collection<GridQueryFieldMetadata> meta = ((QueryCursorImpl<List<?>>)qryCursor).fieldsMeta();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java
index 9093d1500e65a..61f152dd66000 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java
@@ -161,7 +161,7 @@ public JdbcQueryTaskV2(Ignite ignite, String cacheName, String sql,
             qry.setCollocated(collocatedQry);
             qry.setDistributedJoins(distributedJoins);
 
-            QueryCursorImpl<List<?>> qryCursor = (QueryCursorImpl<List<?>>)cache.query(qry);
+            QueryCursorImpl<List<?>> qryCursor = (QueryCursorImpl<List<?>>)cache.withKeepBinary().query(qry);
 
             if (isQry == null)
                 isQry = qryCursor.isQuery();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/jdbc/GridCacheQueryJdbcTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/jdbc/GridCacheQueryJdbcTask.java
index ca08ead4f11eb..4ae8a8a92fdf5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/jdbc/GridCacheQueryJdbcTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/jdbc/GridCacheQueryJdbcTask.java
@@ -268,7 +268,7 @@ private static class JdbcDriverJob extends ComputeJobAdapter {
 
                 qry.setPageSize(pageSize);
 
-                QueryCursor<List<?>> cursor = cache.query(qry);
+                QueryCursor<List<?>> cursor = cache.withKeepBinary().query(qry);
 
                 Collection<GridQueryFieldMetadata> meta = ((QueryCursorImpl<List<?>>)cursor).fieldsMeta();
 
@@ -453,4 +453,4 @@ private Cursor(QueryCursor<List<?>> cursor, Iterator<List<?>> iter, int totalCnt
             return iter;
         }
     }
-}
\ No newline at end of file
+}

From d4979b87e3301f11f52531645028ecd4e947986b Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 30 Mar 2017 17:35:28 +0300
Subject: [PATCH 214/446] IGNITE-4284 - Fix. Failed second client node join
 with continuous query and peer class loading enabled.

---
 .../continuous/GridContinuousProcessor.java   | 35 ++++++++-----------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 6a4f57d1e3ae8..391c266079d2a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -503,28 +503,23 @@ public void unlockStopping() {
                         UUID routineId = e.getKey();
                         LocalRoutineInfo info = e.getValue();
 
-                        GridCacheContext cctx = ctx.cache().context().cacheContext(CU.cacheId(info.hnd.cacheName()));
-
-                        // Do not register handler if it's not affinity node.
-                        if (cctx == null || cctx.affinityNode()) {
-                            try {
-                                if (info.prjPred != null)
-                                    ctx.resource().injectGeneric(info.prjPred);
-
-                                if (info.prjPred == null || info.prjPred.apply(ctx.discovery().localNode())) {
-                                    registerHandler(clientNodeId,
-                                        routineId,
-                                        info.hnd,
-                                        info.bufSize,
-                                        info.interval,
-                                        info.autoUnsubscribe,
-                                        false);
-                                }
-                            }
-                            catch (IgniteCheckedException err) {
-                                U.error(log, "Failed to register continuous handler.", err);
+                        try {
+                            if (info.prjPred != null)
+                                ctx.resource().injectGeneric(info.prjPred);
+
+                            if (info.prjPred == null || info.prjPred.apply(ctx.discovery().localNode())) {
+                                registerHandler(clientNodeId,
+                                    routineId,
+                                    info.hnd,
+                                    info.bufSize,
+                                    info.interval,
+                                    info.autoUnsubscribe,
+                                    false);
                             }
                         }
+                        catch (IgniteCheckedException err) {
+                            U.error(log, "Failed to register continuous handler.", err);
+                        }
                     }
                 }
 

From 2e6bc4416a1f2fa18dad78235b3f71f6fb58a98d Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 30 Mar 2017 17:36:22 +0300
Subject: [PATCH 215/446] Cleanup unused imports.

---
 .../internal/processors/continuous/GridContinuousProcessor.java  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 391c266079d2a..4717e8724d038 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -67,7 +67,6 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;

From e01aee0b1bbdb8ff5583728e539df165029f682d Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 31 Mar 2017 20:19:52 +0300
Subject: [PATCH 216/446] Fixed SSL issue.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../tcp/TcpCommunicationSpi.java              | 35 +++++++++++++++++--
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 791a7d52dbd0b..89ecc36d2ace1 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3194,7 +3194,7 @@ else if (log.isDebugEnabled())
 
                             buf = ByteBuffer.allocate(1000);
 
-                            ByteBuffer decode = null;
+                            ByteBuffer decode = ByteBuffer.allocate(2 * buf.capacity());
 
                             buf.order(ByteOrder.nativeOrder());
 
@@ -3207,13 +3207,17 @@ else if (log.isDebugEnabled())
 
                                 buf.flip();
 
-                                decode = sslHnd.decode(buf);
+                                ByteBuffer decode0 = sslHnd.decode(buf);
 
-                                i += decode.remaining();
+                                i += decode0.remaining();
+
+                                decode = appendAndResizeIfNeeded(decode, decode0);
 
                                 buf.clear();
                             }
 
+                            decode.flip();
+
                             rcvCnt = decode.getLong(1);
 
                             if (decode.limit() > 9) {
@@ -3301,6 +3305,31 @@ else if (log.isDebugEnabled())
                 "is node stopping?) [senderNodeId=" + sndId + ", msg=" + msg + ']');
     }
 
+    /**
+     * @param target Target buffer to append to.
+     * @param src Source buffer to get data.
+     * @return Original or expanded buffer.
+     */
+    private ByteBuffer appendAndResizeIfNeeded(ByteBuffer target, ByteBuffer src) {
+        if (target.remaining() < src.remaining()) {
+            int newSize = Math.max(target.capacity() * 2, target.capacity() + src.remaining());
+
+            ByteBuffer tmp = ByteBuffer.allocate(newSize);
+
+            tmp.order(target.order());
+
+            target.flip();
+
+            tmp.put(target);
+
+            target = tmp;
+        }
+
+        target.put(src);
+
+        return target;
+    }
+
     /**
      * Stops service threads to simulate node failure.
      *

From 925ee11c2002729b1264148ee5db5700ded5a7b7 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 4 Apr 2017 16:06:01 +0700
Subject: [PATCH 217/446] Fixed typo. (cherry picked from commit 3b84f34)

---
 .../app/modules/states/configuration/clusters/general.pug       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
index be56a6c44d08f..dfc49d61ef4c2 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
@@ -49,7 +49,7 @@ include /app/helpers/jade/mixins
                             <li>AWS S3 - AWS S3 based IP finder that automatically discover cluster nodes on Amazon EC2 cloud</li>\
                             <li>Apache jclouds - Apache jclouds multi cloud toolkit based IP finder for cloud platforms with unstable IP addresses</li>\
                             <li>Google cloud storage - Google Cloud Storage based IP finder that automatically discover cluster nodes on Google Compute Engine cluster</li>\
-                            <li>JDBC - JDBC based IP finder that use database to store node IP addres</li>\
+                            <li>JDBC - JDBC based IP finder that use database to store node IP address</li>\
                             <li>Shared filesystem - Shared filesystem based IP finder that use file to store node IP address</li>\
                             <li>Apache ZooKeeper - Apache ZooKeeper based IP finder when you use ZooKeeper to coordinate your distributed environment</li>\
                         </ul>')

From ce4b078c1c97cae4136c318ae38b27a50fe383cc Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 4 Apr 2017 16:14:56 +0700
Subject: [PATCH 218/446] master Updated version. (cherry picked from commit
 5469626)

---
 .../frontend/app/modules/configuration/Version.service.js       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/app/modules/configuration/Version.service.js b/modules/web-console/frontend/app/modules/configuration/Version.service.js
index 3fc719946729f..164bd20c69a0f 100644
--- a/modules/web-console/frontend/app/modules/configuration/Version.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/Version.service.js
@@ -24,7 +24,7 @@ const numberComparator = (a, b) => a > b ? 1 : a < b ? -1 : 0;
 
 export default class IgniteVersion {
     /** Current product version. */
-    static ignite = '1.8.0';
+    static ignite = '1.9.0';
 
     /**
      * Tries to parse product version from it's string representation.

From ae435fb5474429e7ac76faadd680c19a4c657371 Mon Sep 17 00:00:00 2001
From: Sergi Vladykin <sergi.vladykin@gmail.com>
Date: Sun, 2 Apr 2017 15:35:04 +0300
Subject: [PATCH 219/446] ignite-2913 - SQL: EXISTS support added

---
 .../query/h2/sql/GridSqlOperationType.java    | 23 ++++-
 .../query/h2/sql/GridSqlQueryParser.java      | 19 +++-
 .../query/IgniteSqlSplitterSelfTest.java      | 94 ++++++++++++++-----
 .../query/h2/sql/GridQueryParsingTest.java    | 91 +++++++++++++++---
 4 files changed, 183 insertions(+), 44 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java
index 8d3165148219f..9ffabad1b613f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java
@@ -31,7 +31,7 @@ public enum GridSqlOperationType {
     MULTIPLY(2, new BiExpressionSqlGenerator("*")),
     DIVIDE(2, new BiExpressionSqlGenerator("/")),
     MODULUS(2, new BiExpressionSqlGenerator("%")),
-    NEGATE(1, new PrefixSqlGenerator("-")),
+    NEGATE(1, new PrefixSqlGenerator("-", true)),
 
     // from org.h2.expression.Comparison
     EQUAL(2, new BiExpressionSqlGenerator("=")),
@@ -47,7 +47,7 @@ public enum GridSqlOperationType {
     IS_NULL(1, new SuffixSqlGenerator("IS NULL")),
     IS_NOT_NULL(1, new SuffixSqlGenerator("IS NOT NULL")),
 
-    NOT(1, new PrefixSqlGenerator("NOT")),
+    NOT(1, new PrefixSqlGenerator("NOT", true)),
 
     // from org.h2.expression.ConditionAndOr
     AND(2, new BiExpressionSqlGenerator("AND")),
@@ -58,6 +58,7 @@ public enum GridSqlOperationType {
     LIKE(2, new BiExpressionSqlGenerator("LIKE")),
 
     IN(-1, new ConditionInSqlGenerator()),
+    EXISTS(1, new PrefixSqlGenerator("EXISTS", false)),
 
     ;
     /** */
@@ -145,18 +146,32 @@ private static class PrefixSqlGenerator implements SqlGenerator {
         /** */
         private final String text;
 
+        /** */
+        private final boolean addSpace;
+
         /**
          * @param text Text.
+         * @param addSpace Add space char after the prefix.
          */
-        private PrefixSqlGenerator(String text) {
+        private PrefixSqlGenerator(String text, boolean addSpace) {
             this.text = text;
+            this.addSpace = addSpace;
         }
 
         /** {@inheritDoc} */
         @Override public String getSql(GridSqlOperation operation) {
             assert operation.operationType().childrenCnt == 1;
 
-            return '(' + text + ' ' + operation.child().getSQL() + ')';
+            StringBuilder b = new StringBuilder();
+
+            b.append('(').append(text);
+
+            if (addSpace)
+                b.append(' ');
+
+            b.append(operation.child(0).getSQL()).append(')');
+
+            return b.toString();
         }
     }
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index d9c546c7fb104..a3e80ec6424da 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -46,6 +46,7 @@
 import org.h2.expression.CompareLike;
 import org.h2.expression.Comparison;
 import org.h2.expression.ConditionAndOr;
+import org.h2.expression.ConditionExists;
 import org.h2.expression.ConditionIn;
 import org.h2.expression.ConditionInConstantSet;
 import org.h2.expression.ConditionInSelect;
@@ -80,6 +81,7 @@
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.DIVIDE;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.EQUAL;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.EQUAL_NULL_SAFE;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.EXISTS;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.IN;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.IS_NOT_NULL;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.IS_NULL;
@@ -186,7 +188,10 @@ public class GridSqlQueryParser {
         "compareType");
 
     /** */
-    private static final Getter<ConditionInSelect, Query> QUERY = getter(ConditionInSelect.class, "query");
+    private static final Getter<ConditionInSelect, Query> QUERY_IN = getter(ConditionInSelect.class, "query");
+
+    /** */
+    private static final Getter<ConditionExists, Query> QUERY_EXISTS = getter(ConditionExists.class, "query");
 
     /** */
     private static final Getter<CompareLike, Expression> LEFT = getter(CompareLike.class, "left");
@@ -854,7 +859,7 @@ private GridSqlElement parseExpression0(Expression expression, boolean calcTypes
 
             res.addChild(parseExpression(LEFT_CIS.get((ConditionInSelect)expression), calcTypes));
 
-            Query qry = QUERY.get((ConditionInSelect)expression);
+            Query qry = QUERY_IN.get((ConditionInSelect)expression);
 
             assert0(qry instanceof Select, qry);
 
@@ -959,6 +964,16 @@ private GridSqlElement parseExpression0(Expression expression, boolean calcTypes
             return res;
         }
 
+        if (expression instanceof ConditionExists) {
+            Query qry = QUERY_EXISTS.get((ConditionExists)expression);
+
+            GridSqlOperation res = new GridSqlOperation(EXISTS);
+
+            res.addChild(new GridSqlSubquery(parse(qry, null)));
+
+            return res;
+        }
+
         throw new IgniteException("Unsupported expression: " + expression + " [type=" +
             expression.getClass().getSimpleName() + ']');
     }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index e72c9cb5b6d62..c3f1bd277a2de 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -21,9 +21,11 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicLong;
 import javax.cache.CacheException;
@@ -33,10 +35,8 @@
 import org.apache.ignite.cache.CacheKeyConfiguration;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CachePeekMode;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
@@ -63,8 +63,8 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
     private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
         CacheKeyConfiguration keyCfg = new CacheKeyConfiguration(TestKey.class.getName(), "affKey");
 
@@ -81,6 +81,11 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    @Override
+    protected long getTestTimeout() {
+        return 100_000_000;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         startGridsMultiThreaded(3, false);
@@ -148,6 +153,47 @@ public void testOffsetLimit() throws Exception {
         }
     }
 
+    @SuppressWarnings("SuspiciousMethodCalls")
+    public void testExists() {
+        IgniteCache<Integer,Person2> x = ignite(0).getOrCreateCache(cacheConfig("x", true,
+            Integer.class, Person2.class));
+        IgniteCache<Integer,Person2> y = ignite(0).getOrCreateCache(cacheConfig("y", true,
+            Integer.class, Person2.class));
+
+        try {
+            GridRandom rnd = new GridRandom();
+
+            Set<Integer> intersects = new HashSet<>();
+
+            for (int i = 0; i < 3000; i++) {
+                int r = rnd.nextInt(3);
+
+                if (r != 0)
+                    x.put(i, new Person2(i, "pers_x_" + i));
+
+                if (r != 1)
+                    y.put(i, new Person2(i, "pers_y_" + i));
+
+                if (r == 2)
+                    intersects.add(i);
+            }
+
+            assertFalse(intersects.isEmpty());
+
+            List<List<?>> res = x.query(new SqlFieldsQuery("select _key from \"x\".Person2 px " +
+                "where exists(select 1 from \"y\".Person2 py where px._key = py._key)")).getAll();
+
+            assertEquals(intersects.size(), res.size());
+
+            for (List<?> row : res)
+                assertTrue(intersects.contains(row.get(0)));
+        }
+        finally {
+            x.destroy();
+            y.destroy();
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -550,15 +596,15 @@ public void testDistributedJoinsPlan() throws Exception {
                 "\"orgRepl\".Organization o",
                 "where p.affKey = o._key", true);
 
-            checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
-                "(select * from \"persPart\".Person2) p",
-                "\"orgPart\".Organization o",
-                "where p._key = o._key", false);
-
-            checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
-                "\"persPart\".Person2 p",
-                "(select * from \"orgPart\".Organization) o",
-                "where p._key = o._key", false);
+            // TODO Now we can not analyze subqueries to decide if we are collocated or not.
+//            checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
+//                "(select * from \"persPart\".Person2) p",
+//                "\"orgPart\".Organization o",
+//                "where p._key = o._key", false);
+//            checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
+//                "\"persPart\".Person2 p",
+//                "(select * from \"orgPart\".Organization) o",
+//                "where p._key = o._key", false);
 
             // Join multiple.
 
@@ -703,6 +749,7 @@ public void testDistributedJoinsPlan() throws Exception {
                 ignite(0).destroyCache(cache.getName());
         }
     }
+
     /**
      * @throws Exception If failed.
      */
@@ -791,26 +838,26 @@ private void checkNoBatchedJoin(IgniteCache<Object, Object> cache,
             false,
             0,
             select +
-                "from " + cache1 + ","  + cache2 + " "+ where);
+                "from " + cache1 + "," + cache2 + " " + where);
 
         checkQueryPlan(cache,
             false,
             0,
             select +
-                "from " + cache2 + ","  + cache1 + " "+ where);
+                "from " + cache2 + "," + cache1 + " " + where);
 
         if (testEnforceJoinOrder) {
             checkQueryPlan(cache,
                 true,
                 0,
                 select +
-                    "from " + cache1 + ","  + cache2 + " "+ where);
+                    "from " + cache1 + "," + cache2 + " " + where);
 
             checkQueryPlan(cache,
                 true,
                 0,
                 select +
-                    "from " + cache2 + ","  + cache1 + " "+ where);
+                    "from " + cache2 + "," + cache1 + " " + where);
         }
     }
 
@@ -825,7 +872,8 @@ private void checkQueryPlan(IgniteCache<Object, Object> cache,
         boolean enforceJoinOrder,
         int expBatchedJoins,
         String sql,
-        String...expText) {
+        String...expText
+    ) {
         checkQueryPlan(cache,
             enforceJoinOrder,
             expBatchedJoins,
@@ -850,13 +898,13 @@ private void checkQueryPlan(IgniteCache<Object, Object> cache,
         boolean enforceJoinOrder,
         int expBatchedJoins,
         SqlFieldsQuery qry,
-        String...expText) {
+        String... expText) {
         qry.setEnforceJoinOrder(enforceJoinOrder);
         qry.setDistributedJoins(true);
 
         String plan = queryPlan(cache, qry);
 
-        log.info("Plan: " + plan);
+        log.info("\n  Plan:\n" + plan);
 
         assertEquals("Unexpected number of batched joins in plan [plan=" + plan + ", qry=" + qry + ']',
             expBatchedJoins,
@@ -986,7 +1034,7 @@ private void doTestDistributedJoins(IgniteCache<Integer, Object> c, int orgs, in
      * @param args Arguments.
      * @return Column as list.
      */
-    private static <X> List<X> columnQuery(IgniteCache<?,?> c, String qry, Object... args) {
+    private static <X> List<X> columnQuery(IgniteCache<?, ?> c, String qry, Object... args) {
         return column(0, c.query(new SqlFieldsQuery(qry).setArgs(args)).getAll());
     }
 
@@ -1584,4 +1632,4 @@ private static class OrderGood implements Serializable {
         @QuerySqlField
         private int goodId;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
index 537ccdfd57953..ecdb593758f8a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
@@ -39,12 +39,8 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.h2.command.Prepared;
-import org.h2.command.dml.Query;
-import org.h2.command.dml.Update;
 import org.h2.engine.Session;
-import org.h2.expression.Expression;
 import org.h2.jdbc.JdbcConnection;
-import org.h2.util.StringUtils;
 
 import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
@@ -56,13 +52,19 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
     /** */
     private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
 
+    /** */
+    private static final String TEST_SCHEMA = "SCH";
+
+    /** */
+    private static final String TEST_CACHE = "my-cache";
+
     /** */
     private static Ignite ignite;
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration c = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
 
         TcpDiscoverySpi disco = new TcpDiscoverySpi();
 
@@ -73,12 +75,14 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
         // Cache.
         CacheConfiguration cc = defaultCacheConfiguration();
 
+        cc.setName(TEST_CACHE);
         cc.setCacheMode(CacheMode.PARTITIONED);
         cc.setAtomicityMode(CacheAtomicityMode.ATOMIC);
         cc.setNearConfiguration(null);
         cc.setWriteSynchronizationMode(FULL_SYNC);
         cc.setRebalanceMode(SYNC);
         cc.setSwapEnabled(false);
+        cc.setSqlSchema(TEST_SCHEMA);
         cc.setSqlFunctionClasses(GridQueryParsingTest.class);
         cc.setIndexedTypes(
             String.class, Address.class,
@@ -110,6 +114,12 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testParseSelectAndUnion() throws Exception {
+        checkQuery("select 1 from Person p where addrIds in ((1,2,3), (3,4,5))");
+        checkQuery("select 1 from Person p where addrId in ((1,))");
+        checkQuery("select 1 from Person p " +
+            "where p.addrId in (select a.id from Address a)");
+        checkQuery("select 1 from Person p " +
+            "where exists(select 1 from Address a where p.addrId = a.id)");
         checkQuery("select 42");
         checkQuery("select ()");
         checkQuery("select (1)");
@@ -244,16 +254,16 @@ public void testParseSelectAndUnion() throws Exception {
         checkQuery("select street from Person p, (select a.street from Address a where a.street is not null) ");
         checkQuery("select addr.street from Person p, (select a.street from Address a where a.street is not null) addr");
 
-        checkQuery("select p.name n from \"\".Person p order by p.old + 10");
+        checkQuery("select p.name n from sch.Person p order by p.old + 10");
 
-        checkQuery("select case when p.name is null then 'Vasya' end x from \"\".Person p");
-        checkQuery("select case when p.name like 'V%' then 'Vasya' else 'Other' end x from \"\".Person p");
-        checkQuery("select case when upper(p.name) = 'VASYA' then 'Vasya' when p.name is not null then p.name else 'Other' end x from \"\".Person p");
+        checkQuery("select case when p.name is null then 'Vasya' end x from sch.Person p");
+        checkQuery("select case when p.name like 'V%' then 'Vasya' else 'Other' end x from sch.Person p");
+        checkQuery("select case when upper(p.name) = 'VASYA' then 'Vasya' when p.name is not null then p.name else 'Other' end x from sch.Person p");
 
-        checkQuery("select case p.name when 'Vasya' then 1 end z from \"\".Person p");
-        checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 end z from \"\".Person p");
-        checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 else 3 end z from \"\".Person p");
-        checkQuery("select case p.name when 'Vasya' then 1 else 3 end z from \"\".Person p");
+        checkQuery("select case p.name when 'Vasya' then 1 end z from sch.Person p");
+        checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 end z from sch.Person p");
+        checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 else 3 end z from sch.Person p");
+        checkQuery("select case p.name when 'Vasya' then 1 else 3 end z from sch.Person p");
 
         checkQuery("select count(*) as a from Person union select count(*) as a from Address");
         checkQuery("select old, count(*) as a from Person group by old union select 1, count(*) as a from Address");
@@ -268,6 +278,54 @@ public void testParseSelectAndUnion() throws Exception {
         checkQuery("(select 2 a) union all (select 1) order by a desc nulls first limit ? offset ?");
     }
 
+    /**
+     * Query AST transformation heavily depends on this behavior.
+     *
+     * @throws Exception If failed.
+     */
+    public void testParseTableFilter() throws Exception {
+        Prepared prepared = parse("select Person.old, p1.old, p1.addrId from Person, Person p1 " +
+            "where exists(select 1 from Address a where a.id = p1.addrId)");
+
+        GridSqlSelect select = (GridSqlSelect)new GridSqlQueryParser().parse(prepared);
+
+        GridSqlJoin join = (GridSqlJoin)select.from();
+
+        GridSqlTable tbl1 = (GridSqlTable)join.leftTable();
+        GridSqlAlias tbl2Alias = (GridSqlAlias)join.rightTable();
+        GridSqlTable tbl2 = tbl2Alias.child();
+
+        // Must be distinct objects, even if it is the same table.
+        //assertNotSame(tbl1, tbl2);
+
+        assertNotNull(tbl1.dataTable());
+        assertNotNull(tbl2.dataTable());
+        assertSame(tbl1.dataTable(), tbl2.dataTable());
+
+        GridSqlColumn col1 = (GridSqlColumn)select.column(0);
+        GridSqlColumn col2 = (GridSqlColumn)select.column(1);
+
+        assertSame(tbl1, col1.expressionInFrom());
+
+        // Alias in FROM must be included in column.
+        assertSame(tbl2Alias, col2.expressionInFrom());
+
+        // In EXISTS we must correctly reference the column from the outer query.
+        GridSqlElement exists = select.where();
+        GridSqlSubquery subqry = exists.child();
+        GridSqlSelect subSelect = (GridSqlSelect)subqry.select();
+
+        GridSqlColumn p1AddrIdCol = (GridSqlColumn)select.column(2);
+
+        assertEquals("ADDRID", p1AddrIdCol.column().getName());
+        assertSame(tbl2Alias, p1AddrIdCol.expressionInFrom());
+
+        GridSqlColumn p1AddrIdColExists = subSelect.where().child(1);
+        assertEquals("ADDRID", p1AddrIdCol.column().getName());
+
+        assertSame(tbl2Alias, p1AddrIdColExists.expressionInFrom());
+    }
+
     /** */
     public void testParseMerge() throws Exception {
         /* Plain rows w/functions, operators, defaults, and placeholders. */
@@ -378,7 +436,7 @@ private JdbcConnection connection() throws Exception {
 
         IgniteH2Indexing idx = U.field(qryProcessor, "idx");
 
-        return (JdbcConnection)idx.connectionForSpace(null);
+        return (JdbcConnection)idx.connectionForSpace(TEST_CACHE);
     }
 
     /**
@@ -455,6 +513,9 @@ public static class Person implements Serializable {
         @QuerySqlField(index = true)
         public int addrId;
 
+        @QuerySqlField
+        public Integer[] addrIds;
+
         @QuerySqlField(index = true)
         public int old;
     }

From b7ab27301b59bf93fc73b52fdf8e0bcf124fec1d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 6 Apr 2017 14:43:50 +0300
Subject: [PATCH 220/446] IGNITE-4832: Prevent service deployment on client by
 default when configuration is provided on startup. This closes #1748.

---
 .../service/GridServiceProcessor.java         |  10 +-
 .../GridServiceProcessorAbstractSelfTest.java |  11 ++
 ...rviceProcessorMultiNodeConfigSelfTest.java |  74 +++++++++-
 ...GridServiceProcessorMultiNodeSelfTest.java | 139 +++++++++++++++---
 4 files changed, 202 insertions(+), 32 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index bd815189fb5ba..a8af9832e9355 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -70,11 +70,11 @@
 import org.apache.ignite.internal.processors.cache.query.CacheQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
-import org.apache.ignite.internal.processors.continuous.AbstractContinuousMessage;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridEmptyIterator;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.SerializableTransient;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -92,7 +92,6 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
-import org.apache.ignite.internal.util.SerializableTransient;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
@@ -296,8 +295,13 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         if (cfgs != null) {
             Collection<IgniteInternalFuture<?>> futs = new ArrayList<>();
 
-            for (ServiceConfiguration c : ctx.config().getServiceConfiguration())
+            for (ServiceConfiguration c : cfgs) {
+                // Deploy only on server nodes by default.
+                if (c.getNodeFilter() == null)
+                    c.setNodeFilter(ctx.cluster().get().forServers().predicate());
+
                 futs.add(deploy(c));
+            }
 
             // Await for services to deploy.
             for (IgniteInternalFuture<?> f : futs)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
index 111cb714e8f32..0f79855518e3d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
@@ -129,6 +129,17 @@ protected void startExtraNodes(int cnt) throws Exception {
             startGrid(nodeCount() + i);
     }
 
+    /** */
+    protected void startExtraNodes(int servers, int clients) throws Exception {
+        startExtraNodes(servers);
+
+        for (int i = 0; i < clients; i++) {
+            final String nodeName = getTestGridName(nodeCount() + servers + i);
+
+            startGrid(nodeName, getConfiguration(nodeName).setClientMode(true));
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index b819cc93e6fb8..1bd3b035a446c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -19,7 +19,9 @@
 
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
+import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -33,6 +35,9 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
     /** Node singleton name. */
     private static final String NODE_SINGLE = "serviceConfigEachNode";
 
+    /** Node singleton name. */
+    private static final String NODE_SINGLE_BUT_CLIENT = "serviceConfigEachNodeButClient";
+
     /** Affinity service name. */
     private static final String AFFINITY = "serviceConfigAffinity";
 
@@ -46,7 +51,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        ServiceConfiguration[] arr = new ServiceConfiguration[3];
+        ServiceConfiguration[] arr = new ServiceConfiguration[4];
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -59,7 +64,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
         cfg = new ServiceConfiguration();
 
-        cfg.setName(NODE_SINGLE);
+        cfg.setName(NODE_SINGLE_BUT_CLIENT);
         cfg.setMaxPerNodeCount(1);
         cfg.setService(new DummyService());
 
@@ -76,6 +81,15 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
         arr[2] = cfg;
 
+        cfg = new ServiceConfiguration();
+
+        cfg.setName(NODE_SINGLE);
+        cfg.setMaxPerNodeCount(1);
+        cfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
+        cfg.setService(new DummyService());
+
+        arr[3] = cfg;
+
         return arr;
     }
 
@@ -91,6 +105,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
                         DummyService.cancelled(CLUSTER_SINGLE) == 0 &&
                         DummyService.started(NODE_SINGLE) == nodeCount() &&
                         DummyService.cancelled(NODE_SINGLE) == 0 &&
+                        DummyService.started(NODE_SINGLE_BUT_CLIENT) == nodeCount() &&
+                        DummyService.cancelled(NODE_SINGLE_BUT_CLIENT) == 0 &&
                         actualCount(AFFINITY, randomGrid().services().serviceDescriptors()) == 1;
                 }
             },
@@ -112,6 +128,13 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
         checkDeployOnEachNodeUpdateTopology(NODE_SINGLE);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
+        checkDeployOnEachNodeButClientUpdateTopology(NODE_SINGLE_BUT_CLIENT);
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -120,6 +143,10 @@ public void testAll() throws Exception {
 
         DummyService.reset();
 
+        checkDeployOnEachNodeButClientUpdateTopology(NODE_SINGLE_BUT_CLIENT);
+
+        DummyService.reset();
+
         checkDeployOnEachNodeUpdateTopology(NODE_SINGLE);
 
         DummyService.reset();
@@ -152,9 +179,7 @@ public void testAffinityUpdateTopology() throws Exception {
     private void checkSingletonUpdateTopology(String name) throws Exception {
         Ignite g = randomGrid();
 
-        int nodeCnt = 2;
-
-        startExtraNodes(nodeCnt);
+        startExtraNodes(2, 2);
 
         try {
             assertEquals(name, 0, DummyService.started(name));
@@ -165,7 +190,7 @@ private void checkSingletonUpdateTopology(String name) throws Exception {
             checkCount(name, g.services().serviceDescriptors(), 1);
         }
         finally {
-            stopExtraNodes(nodeCnt);
+            stopExtraNodes(4);
         }
     }
 
@@ -176,17 +201,21 @@ private void checkSingletonUpdateTopology(String name) throws Exception {
     private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         Ignite g = randomGrid();
 
-        int newNodes = 2;
+        int newNodes = 4;
 
         CountDownLatch latch = new CountDownLatch(newNodes);
 
         DummyService.exeLatch(name, latch);
 
-        startExtraNodes(newNodes);
+        startExtraNodes(2, 2);
 
         try {
             latch.await();
 
+            // Ensure service is deployed.
+            assertNotNull(grid(nodeCount() + newNodes - 1).services()
+                .serviceProxy(NODE_SINGLE_BUT_CLIENT, Service.class, false, 2000));
+
             assertEquals(name, newNodes, DummyService.started(name));
             assertEquals(name, 0, DummyService.cancelled(name));
 
@@ -196,4 +225,33 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
             stopExtraNodes(newNodes);
         }
     }
+
+    /**
+     * @param name Name.
+     * @throws Exception If failed.
+     */
+    private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Exception {
+        Ignite g = randomGrid();
+
+        int servers = 2;
+        int clients = 2;
+
+        CountDownLatch latch = new CountDownLatch(servers);
+
+        DummyService.exeLatch(name, latch);
+
+        startExtraNodes(servers, clients);
+
+        try {
+            latch.await();
+
+            assertEquals(name, servers, DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
+        }
+        finally {
+            stopExtraNodes(servers + clients);
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index 39336ef001a02..f7403dcc820ca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -21,7 +21,10 @@
 import junit.framework.TestCase;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteServices;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
 
 /**
  * Single node services test.
@@ -121,50 +124,144 @@ public void testAffinityDeployUpdateTopology() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testDeployOnEachNodeUpdateTopology() throws Exception {
-        String name = "serviceOnEachNodeUpdateTopology";
+    public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
-        Ignite g = randomGrid();
+        try {
+            final int prestartedNodes = nodeCount() + 1;
 
-        CountDownLatch latch = new CountDownLatch(nodeCount());
+            String name = "serviceOnEachNodeButClientUpdateTopology";
 
-        DummyService.exeLatch(name, latch);
+            Ignite g = randomGrid();
 
-        IgniteServices svcs = g.services().withAsync();
+            CountDownLatch latch = new CountDownLatch(nodeCount());
 
-        svcs.deployNodeSingleton(name, new DummyService());
+            DummyService.exeLatch(name, latch);
 
-        IgniteFuture<?> fut = svcs.future();
+            IgniteServices svcs = g.services().withAsync();
 
-        info("Deployed service: " + name);
+            svcs.deployNodeSingleton(name, new DummyService());
 
-        fut.get();
+            IgniteFuture<?> fut = svcs.future();
 
-        info("Finished waiting for service future: " + name);
+            info("Deployed service: " + name);
 
-        latch.await();
+            fut.get();
 
-        TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
-        TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            info("Finished waiting for service future: " + name);
 
-        int newNodes = 2;
+            latch.await();
 
-        latch = new CountDownLatch(newNodes);
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
 
-        DummyService.exeLatch(name, latch);
+            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
+            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+            int servers = 2;
+            int clients = 2;
+
+            latch = new CountDownLatch(servers);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                // Ensure service is deployed
+                assertNotNull(grid(prestartedNodes + servers - 1)
+                    .services().serviceProxy(name, Service.class, false, 2000));
+
+                TestCase.assertEquals(name, nodeCount() + servers, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
+            }
+            finally {
+                stopExtraNodes(servers + clients);
+            }
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
 
-        startExtraNodes(newNodes);
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachNodeUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
         try {
+            String name = "serviceOnEachNodeUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            final int prestartedNodes = nodeCount() + 1;
+
+            CountDownLatch latch = new CountDownLatch(prestartedNodes);
+
+            DummyService.exeLatch(name, latch);
+
+            ServiceConfiguration srvcCfg = new ServiceConfiguration();
+
+            srvcCfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
+            srvcCfg.setName(name);
+            srvcCfg.setMaxPerNodeCount(1);
+            srvcCfg.setService(new DummyService());
+
+            IgniteServices svcs = g.services().withAsync();
+
+            svcs.deploy(srvcCfg);
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
             latch.await();
 
-            TestCase.assertEquals(name, nodeCount() + newNodes, DummyService.started(name));
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+            TestCase.assertEquals(name, prestartedNodes, DummyService.started(name));
             TestCase.assertEquals(name, 0, DummyService.cancelled(name));
 
-            checkCount(name, g.services().serviceDescriptors(), nodeCount() + newNodes);
+            int servers = 2;
+            int clients = 2;
+
+            int extraNodes = servers + clients;
+
+            latch = new CountDownLatch(extraNodes);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                // Ensure service is deployed
+                assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+                TestCase.assertEquals(name, prestartedNodes + extraNodes, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), prestartedNodes + extraNodes);
+            }
+            finally {
+                stopExtraNodes(extraNodes);
+            }
         }
         finally {
-            stopExtraNodes(newNodes);
+            stopGrid("client");
         }
     }
 }
\ No newline at end of file

From b214211eb3461746b6931c0623e086bb208e5dda Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Thu, 6 Apr 2017 15:00:12 +0300
Subject: [PATCH 221/446] IGNITE-4644 Value from IgniteQueue in atomic mode
 could be lost

---
 .../java/org/apache/ignite/IgniteSystemProperties.java     | 5 +++++
 .../datastructures/GridAtomicCacheQueueImpl.java           | 7 +++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index f0270986fc4b5..2f88d47a4c285 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -304,6 +304,11 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_ATOMIC_DEFERRED_ACK_TIMEOUT = "IGNITE_ATOMIC_DEFERRED_ACK_TIMEOUT";
 
+    /**
+     * Atomic cache deferred update timeout.
+     */
+    public static final String IGNITE_ATOMIC_CACHE_QUEUE_RETRY_TIMEOUT = "IGNITE_ATOMIC_CACHE_QUEUE_RETRY_TIMEOUT";
+
     /**
      * One phase commit deferred ack request timeout.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridAtomicCacheQueueImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridAtomicCacheQueueImpl.java
index 58d3efefd7605..3cee09e0be28a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridAtomicCacheQueueImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridAtomicCacheQueueImpl.java
@@ -28,12 +28,14 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_ATOMIC_CACHE_QUEUE_RETRY_TIMEOUT;
+
 /**
  * {@link org.apache.ignite.IgniteQueue} implementation using atomic cache.
  */
 public class GridAtomicCacheQueueImpl<T> extends GridCacheQueueAdapter<T> {
     /** */
-    private static final long RETRY_TIMEOUT = 3000;
+    private static final long RETRY_TIMEOUT = Integer.getInteger(IGNITE_ATOMIC_CACHE_QUEUE_RETRY_TIMEOUT, 10000);
 
     /**
      * @param queueName Queue name.
@@ -94,7 +96,8 @@ public GridAtomicCacheQueueImpl(String queueName, GridCacheQueueHeader hdr, Grid
                         return data;
                 }
 
-                U.warn(log, "Failed to get item, will retry poll [queue=" + queueName + ", idx=" + idx + ']');
+                U.warn(log, "Failed to get item due to poll timeout [queue=" + queueName + ", idx=" + idx + "]. " +
+                    "Poll timeout can be redefined by 'IGNITE_ATOMIC_CACHE_QUEUE_RETRY_TIMEOUT' system property.");
             }
         }
         catch (IgniteCheckedException e) {

From 014161427fb603b6df7c8ecc3c0904f5df47a21d Mon Sep 17 00:00:00 2001
From: Denis Magda <dmagda@gridgain.com>
Date: Mon, 13 Feb 2017 20:33:32 -0500
Subject: [PATCH 222/446] IGNITE-4159: Kubernetes IP finder. (cherry picked
 from commit 37c0a22)

---
 modules/kubernetes/DEVNOTES.txt               |  63 ++++
 modules/kubernetes/README.txt                 |  33 ++
 modules/kubernetes/config/Dockerfile          |  45 +++
 modules/kubernetes/config/example-kube.xml    |  44 +++
 .../kubernetes/config/ignite-deployment.yaml  |  26 ++
 modules/kubernetes/config/ignite-service.yaml |  14 +
 modules/kubernetes/config/run.sh              |  50 +++
 modules/kubernetes/licenses/apache-2.0.txt    | 202 +++++++++++
 modules/kubernetes/pom.xml                    |  93 ++++++
 .../TcpDiscoveryKubernetesIpFinder.java       | 315 ++++++++++++++++++
 .../tcp/ipfinder/kubernetes/package-info.java |  22 ++
 ...cpDiscoveryKubernetesIpFinderSelfTest.java |  93 ++++++
 .../tcp/ipfinder/kubernetes/package-info.java |  22 ++
 .../testsuites/IgniteKubernetesTestSuite.java |  41 +++
 pom.xml                                       |   1 +
 15 files changed, 1064 insertions(+)
 create mode 100644 modules/kubernetes/DEVNOTES.txt
 create mode 100644 modules/kubernetes/README.txt
 create mode 100644 modules/kubernetes/config/Dockerfile
 create mode 100644 modules/kubernetes/config/example-kube.xml
 create mode 100644 modules/kubernetes/config/ignite-deployment.yaml
 create mode 100644 modules/kubernetes/config/ignite-service.yaml
 create mode 100644 modules/kubernetes/config/run.sh
 create mode 100644 modules/kubernetes/licenses/apache-2.0.txt
 create mode 100644 modules/kubernetes/pom.xml
 create mode 100644 modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
 create mode 100644 modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
 create mode 100644 modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinderSelfTest.java
 create mode 100644 modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
 create mode 100644 modules/kubernetes/src/test/java/org/apache/ignite/testsuites/IgniteKubernetesTestSuite.java

diff --git a/modules/kubernetes/DEVNOTES.txt b/modules/kubernetes/DEVNOTES.txt
new file mode 100644
index 0000000000000..b2a8173587bf7
--- /dev/null
+++ b/modules/kubernetes/DEVNOTES.txt
@@ -0,0 +1,63 @@
+Building and testing Kubernetes module
+=========================================
+
+The instructions provide a guidance on how to build and test Ignite Kubernetes IP finder in Kubernetes environment.
+
+To test the IP finder you have to build the whole Apache Ignite project, package the binary as a Docker image and
+feed the image to your kubernetes environment.
+
+Building Apache Ignite
+=========================
+
+Use the command below to assemble an Apache Ignite binary:
+    mvn clean package -Prelease -Dignite.edition=fabric-lgpl -DskipTests
+
+Note, if you alter the build instruction somehow make sure to update the files under 'config' folder if needed.
+
+Installing Docker and Minikube
+==============================
+
+Install Docker and Minikube for testing purpose in your development environment.
+
+Once this is done, make sure that Minikube sees Docker images registered locally:
+    eval $(minikube docker-env)
+
+Start Minikube:
+    minikube start --vm-driver=xhyve
+
+Assembling Apache Ignite Docker Image
+=====================================
+
+Create a folder for all the files to be placed in the Docker image and copy the following there:
+    - Apache Ignite binary in a zip form built at the step above.
+    - Dockerfile from `ignite-kubernetes/config/Dockerfile`.
+    - Ignite configuration with enabled Kubernetes IP finder from `ignite-kubernetes/config/example-kube.xml`.
+    - The executable file that will start an Ignite node process from `ignite-kubernetes/config/run.sh`
+
+Go to the folder and execute a command below to prepare the image:
+    docker build -t ignite-kube:v1 .
+
+Creating containerized Ignite pods and Ignite lookup service
+============================================================
+
+Start the Kubernetes service that is used for IP addresses lookup. Use `ignite-kubernetes/config/ignite-service.yaml`:
+	kubectl create -f {path_to}/ignite-service.yaml
+
+Create and deploy Ignite pods using `ignite-kubernetes/config/ignite-deployment.yaml` configuration:
+    kubectl create -f {path_to}/ignite-deployment.yaml
+
+Make sure that the pods were deployed and running properly:
+    kubectl get pod
+    kubectl logs {pod name}
+
+Increase or decrease number of Ignite pods checking that Kubernetes IP finder works as expected:
+    kubectl scale --replicas=4 -f {path_to}/ignite-deployment.yaml
+
+Docker Image Redeployment
+=========================
+
+If you need to redeploy the docker image after it gets updated and you prefer not to change the image version then
+delete a current Kubernetes Ignite deployment (don't delete the service):
+    kubectl delete deployment ignite-cluster
+
+After that you are free to build and deploy an updated docker image using the same commands as listed above.
diff --git a/modules/kubernetes/README.txt b/modules/kubernetes/README.txt
new file mode 100644
index 0000000000000..a9a5a092ce2bb
--- /dev/null
+++ b/modules/kubernetes/README.txt
@@ -0,0 +1,33 @@
+Apache Ignite Kubernetes Module
+------------------------
+
+Apache Ignite Kubernetes module provides a TCP Discovery IP Finder that uses a dedicated Kubernetes service
+for IP addresses lookup of Apache Ignite pods containerized by Kubernetes.
+
+To enable Kubernetes module when starting a standalone node, move 'optional/ignite-kubernetes' folder to
+'libs' folder before running 'ignite.{sh|bat}' script. The content of the module folder will
+be added to classpath in this case.
+
+Importing Kubernetes Module In Maven Project
+-------------------------------------
+
+If you are using Maven to manage dependencies of your project, you can add Kubernetes module
+dependency like this (replace '${ignite.version}' with actual Ignite version you are
+interested in):
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                        http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    ...
+    <dependencies>
+        ...
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-kubernetes</artifactId>
+            <version>${ignite.version}</version>
+        </dependency>
+        ...
+    </dependencies>
+    ...
+</project>
diff --git a/modules/kubernetes/config/Dockerfile b/modules/kubernetes/config/Dockerfile
new file mode 100644
index 0000000000000..4e08ce80e2147
--- /dev/null
+++ b/modules/kubernetes/config/Dockerfile
@@ -0,0 +1,45 @@
+# Use Java 8 image as default one.
+FROM java:8
+
+# Set Apache Ignite version.
+ENV IGNITE_VERSION 2.0.0-SNAPSHOT
+
+# Set IGNITE_HOME variable.
+ENV IGNITE_HOME /opt/ignite/apache-ignite-fabric-lgpl-${IGNITE_VERSION}-bin
+
+# Setting a path to an Apache Ignite configuration file. Used by run.sh script below.
+ENV CONFIG_URI ${IGNITE_HOME}/config/example-kube.xml
+
+# Make sure kubernetes lib is copied to 'libs' folder.
+ENV OPTION_LIBS ignite-kubernetes
+
+# Disabling quiet mode.
+ENV IGNITE_QUIET=false
+
+# Install or update needed tools.
+RUN apt-get update && apt-get install -y --no-install-recommends unzip
+
+# Creating and setting a working directory for following commands.
+WORKDIR /opt/ignite
+
+# Copying local Apache Ignite build to the docker image.
+COPY ./apache-ignite-fabric-lgpl-${IGNITE_VERSION}-bin.zip apache-ignite-fabric-lgpl-${IGNITE_VERSION}-bin.zip
+
+# Unpacking the build.
+RUN unzip apache-ignite-fabric-lgpl-${IGNITE_VERSION}-bin.zip
+RUN rm apache-ignite-fabric-lgpl-${IGNITE_VERSION}-bin.zip
+
+# Copying the executable file and setting permissions.
+COPY ./run.sh $IGNITE_HOME/
+RUN chmod +x $IGNITE_HOME/run.sh
+
+# Copying the configuration.
+COPY ./example-kube.xml $IGNITE_HOME/config
+
+# Starting an Apache Ignite node.
+CMD $IGNITE_HOME/run.sh
+
+# Exposing the ports.
+EXPOSE 11211 47100 47500 49112
+
+
diff --git a/modules/kubernetes/config/example-kube.xml b/modules/kubernetes/config/example-kube.xml
new file mode 100644
index 0000000000000..bc04463f71a88
--- /dev/null
+++ b/modules/kubernetes/config/example-kube.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    Configuration example with Kubernetes IP finder enabled.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+    <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+
+        <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. -->
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <!--
+                        Enables Kubernetes IP finder with default settings.
+                    -->
+                    <bean
+                        class="org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder"/>
+                </property>
+            </bean>
+        </property>
+    </bean>
+</beans>
diff --git a/modules/kubernetes/config/ignite-deployment.yaml b/modules/kubernetes/config/ignite-deployment.yaml
new file mode 100644
index 0000000000000..ed5c102a666a6
--- /dev/null
+++ b/modules/kubernetes/config/ignite-deployment.yaml
@@ -0,0 +1,26 @@
+# An example of a Kubernetes configuration for Ignite pods deployment.
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  # Custom Ignite cluster's name.
+  name: ignite-cluster
+spec:
+  # Number of nodes to be started by Kubernetes initially.
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        # Must be equal to Ignite's Kubernetes service name.
+        app: ignite
+    spec:
+      containers:
+        # Custom Ignite node's pod  name.
+      - name: ignite-node
+        # Custom Ignite Docker image name.
+        image: ignite-kube:v1
+        ports:
+        # Ports to open.
+        - containerPort: 11211 # REST port number.
+        - containerPort: 47100 # communication SPI port number.
+        - containerPort: 47500 # discovery SPI port number.
+        - containerPort: 49112 # JMX port number.
diff --git a/modules/kubernetes/config/ignite-service.yaml b/modules/kubernetes/config/ignite-service.yaml
new file mode 100644
index 0000000000000..07b751624cbbf
--- /dev/null
+++ b/modules/kubernetes/config/ignite-service.yaml
@@ -0,0 +1,14 @@
+# An example of a Kubernetes configuration for Ignite lookup service deployment.
+apiVersion: v1
+kind: Service
+metadata:
+  # Name of Ignite Service used by Kubernetes IP finder for IP addresses lookup.
+  # The name must be equal to TcpDiscoveryKubernetesIpFinder.setServiceName parameter.
+  name: ignite
+spec:
+  clusterIP: None
+  ports:
+    - port: 9042 # some custom port (optional).
+  selector:
+    # Must be equal to the label set for Ignite pods.
+    app: ignite
diff --git a/modules/kubernetes/config/run.sh b/modules/kubernetes/config/run.sh
new file mode 100644
index 0000000000000..dbf287118aa41
--- /dev/null
+++ b/modules/kubernetes/config/run.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ ! -z "$OPTION_LIBS" ]; then
+  IFS=, LIBS_LIST=("$OPTION_LIBS")
+
+  for lib in ${LIBS_LIST[@]}; do
+    cp -r $IGNITE_HOME/libs/optional/"$lib"/* \
+        $IGNITE_HOME/libs/
+  done
+fi
+
+if [ ! -z "$EXTERNAL_LIBS" ]; then
+  IFS=, LIBS_LIST=("$EXTERNAL_LIBS")
+
+  for lib in ${LIBS_LIST[@]}; do
+    echo $lib >> temp
+  done
+
+  wget -i temp -P $IGNITE_HOME/libs
+
+  rm temp
+fi
+
+QUIET=""
+
+if [ "$IGNITE_QUIET" = "false" ]; then
+  QUIET="-v"
+fi
+
+if [ -z $CONFIG_URI ]; then
+  $IGNITE_HOME/bin/ignite.sh $QUIET
+else
+  $IGNITE_HOME/bin/ignite.sh $QUIET $CONFIG_URI
+fi
diff --git a/modules/kubernetes/licenses/apache-2.0.txt b/modules/kubernetes/licenses/apache-2.0.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/modules/kubernetes/licenses/apache-2.0.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/modules/kubernetes/pom.xml b/modules/kubernetes/pom.xml
new file mode 100644
index 0000000000000..5d4e5f0c8b509
--- /dev/null
+++ b/modules/kubernetes/pom.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    POM file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.ignite</groupId>
+        <artifactId>ignite-parent</artifactId>
+        <version>1</version>
+        <relativePath>../../parent</relativePath>
+    </parent>
+
+    <artifactId>ignite-kubernetes</artifactId>
+    <version>2.0.0-SNAPSHOT</version>
+    <url>http://ignite.apache.org</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.codehaus.jackson</groupId>
+            <artifactId>jackson-core-asl</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.codehaus.jackson</groupId>
+            <artifactId>jackson-mapper-asl</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-core</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-beans</artifactId>
+            <version>${spring.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+            <version>${spring.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-core</artifactId>
+            <version>${spring.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>
diff --git a/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
new file mode 100644
index 0000000000000..a5bd24f0affcc
--- /dev/null
+++ b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes;
+
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+
+/**
+ * IP finder for automatic lookup of Ignite nodes running in Kubernetes environment. All Ignite nodes have to deployed
+ * as Kubernetes pods in order to be discovered. An application that uses Ignite client nodes as a gateway to the
+ * cluster is required to be containerized as well. Applications and Ignite nodes running outside of Kubernetes will
+ * not be able to reach the containerized counterparts.
+ * <p>
+ * The implementation is based on a dedicated Kubernetes service that has to be created and should be deployed prior
+ * Ignite nodes startup. The service will maintain a list of all endpoints (internal IP addresses) of all containerized
+ * Ignite pods running so far. The name of the service must be equal to {@link #setServiceName(String)} which is
+ * `ignite` by default.
+ * <p>
+ * As for Ignite pods, it's recommended to label them in such a way that the service will use the label in its selector
+ * configuration excluding endpoints of irrelevant Kubernetes pods running in parallel.
+ * <p>
+ * The IP finder, in its turn, will call this service to retrieve Ignite pods IP addresses. The port will be
+ * either the one that is set with {@link TcpDiscoverySpi#setLocalPort(int)} or {@link TcpDiscoverySpi#DFLT_PORT}.
+ * Make sure that all Ignite pods occupy a similar discovery port, otherwise they will not be able to discover each
+ * other using this IP finder.
+ * <h2 class="header">Optional configuration</h2>
+ * <ul>
+ *      <li>The Kubernetes service name for IP addresses lookup (see {@link #setServiceName(String)})</li>
+ *      <li>The Kubernetes service namespace for IP addresses lookup (see {@link #setNamespace(String)}</li>
+ *      <li>The host name of the Kubernetes API server (see {@link #setMasterUrl(String)})</li>
+ *      <li>Path to the service token (see {@link #setAccountToken(String)}</li>
+ * </ul>
+ * <p>
+ * Both {@link #registerAddresses(Collection)} and {@link #unregisterAddresses(Collection)} have no effect.
+ * <p>
+ * Note, this IP finder is only workable when it used in Kubernetes environment.
+ * Choose another implementation of {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} for local
+ * or home network tests.
+ */
+public class TcpDiscoveryKubernetesIpFinder extends TcpDiscoveryIpFinderAdapter {
+    /** Grid logger. */
+    @LoggerResource
+    private IgniteLogger log;
+
+    /** Init routine guard. */
+    private final AtomicBoolean initGuard = new AtomicBoolean();
+
+    /** Init routine latch. */
+    private final CountDownLatch initLatch = new CountDownLatch(1);
+
+    /** Trust manager. */
+    private TrustManager[] trustAll = new TrustManager[] {
+        new X509TrustManager() {
+            public void checkServerTrusted(X509Certificate[] certs, String authType) {}
+            public void checkClientTrusted(X509Certificate[] certs, String authType) {}
+            public X509Certificate[] getAcceptedIssuers() { return null; }
+        }
+    };
+
+    /** Host verifier. */
+    private HostnameVerifier trustAllHosts = new HostnameVerifier() {
+        public boolean verify(String hostname, SSLSession session) {
+            return true;
+        }
+    };
+
+    /** Ignite's Kubernetes Service name. */
+    private String serviceName = "ignite";
+
+    /** Ignite Pod setNamespace name. */
+    private String namespace = "default";
+
+    /** Kubernetes API server URL in a string form. */
+    private String master = "https://kubernetes.default.svc.cluster.local:443";
+
+    /** Account token location. */
+    private String accountToken = "/var/run/secrets/kubernetes.io/serviceaccount/token";
+
+    /** Kubernetes API server URL. */
+    private URL url;
+
+    /** SSL context */
+    private SSLContext ctx;
+
+    /**
+     * Creates an instance of Kubernetes IP finder.
+     */
+    public TcpDiscoveryKubernetesIpFinder() {
+        setShared(true);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException {
+        init();
+
+        Collection<InetSocketAddress> addrs = new ArrayList<>();
+
+        try {
+            System.out.println("Getting Apache Ignite endpoints from: " + url);
+
+            HttpsURLConnection conn = (HttpsURLConnection)url.openConnection();
+
+            conn.setHostnameVerifier(trustAllHosts);
+
+            conn.setSSLSocketFactory(ctx.getSocketFactory());
+            conn.addRequestProperty("Authorization", "Bearer " + serviceAccountToken(accountToken));
+
+            // Sending the request and processing a response.
+            ObjectMapper mapper = new ObjectMapper();
+
+            Endpoints endpoints = mapper.readValue(conn.getInputStream(), Endpoints.class);
+
+            if (endpoints != null) {
+                if (endpoints.subsets != null && !endpoints.subsets.isEmpty()) {
+                    for (Subset subset : endpoints.subsets) {
+
+                        if (subset.addresses != null && !subset.addresses.isEmpty()) {
+                            for (Address address : subset.addresses) {
+                                addrs.add(new InetSocketAddress(address.ip, 0));
+
+                                System.out.println("Added an address to the list: " + address.ip);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        catch (Exception e) {
+            throw new IgniteSpiException("Failed to retrieve Ignite pods IP addresses.", e);
+        }
+
+        return addrs;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unregisterAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
+        // No-op
+    }
+
+    /**
+     * Sets the name of Kubernetes service for Ignite pods' IP addresses lookup. The name of the service must be equal
+     * to the name set in service's Kubernetes configuration. If this parameter is not changed then the name of the
+     * service has to be set to 'ignite' in the corresponding Kubernetes configuration.
+     *
+     * @param service Kubernetes service name for IP addresses lookup. If it's not set then 'ignite' is used by default.
+     */
+    public void setServiceName(String service) {
+        this.serviceName = service;
+    }
+
+    /**
+     * Sets the namespace the Kubernetes service belongs to. By default, it's supposed that the service is running under
+     * Kubernetes `default` namespace.
+     *
+     * @param namespace The Kubernetes service namespace for IP addresses lookup.
+     */
+    public void setNamespace(String namespace) {
+        this.namespace = namespace;
+    }
+
+    /**
+     * Sets the host name of the Kubernetes API server. By default the following host name is used:
+     * 'https://kubernetes.default.svc.cluster.local:443'.
+     *
+     * @param master The host name of the Kubernetes API server.
+     */
+    public void setMasterUrl(String master) {
+        this.master = master;
+    }
+
+    /**
+     * Specifies the path to the service token file. By default the following account token is used:
+     * '/var/run/secrets/kubernetes.io/serviceaccount/token'.
+     *
+     * @param accountToken The path to the service token file.
+     */
+    public void setAccountToken(String accountToken) {
+        this.accountToken = accountToken;
+    }
+
+    /**
+     * Kubernetes IP finder initialization.
+     *
+     * @throws IgniteSpiException In case of error.
+     */
+    private void init() throws IgniteSpiException {
+        if (initGuard.compareAndSet(false, true)) {
+
+            if (serviceName == null || serviceName.isEmpty() ||
+                namespace == null || namespace.isEmpty() ||
+                master == null || master.isEmpty() ||
+                accountToken == null || accountToken.isEmpty()) {
+                throw new IgniteSpiException(
+                    "One or more configuration parameters are invalid [setServiceName=" +
+                        serviceName + ", setNamespace=" + namespace + ", setMasterUrl=" +
+                        master + ", setAccountToken=" + accountToken + "]");
+            }
+
+            try {
+                // Preparing the URL and SSL context to be used for connection purposes.
+                String path = String.format("/api/v1/namespaces/%s/endpoints/%s", namespace, serviceName);
+
+                url = new URL(master + path);
+
+                ctx = SSLContext.getInstance("SSL");
+
+                ctx.init(null, trustAll, new SecureRandom());
+            }
+            catch (Exception e) {
+                throw new IgniteSpiException("Failed to connect to Ignite's Kubernetes Service.", e);
+            }
+            finally {
+                initLatch.countDown();
+            }
+        }
+        else {
+            try {
+                U.await(initLatch);
+            }
+            catch (IgniteInterruptedCheckedException e) {
+                throw new IgniteSpiException("Thread has been interrupted.", e);
+            }
+
+            if (url == null || ctx == null)
+                throw new IgniteSpiException("IP finder has not been initialized properly.");
+        }
+    }
+
+    /**
+     * Reads content of the service account token file.
+     *
+     * @param file The path to the service account token.
+     * @return Service account token.
+     */
+    private String serviceAccountToken(String file)  {
+        try {
+            return new String(Files.readAllBytes(Paths.get(file)));
+        } catch (IOException e) {
+            throw new IgniteSpiException("Failed to load services account token [setAccountToken= " + file + "]", e);
+        }
+    }
+
+    /**
+     * Object used by Jackson for processing of Kubernetes lookup service's response.
+     */
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private static class Address {
+        /** */
+        public String ip;
+    }
+
+    /**
+     * Object used by Jackson for processing of Kubernetes lookup service's response.
+     */
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private static class Subset {
+        /** */
+        public List<Address> addresses;
+    }
+
+    /**
+     * Object used by Jackson for processing of Kubernetes lookup service's response.
+     */
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private static class Endpoints {
+        /** */
+        public List<Subset> subsets;
+    }
+}
diff --git a/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
new file mode 100644
index 0000000000000..a572cb2d8cbc1
--- /dev/null
+++ b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains Kubernetes IP finder implementation.
+ */
+package org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes;
\ No newline at end of file
diff --git a/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinderSelfTest.java b/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinderSelfTest.java
new file mode 100644
index 0000000000000..fd3e2a35e432c
--- /dev/null
+++ b/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinderSelfTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes;
+
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAbstractSelfTest;
+
+/**
+ * TcpDiscoveryKubernetesIpFinder test.
+ */
+public class TcpDiscoveryKubernetesIpFinderSelfTest extends
+    TcpDiscoveryIpFinderAbstractSelfTest<TcpDiscoveryKubernetesIpFinder> {
+    /**
+     * Constructor.
+     *
+     * @throws Exception If any error occurs.
+     */
+    public TcpDiscoveryKubernetesIpFinderSelfTest() throws Exception {
+        // No-op.
+    }
+
+    @Override protected void beforeTest() throws Exception {
+        // No-op.
+    }
+
+    /* {@inheritDoc} */
+    @Override protected TcpDiscoveryKubernetesIpFinder ipFinder() throws Exception {
+        // No-op.
+        return null;
+    }
+
+    /* {@inheritDoc} */
+    @Override public void testIpFinder() throws Exception {
+        TcpDiscoveryKubernetesIpFinder ipFinder = new TcpDiscoveryKubernetesIpFinder();
+
+        ipFinder.setAccountToken(null);
+
+        try {
+            ipFinder.getRegisteredAddresses();
+        }
+        catch (IgniteSpiException e) {
+            assertTrue(e.getMessage().startsWith("One or more configuration parameters are invalid"));
+        }
+
+        ipFinder = new TcpDiscoveryKubernetesIpFinder();
+
+        ipFinder.setMasterUrl(null);
+
+        try {
+            ipFinder.getRegisteredAddresses();
+        }
+        catch (IgniteSpiException e) {
+            assertTrue(e.getMessage().startsWith("One or more configuration parameters are invalid"));
+        }
+
+        ipFinder = new TcpDiscoveryKubernetesIpFinder();
+
+        ipFinder.setNamespace(null);
+
+        try {
+            ipFinder.getRegisteredAddresses();
+        }
+        catch (IgniteSpiException e) {
+            assertTrue(e.getMessage().startsWith("One or more configuration parameters are invalid"));
+        }
+
+        ipFinder = new TcpDiscoveryKubernetesIpFinder();
+
+        ipFinder.setServiceName("");
+
+        try {
+            ipFinder.getRegisteredAddresses();
+        }
+        catch (IgniteSpiException e) {
+            assertTrue(e.getMessage().startsWith("One or more configuration parameters are invalid"));
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java b/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
new file mode 100644
index 0000000000000..83ab56ff75163
--- /dev/null
+++ b/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains Kubernetes IP finder internal tests.
+ */
+package org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes;
\ No newline at end of file
diff --git a/modules/kubernetes/src/test/java/org/apache/ignite/testsuites/IgniteKubernetesTestSuite.java b/modules/kubernetes/src/test/java/org/apache/ignite/testsuites/IgniteKubernetesTestSuite.java
new file mode 100644
index 0000000000000..540657e8b2601
--- /dev/null
+++ b/modules/kubernetes/src/test/java/org/apache/ignite/testsuites/IgniteKubernetesTestSuite.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+
+import junit.framework.TestSuite;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinderSelfTest;
+import org.apache.ignite.testframework.IgniteTestSuite;
+
+/**
+ * Ignite Kubernetes integration test.
+ */
+public class IgniteKubernetesTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        TestSuite suite = new IgniteTestSuite("Kubernetes Integration Test Suite");
+
+        // Cloud Nodes IP finder.
+        suite.addTestSuite(TcpDiscoveryKubernetesIpFinderSelfTest.class);
+
+        return suite;
+    }
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index d83bb1b969b9e..ef8573d3d5390 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,6 +88,7 @@
         <module>modules/web/ignite-websphere-test</module>
         <module>modules/cassandra</module>
         <module>modules/flink</module>
+        <module>modules/kubernetes</module>
     </modules>
 
     <profiles>

From 1db238402f11c67d2b28bfb7ff47955415f00c25 Mon Sep 17 00:00:00 2001
From: Denis Magda <dmagda@gridgain.com>
Date: Wed, 15 Feb 2017 23:37:26 -0500
Subject: [PATCH 223/446] IGNITE-4159: fixing logging (cherry picked from
 commit 06908d2) (cherry picked from commit fa27ee3)

---
 .../kubernetes/TcpDiscoveryKubernetesIpFinder.java        | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
index a5bd24f0affcc..53b6df6a0b711 100644
--- a/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
+++ b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
@@ -52,7 +52,7 @@
  * cluster is required to be containerized as well. Applications and Ignite nodes running outside of Kubernetes will
  * not be able to reach the containerized counterparts.
  * <p>
- * The implementation is based on a dedicated Kubernetes service that has to be created and should be deployed prior
+ * The implementation is based on a distinct Kubernetes service that has to be created and should be deployed prior
  * Ignite nodes startup. The service will maintain a list of all endpoints (internal IP addresses) of all containerized
  * Ignite pods running so far. The name of the service must be equal to {@link #setServiceName(String)} which is
  * `ignite` by default.
@@ -137,7 +137,8 @@ public TcpDiscoveryKubernetesIpFinder() {
         Collection<InetSocketAddress> addrs = new ArrayList<>();
 
         try {
-            System.out.println("Getting Apache Ignite endpoints from: " + url);
+            if (log.isDebugEnabled())
+                log.debug("Getting Apache Ignite endpoints from: " + url);
 
             HttpsURLConnection conn = (HttpsURLConnection)url.openConnection();
 
@@ -159,7 +160,8 @@ public TcpDiscoveryKubernetesIpFinder() {
                             for (Address address : subset.addresses) {
                                 addrs.add(new InetSocketAddress(address.ip, 0));
 
-                                System.out.println("Added an address to the list: " + address.ip);
+                                if (log.isDebugEnabled())
+                                    log.debug("Added an address to the list: " + address.ip);
                             }
                         }
                     }

From 5dfe16f7e91374008b9f6dfbb899364f5b2e1164 Mon Sep 17 00:00:00 2001
From: Denis Magda <dmagda@gridgain.com>
Date: Tue, 14 Feb 2017 01:03:30 -0500
Subject: [PATCH 224/446] IGNITE-4159: using logger instead of
 system.out.println (cherry picked from commit b9bf77c)

---
 modules/kubernetes/config/example-kube.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/kubernetes/config/example-kube.xml b/modules/kubernetes/config/example-kube.xml
index bc04463f71a88..11309d8933b14 100644
--- a/modules/kubernetes/config/example-kube.xml
+++ b/modules/kubernetes/config/example-kube.xml
@@ -41,4 +41,4 @@
             </bean>
         </property>
     </bean>
-</beans>
+</beans>
\ No newline at end of file

From 6e596d1ef426b66abd866d011a8f5cf5c5c25124 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 6 Apr 2017 14:43:50 +0300
Subject: [PATCH 225/446] IGNITE-4832: Prevent service deployment on client by
 default when configuration is provided on startup. This closes #1748.

(cherry picked from commit b7ab273)
---
 .../service/GridServiceProcessor.java         |  10 +-
 .../GridServiceProcessorAbstractSelfTest.java |  11 ++
 ...rviceProcessorMultiNodeConfigSelfTest.java |  74 +++++++++-
 ...GridServiceProcessorMultiNodeSelfTest.java | 139 +++++++++++++++---
 4 files changed, 202 insertions(+), 32 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index adfbc116aa293..d0b2733473333 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -70,11 +70,11 @@
 import org.apache.ignite.internal.processors.cache.query.CacheQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
-import org.apache.ignite.internal.processors.continuous.AbstractContinuousMessage;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridEmptyIterator;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.SerializableTransient;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -92,7 +92,6 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
-import org.apache.ignite.internal.util.SerializableTransient;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
@@ -296,8 +295,13 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         if (cfgs != null) {
             Collection<IgniteInternalFuture<?>> futs = new ArrayList<>();
 
-            for (ServiceConfiguration c : ctx.config().getServiceConfiguration())
+            for (ServiceConfiguration c : cfgs) {
+                // Deploy only on server nodes by default.
+                if (c.getNodeFilter() == null)
+                    c.setNodeFilter(ctx.cluster().get().forServers().predicate());
+
                 futs.add(deploy(c));
+            }
 
             // Await for services to deploy.
             for (IgniteInternalFuture<?> f : futs)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
index 111cb714e8f32..0f79855518e3d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
@@ -129,6 +129,17 @@ protected void startExtraNodes(int cnt) throws Exception {
             startGrid(nodeCount() + i);
     }
 
+    /** */
+    protected void startExtraNodes(int servers, int clients) throws Exception {
+        startExtraNodes(servers);
+
+        for (int i = 0; i < clients; i++) {
+            final String nodeName = getTestGridName(nodeCount() + servers + i);
+
+            startGrid(nodeName, getConfiguration(nodeName).setClientMode(true));
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index b819cc93e6fb8..1bd3b035a446c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -19,7 +19,9 @@
 
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
+import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -33,6 +35,9 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
     /** Node singleton name. */
     private static final String NODE_SINGLE = "serviceConfigEachNode";
 
+    /** Node singleton name. */
+    private static final String NODE_SINGLE_BUT_CLIENT = "serviceConfigEachNodeButClient";
+
     /** Affinity service name. */
     private static final String AFFINITY = "serviceConfigAffinity";
 
@@ -46,7 +51,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        ServiceConfiguration[] arr = new ServiceConfiguration[3];
+        ServiceConfiguration[] arr = new ServiceConfiguration[4];
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -59,7 +64,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
         cfg = new ServiceConfiguration();
 
-        cfg.setName(NODE_SINGLE);
+        cfg.setName(NODE_SINGLE_BUT_CLIENT);
         cfg.setMaxPerNodeCount(1);
         cfg.setService(new DummyService());
 
@@ -76,6 +81,15 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
         arr[2] = cfg;
 
+        cfg = new ServiceConfiguration();
+
+        cfg.setName(NODE_SINGLE);
+        cfg.setMaxPerNodeCount(1);
+        cfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
+        cfg.setService(new DummyService());
+
+        arr[3] = cfg;
+
         return arr;
     }
 
@@ -91,6 +105,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
                         DummyService.cancelled(CLUSTER_SINGLE) == 0 &&
                         DummyService.started(NODE_SINGLE) == nodeCount() &&
                         DummyService.cancelled(NODE_SINGLE) == 0 &&
+                        DummyService.started(NODE_SINGLE_BUT_CLIENT) == nodeCount() &&
+                        DummyService.cancelled(NODE_SINGLE_BUT_CLIENT) == 0 &&
                         actualCount(AFFINITY, randomGrid().services().serviceDescriptors()) == 1;
                 }
             },
@@ -112,6 +128,13 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
         checkDeployOnEachNodeUpdateTopology(NODE_SINGLE);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
+        checkDeployOnEachNodeButClientUpdateTopology(NODE_SINGLE_BUT_CLIENT);
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -120,6 +143,10 @@ public void testAll() throws Exception {
 
         DummyService.reset();
 
+        checkDeployOnEachNodeButClientUpdateTopology(NODE_SINGLE_BUT_CLIENT);
+
+        DummyService.reset();
+
         checkDeployOnEachNodeUpdateTopology(NODE_SINGLE);
 
         DummyService.reset();
@@ -152,9 +179,7 @@ public void testAffinityUpdateTopology() throws Exception {
     private void checkSingletonUpdateTopology(String name) throws Exception {
         Ignite g = randomGrid();
 
-        int nodeCnt = 2;
-
-        startExtraNodes(nodeCnt);
+        startExtraNodes(2, 2);
 
         try {
             assertEquals(name, 0, DummyService.started(name));
@@ -165,7 +190,7 @@ private void checkSingletonUpdateTopology(String name) throws Exception {
             checkCount(name, g.services().serviceDescriptors(), 1);
         }
         finally {
-            stopExtraNodes(nodeCnt);
+            stopExtraNodes(4);
         }
     }
 
@@ -176,17 +201,21 @@ private void checkSingletonUpdateTopology(String name) throws Exception {
     private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         Ignite g = randomGrid();
 
-        int newNodes = 2;
+        int newNodes = 4;
 
         CountDownLatch latch = new CountDownLatch(newNodes);
 
         DummyService.exeLatch(name, latch);
 
-        startExtraNodes(newNodes);
+        startExtraNodes(2, 2);
 
         try {
             latch.await();
 
+            // Ensure service is deployed.
+            assertNotNull(grid(nodeCount() + newNodes - 1).services()
+                .serviceProxy(NODE_SINGLE_BUT_CLIENT, Service.class, false, 2000));
+
             assertEquals(name, newNodes, DummyService.started(name));
             assertEquals(name, 0, DummyService.cancelled(name));
 
@@ -196,4 +225,33 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
             stopExtraNodes(newNodes);
         }
     }
+
+    /**
+     * @param name Name.
+     * @throws Exception If failed.
+     */
+    private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Exception {
+        Ignite g = randomGrid();
+
+        int servers = 2;
+        int clients = 2;
+
+        CountDownLatch latch = new CountDownLatch(servers);
+
+        DummyService.exeLatch(name, latch);
+
+        startExtraNodes(servers, clients);
+
+        try {
+            latch.await();
+
+            assertEquals(name, servers, DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
+        }
+        finally {
+            stopExtraNodes(servers + clients);
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index 39336ef001a02..f7403dcc820ca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -21,7 +21,10 @@
 import junit.framework.TestCase;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteServices;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
 
 /**
  * Single node services test.
@@ -121,50 +124,144 @@ public void testAffinityDeployUpdateTopology() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testDeployOnEachNodeUpdateTopology() throws Exception {
-        String name = "serviceOnEachNodeUpdateTopology";
+    public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
-        Ignite g = randomGrid();
+        try {
+            final int prestartedNodes = nodeCount() + 1;
 
-        CountDownLatch latch = new CountDownLatch(nodeCount());
+            String name = "serviceOnEachNodeButClientUpdateTopology";
 
-        DummyService.exeLatch(name, latch);
+            Ignite g = randomGrid();
 
-        IgniteServices svcs = g.services().withAsync();
+            CountDownLatch latch = new CountDownLatch(nodeCount());
 
-        svcs.deployNodeSingleton(name, new DummyService());
+            DummyService.exeLatch(name, latch);
 
-        IgniteFuture<?> fut = svcs.future();
+            IgniteServices svcs = g.services().withAsync();
 
-        info("Deployed service: " + name);
+            svcs.deployNodeSingleton(name, new DummyService());
 
-        fut.get();
+            IgniteFuture<?> fut = svcs.future();
 
-        info("Finished waiting for service future: " + name);
+            info("Deployed service: " + name);
 
-        latch.await();
+            fut.get();
 
-        TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
-        TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            info("Finished waiting for service future: " + name);
 
-        int newNodes = 2;
+            latch.await();
 
-        latch = new CountDownLatch(newNodes);
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
 
-        DummyService.exeLatch(name, latch);
+            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
+            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+            int servers = 2;
+            int clients = 2;
+
+            latch = new CountDownLatch(servers);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                // Ensure service is deployed
+                assertNotNull(grid(prestartedNodes + servers - 1)
+                    .services().serviceProxy(name, Service.class, false, 2000));
+
+                TestCase.assertEquals(name, nodeCount() + servers, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
+            }
+            finally {
+                stopExtraNodes(servers + clients);
+            }
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
 
-        startExtraNodes(newNodes);
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachNodeUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
         try {
+            String name = "serviceOnEachNodeUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            final int prestartedNodes = nodeCount() + 1;
+
+            CountDownLatch latch = new CountDownLatch(prestartedNodes);
+
+            DummyService.exeLatch(name, latch);
+
+            ServiceConfiguration srvcCfg = new ServiceConfiguration();
+
+            srvcCfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
+            srvcCfg.setName(name);
+            srvcCfg.setMaxPerNodeCount(1);
+            srvcCfg.setService(new DummyService());
+
+            IgniteServices svcs = g.services().withAsync();
+
+            svcs.deploy(srvcCfg);
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
             latch.await();
 
-            TestCase.assertEquals(name, nodeCount() + newNodes, DummyService.started(name));
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+            TestCase.assertEquals(name, prestartedNodes, DummyService.started(name));
             TestCase.assertEquals(name, 0, DummyService.cancelled(name));
 
-            checkCount(name, g.services().serviceDescriptors(), nodeCount() + newNodes);
+            int servers = 2;
+            int clients = 2;
+
+            int extraNodes = servers + clients;
+
+            latch = new CountDownLatch(extraNodes);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                // Ensure service is deployed
+                assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+                TestCase.assertEquals(name, prestartedNodes + extraNodes, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), prestartedNodes + extraNodes);
+            }
+            finally {
+                stopExtraNodes(extraNodes);
+            }
         }
         finally {
-            stopExtraNodes(newNodes);
+            stopGrid("client");
         }
     }
 }
\ No newline at end of file

From 443ac9a7aa82af1359a03bcfc8f9212b108300e4 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 5 Apr 2017 15:01:02 +0300
Subject: [PATCH 226/446] IGNITE-4917: Fixed failure when accessing
 BinaryObjectBuilder field value serialized with OptimizedMarshaller . This
 closes #1736.

---
 .../internal/binary/builder/BinaryBuilderReader.java     | 8 ++++++++
 .../binary/mutabletest/GridBinaryTestClasses.java        | 9 +++++++++
 2 files changed, 17 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
index baaabd6a1a0bd..02264e3311529 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
@@ -476,6 +476,14 @@ public Object getValueQuickly(int pos, int len) {
                 return new BinaryPlainBinaryObject(binaryObj);
             }
 
+            case GridBinaryMarshaller.OPTM_MARSH: {
+                final BinaryHeapInputStream bin = BinaryHeapInputStream.create(arr, pos + 1);
+
+                final Object obj = BinaryUtils.doReadOptimized(bin, ctx, U.resolveClassLoader(ctx.configuration()));
+
+                return obj;
+            }
+
             default:
                 throw new BinaryObjectException("Invalid flag value: " + type);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
index 5ddb87dcefb31..0d4de7fa0e215 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
@@ -24,6 +24,7 @@
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Date;
@@ -141,6 +142,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Integer i_;
 
+        /** */
+        public BigInteger bi_;
+
         /** */
         public Long l_;
 
@@ -150,6 +154,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Double d_;
 
+        /** */
+        public BigDecimal bd_;
+
         /** */
         public Character c_;
 
@@ -267,9 +274,11 @@ public void setDefaultData() {
             b_ = 11;
             s_ = 22;
             i_ = 33;
+            bi_ = new BigInteger("33000000000000");
             l_ = 44L;
             f_ = 55f;
             d_ = 66d;
+            bd_ = new BigDecimal("33000000000000.123456789");
             c_ = 'e';
             z_ = true;
 

From 05f3c747921aed6838804d2f5f2c8d2bd7985337 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 5 Apr 2017 15:01:02 +0300
Subject: [PATCH 227/446] IGNITE-4917: Fixed failure when accessing
 BinaryObjectBuilder field value serialized with OptimizedMarshaller . This
 closes #1736.

(cherry picked from commit 443ac9a)
---
 .../internal/binary/builder/BinaryBuilderReader.java     | 8 ++++++++
 .../binary/mutabletest/GridBinaryTestClasses.java        | 9 +++++++++
 2 files changed, 17 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
index baaabd6a1a0bd..02264e3311529 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
@@ -476,6 +476,14 @@ public Object getValueQuickly(int pos, int len) {
                 return new BinaryPlainBinaryObject(binaryObj);
             }
 
+            case GridBinaryMarshaller.OPTM_MARSH: {
+                final BinaryHeapInputStream bin = BinaryHeapInputStream.create(arr, pos + 1);
+
+                final Object obj = BinaryUtils.doReadOptimized(bin, ctx, U.resolveClassLoader(ctx.configuration()));
+
+                return obj;
+            }
+
             default:
                 throw new BinaryObjectException("Invalid flag value: " + type);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
index 5ddb87dcefb31..0d4de7fa0e215 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
@@ -24,6 +24,7 @@
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Date;
@@ -141,6 +142,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Integer i_;
 
+        /** */
+        public BigInteger bi_;
+
         /** */
         public Long l_;
 
@@ -150,6 +154,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Double d_;
 
+        /** */
+        public BigDecimal bd_;
+
         /** */
         public Character c_;
 
@@ -267,9 +274,11 @@ public void setDefaultData() {
             b_ = 11;
             s_ = 22;
             i_ = 33;
+            bi_ = new BigInteger("33000000000000");
             l_ = 44L;
             f_ = 55f;
             d_ = 66d;
+            bd_ = new BigDecimal("33000000000000.123456789");
             c_ = 'e';
             z_ = true;
 

From 3d616799efb472227b3b313516e6b40729654631 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 10 Apr 2017 10:36:11 +0300
Subject: [PATCH 228/446] IGNITE-2466 - Use current NIO back pressure mechanism
 to limit received messages. Mark them process only when backups acknowledged.

(backport from 1.9.2)

(cherry picked from commit 220db882b466c03eadd148b3a19a0bf70d82d4a6)
---
 .../managers/communication/GridIoManager.java |  12 +-
 .../dht/atomic/GridDhtAtomicCache.java        |  23 ++-
 .../util/nio/GridNioBackPressureControl.java  |  39 ++++-
 .../util/nio/GridNioMessageTracker.java       |   7 +
 ...acheAtomicPrimarySyncBackPressureTest.java | 151 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   6 +
 6 files changed, 223 insertions(+), 15 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 2eda4b715ee01..a1315dbd96fb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -731,7 +731,7 @@ private void processP2PMessage(
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     GridMessageListener lsnr = listenerGet0(msg.topic());
 
@@ -745,7 +745,7 @@ private void processP2PMessage(
                     invokeListener(msg.policy(), lsnr, nodeId, obj);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
 
                     msgC.run();
                 }
@@ -780,12 +780,12 @@ private void processRegularMessage(
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     processRegularMessage0(msg, nodeId);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
 
                     msgC.run();
                 }
@@ -1141,12 +1141,12 @@ else if (log.isDebugEnabled()) {
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     unwindMessageSet(msgSet0, lsnr);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
                 }
             }
         };
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index c91f881b60378..7d360766dd0fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -84,6 +84,8 @@
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.nio.GridNioBackPressureControl;
+import org.apache.ignite.internal.util.nio.GridNioMessageTracker;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -92,14 +94,15 @@
 import org.apache.ignite.internal.util.typedef.CX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteOutClosure;
+import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
@@ -110,6 +113,7 @@
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
@@ -1908,8 +1912,23 @@ private void updateAllAsyncInternal0(
                         if (req.writeSynchronizationMode() != FULL_ASYNC)
                             req.cleanup(!node.isLocal());
 
-                        if (dhtFut != null)
+                        if (dhtFut != null) {
+                            if (req.writeSynchronizationMode() == PRIMARY_SYNC && !dhtFut.isDone()) {
+                                final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
+
+                                if (tracker != null && tracker instanceof GridNioMessageTracker) {
+                                    ((GridNioMessageTracker)tracker).onMessageReceived();
+
+                                    dhtFut.listen(new IgniteInClosure<IgniteInternalFuture<Void>>() {
+                                        @Override public void apply(IgniteInternalFuture<Void> fut) {
+                                            ((GridNioMessageTracker)tracker).onMessageProcessed();
+                                        }
+                                    });
+                                }
+                            }
+
                             ctx.mvcc().addAtomicFuture(dhtFut.version(), dhtFut);
+                        }
                     }
                     else
                         // Should remap all keys.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
index 96a1ab347d402..37d985f7ff100 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
@@ -17,14 +17,17 @@
 
 package org.apache.ignite.internal.util.nio;
 
+import org.apache.ignite.lang.IgniteRunnable;
+import org.jetbrains.annotations.Nullable;
+
 /**
  * Utility class that allows to ignore back-pressure control for threads that are processing messages.
  */
 public class GridNioBackPressureControl {
     /** Thread local flag indicating that thread is processing message. */
-    private static ThreadLocal<Boolean> threadProcMsg = new ThreadLocal<Boolean>() {
-        @Override protected Boolean initialValue() {
-            return Boolean.FALSE;
+    private static ThreadLocal<Holder> threadProcMsg = new ThreadLocal<Holder>() {
+        @Override protected Holder initialValue() {
+            return new Holder();
         }
     };
 
@@ -32,13 +35,35 @@ public class GridNioBackPressureControl {
      * @return Flag indicating whether current thread is processing message.
      */
     public static boolean threadProcessingMessage() {
-        return threadProcMsg.get();
+        return threadProcMsg.get().procMsg;
     }
 
     /**
      * @param processing Flag indicating whether current thread is processing message.
+     * @param tracker Thread local back pressure tracker of messages, associated with one connection.
+     */
+    public static void threadProcessingMessage(boolean processing, @Nullable IgniteRunnable tracker) {
+        Holder holder = threadProcMsg.get();
+
+        holder.procMsg = processing;
+        holder.tracker = tracker;
+    }
+
+    /**
+     * @return Thread local back pressure tracker of messages, associated with one connection.
      */
-    public static void threadProcessingMessage(boolean processing) {
-        threadProcMsg.set(processing);
+    @Nullable public static IgniteRunnable threadTracker() {
+        return threadProcMsg.get().tracker;
+    }
+
+    /**
+     *
+     */
+    private static class Holder {
+        /** Process message. */
+        private boolean procMsg;
+
+        /** Tracker. */
+        private IgniteRunnable tracker;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
index e02c7ca7fd6ee..f05ee0c60d045 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
@@ -92,6 +92,13 @@ public GridNioMessageTracker(GridNioSession ses, int msgQueueLimit) {
         }
     }
 
+    /**
+     *
+     */
+    public void onMessageProcessed() {
+        run();
+    }
+
     /**
      */
     public void onMessageReceived() {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
new file mode 100644
index 0000000000000..49e3e5ca07a15
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Checks that back-pressure control restricts uncontrolled growing
+ * of backup message queue. This means, if queue too big - any reads
+ * will be stopped until received acks from backup nodes.
+ */
+public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = new CacheConfiguration("cache");
+
+        ccfg.setBackups(1);
+
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
+        ccfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED);
+        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+        TestCommunicationSpi spi = new TestCommunicationSpi();
+
+        spi.setMessageQueueLimit(100);
+
+        cfg.setCommunicationSpi(spi);
+        cfg.setClientMode(gridName.contains("client"));
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientPut() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+
+        final Ignite client = startGrid("client");
+
+        checkBackPressure(client, srv1, srv2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testServerPut() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+
+        final Ignite client = startGrid("server3");
+
+        checkBackPressure(client, srv1, srv2);
+    }
+
+    /**
+     * @param client Producer node.
+     * @throws InterruptedException If failed.
+     */
+    private void checkBackPressure(Ignite client, final Ignite srv1, final Ignite srv2) throws Exception {
+        final IgniteCache<Integer, String> cache = client.cache("cache");
+
+        awaitPartitionMapExchange();
+
+        for (int i = 0; i < 10000; i++) {
+            cache.put(i, String.valueOf(i));
+
+            if (i % 100 == 0) {
+                int size1 = futuresNum(srv1);
+                int size2 = futuresNum(srv2);
+
+                assert size1 < 150 : size1;
+                assert size2 < 150 : size2;
+            }
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @return Size of the backup queue.
+     */
+    private int futuresNum(Ignite ignite) {
+        return ((IgniteKernal)ignite).context().cache().context().mvcc().atomicFutures().size();
+    }
+
+    /**
+     * Delays backup update acks.
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (((GridIoMessage)msg).message() instanceof GridDhtAtomicDeferredUpdateResponse)
+                sleep(100);
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     * @param millis Millis.
+     */
+    private static void sleep(long millis) {
+        try {
+            Thread.sleep(millis);
+        }
+        catch (InterruptedException e) {
+            throw new IgniteSpiException(e);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 2b446bbea8e6d..9fcf31a322f2a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -94,6 +94,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteSystemCacheOnClientTest;
 import org.apache.ignite.internal.processors.cache.MarshallerCacheJobRunNodeRestartTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheAffinityEarlyTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheGetFutureHangsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheNoValueClassOnServerNodeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheCreatePutMultiNodeSelfTest;
@@ -108,6 +109,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheMultiTxLockSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCrossCacheTxSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearTxPreloadSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheNearOnlyTxTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheNearReadCommittedTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridReplicatedTxPreloadTest;
 import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicLoadAllTest;
@@ -331,6 +333,10 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(MarshallerCacheJobRunNodeRestartTest.class);
 
+        suite.addTestSuite(IgniteCacheNearOnlyTxTest.class);
+
+        suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class);
+
         return suite;
     }
 }
\ No newline at end of file

From 2a88a7a7581465ff0a6f8733550e6d42d7f71a6c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 10 Apr 2017 10:54:37 +0300
Subject: [PATCH 229/446] IGNITE-4667 - Throw exception on starting client
 cache when indexed types cannot be loaded

---
 .../cache/CacheAffinitySharedManager.java     |  52 +--
 .../GridCachePartitionExchangeManager.java    |   4 +-
 .../processors/cache/GridCacheProcessor.java  |  42 ++-
 .../dht/atomic/GridDhtAtomicCache.java        |   3 +-
 .../GridDhtPartitionsExchangeFuture.java      |   2 +-
 ...niteClientCacheInitializationFailTest.java | 346 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   2 +
 7 files changed, 415 insertions(+), 36 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 2642d16c56fc5..84372febb2c91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -379,38 +379,48 @@ else if (req.start() && !req.clientStartOnly()) {
             Integer cacheId = CU.cacheId(req.cacheName());
 
             if (req.start()) {
-                cctx.cache().prepareCacheStart(req, fut.topologyVersion());
+                try {
+                    cctx.cache().prepareCacheStart(req, fut.topologyVersion());
 
-                if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
-                    if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
-                        U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
-                }
+                    if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
+                        if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
+                            U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+                    }
 
-                if (!crd || !lateAffAssign) {
-                    GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+                    if (!crd || !lateAffAssign) {
+                        GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
-                    if (cacheCtx != null && !cacheCtx.isLocal()) {
-                        boolean clientCacheStarted =
-                            req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
+                        if (cacheCtx != null && !cacheCtx.isLocal()) {
+                            boolean clientCacheStarted =
+                                req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
 
-                        if (clientCacheStarted)
-                            initAffinity(cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
-                        else if (!req.clientStartOnly()) {
-                            assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
+                            if (clientCacheStarted)
+                                initAffinity(cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
+                            else if (!req.clientStartOnly()) {
+                                assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
 
-                            GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
+                                GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
 
-                            assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
+                                assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
 
-                            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
-                                fut.discoveryEvent(), fut.discoCache());
+                                List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
+                                    fut.discoveryEvent(), fut.discoCache());
 
-                            aff.initialize(fut.topologyVersion(), assignment);
+                                aff.initialize(fut.topologyVersion(), assignment);
+                            }
                         }
                     }
+                    else
+                        initStartedCacheOnCoordinator(fut, cacheId);
+                }
+                catch (IgniteCheckedException | RuntimeException e) {
+                    U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
+                        "[cacheName=" + req.cacheName() + ']', e);
+
+                    cctx.cache().forceCloseCache(fut.topologyVersion(), req, e);
+
+                    throw e;
                 }
-                else
-                    initStartedCacheOnCoordinator(fut, cacheId);
             }
             else if (req.stop() || req.close()) {
                 cctx.cache().blockGateway(req);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 16b07694ed5c0..679395727123a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -241,11 +241,11 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                                     fut = affinityReadyFuture(req.cacheFutureTopologyVersion());
 
                                 if (fut == null || fut.isDone())
-                                    cctx.cache().completeStartFuture(req);
+                                    cctx.cache().completeStartFuture(req, null);
                                 else {
                                     fut.listen(new CI1<IgniteInternalFuture<?>>() {
                                         @Override public void apply(IgniteInternalFuture<?> fut) {
-                                            cctx.cache().completeStartFuture(req);
+                                            cctx.cache().completeStartFuture(req, null);
                                         }
                                     });
                                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 9487589aa4c05..5740ad17f0617 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1612,6 +1612,8 @@ public void prepareCacheStart(DynamicCacheChangeRequest req, AffinityTopologyVer
         assert req.start() : req;
         assert req.cacheType() != null : req;
 
+        String cacheName = maskNull(req.cacheName());
+
         prepareCacheStart(
             req.startCacheConfiguration(),
             req.nearCacheConfiguration(),
@@ -1622,7 +1624,7 @@ public void prepareCacheStart(DynamicCacheChangeRequest req, AffinityTopologyVer
             topVer
         );
 
-        DynamicCacheDescriptor desc = registeredCaches.get(maskNull(req.cacheName()));
+        DynamicCacheDescriptor desc = registeredCaches.get(cacheName);
 
         if (desc != null)
             desc.onStart();
@@ -1761,8 +1763,8 @@ private void stopGateway(DynamicCacheChangeRequest req) {
     /**
      * @param req Stop request.
      */
-    private void prepareCacheStop(DynamicCacheChangeRequest req) {
-        assert req.stop() || req.close() : req;
+    private void prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose) {
+        assert req.stop() || req.close() || forceClose : req;
 
         GridCacheAdapter<?, ?> cache = caches.remove(maskNull(req.cacheName()));
 
@@ -1779,18 +1781,35 @@ private void prepareCacheStop(DynamicCacheChangeRequest req) {
         }
     }
 
+    /**
+     * Closes cache even if it's not fully initialized (e.g. fail on cache init stage).
+     *
+     * @param topVer Completed topology version.
+     * @param req Change request.
+     * @param err Error.
+     */
+    void forceCloseCache(
+        AffinityTopologyVersion topVer,
+        DynamicCacheChangeRequest req,
+        Throwable err
+    ) {
+        onExchangeDone(topVer, Collections.singleton(req), err, true);
+    }
+
     /**
      * Callback invoked when first exchange future for dynamic cache is completed.
      *
      * @param topVer Completed topology version.
      * @param reqs Change requests.
      * @param err Error.
+     * @param forceClose Close cache despite flags in requests.
      */
     @SuppressWarnings("unchecked")
     public void onExchangeDone(
         AffinityTopologyVersion topVer,
         Collection<DynamicCacheChangeRequest> reqs,
-        Throwable err
+        Throwable err,
+        boolean forceClose
     ) {
         for (GridCacheAdapter<?, ?> cache : caches.values()) {
             GridCacheContext<?, ?> cacheCtx = cache.context();
@@ -1805,16 +1824,16 @@ public void onExchangeDone(
             }
         }
 
-        if (!F.isEmpty(reqs) && err == null) {
+        if (!F.isEmpty(reqs) && (err == null || forceClose)) {
             for (DynamicCacheChangeRequest req : reqs) {
                 String masked = maskNull(req.cacheName());
 
                 if (req.stop()) {
                     stopGateway(req);
 
-                    prepareCacheStop(req);
+                    prepareCacheStop(req, forceClose);
                 }
-                else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId())) {
+                else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId()) || forceClose) {
                     IgniteCacheProxy<?, ?> proxy = jCacheProxies.remove(masked);
 
                     if (proxy != null) {
@@ -1827,20 +1846,21 @@ else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId())) {
                         else {
                             proxy.context().gate().onStopped();
 
-                            prepareCacheStop(req);
+                            prepareCacheStop(req, forceClose);
                         }
                     }
                 }
 
-                completeStartFuture(req);
+                completeStartFuture(req, err);
             }
         }
     }
 
     /**
      * @param req Request to complete future for.
+     * @param err Error to be passed to futures.
      */
-    public void completeStartFuture(DynamicCacheChangeRequest req) {
+    public void completeStartFuture(DynamicCacheChangeRequest req, @Nullable Throwable err) {
         DynamicCacheStartFuture fut = (DynamicCacheStartFuture)pendingFuts.get(maskNull(req.cacheName()));
 
         assert req.deploymentId() != null;
@@ -1848,7 +1868,7 @@ public void completeStartFuture(DynamicCacheChangeRequest req) {
 
         if (fut != null && fut.deploymentId().equals(req.deploymentId()) &&
             F.eq(req.initiatingNodeId(), ctx.localNodeId()))
-            fut.onDone();
+            fut.onDone(err);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 7d360766dd0fc..5b9b1416d8f84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -457,7 +457,8 @@ else if (res.error() != null) {
 
     /** {@inheritDoc} */
     @Override public void stop() {
-        deferredUpdateMsgSnd.stop();
+        if (deferredUpdateMsgSnd != null)
+            deferredUpdateMsgSnd.stop();
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index f11f868cf0e33..9aa0755269b9f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1069,7 +1069,7 @@ private void sendPartitions(ClusterNode oldestNode) {
 
         cctx.exchange().onExchangeDone(this, err);
 
-        cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err);
+        cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err, false);
 
         if (super.onDone(res, err) && realExchange) {
             if (log.isDebugEnabled())
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
new file mode 100644
index 0000000000000..63dd57527183b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import javax.cache.Cache;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.query.GridQueryCancel;
+import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
+import org.apache.ignite.internal.processors.query.GridQueryIndexing;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.lang.GridCloseableIterator;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.indexing.IndexingQueryFilter;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Test checks whether cache initialization error on client side
+ * doesn't causes hangs and doesn't impact other caches.
+ */
+public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractTest {
+    /** Failed cache name. */
+    private static final String CACHE_NAME = "cache";
+
+    /** Atomic cache name. */
+    private static final String ATOMIC_CACHE_NAME = "atomic-cache";
+
+    /** Tx cache name. */
+    private static final String TX_CACHE_NAME = "tx-cache";
+
+    /** Near atomic cache name. */
+    private static final String NEAR_ATOMIC_CACHE_NAME = "near-atomic-cache";
+
+    /** Near tx cache name. */
+    private static final String NEAR_TX_CACHE_NAME = "near-tx-cache";
+
+    /** Failed caches. */
+    private static final Set<String> FAILED_CACHES;
+
+    static {
+        Set<String> set = new HashSet<>();
+
+        set.add(ATOMIC_CACHE_NAME);
+        set.add(TX_CACHE_NAME);
+        set.add(NEAR_ATOMIC_CACHE_NAME);
+        set.add(NEAR_TX_CACHE_NAME);
+
+        FAILED_CACHES = Collections.unmodifiableSet(set);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrid("server");
+        startGrid("client");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("server")) {
+            CacheConfiguration<Integer, String> ccfg1 = new CacheConfiguration<>();
+
+            ccfg1.setIndexedTypes(Integer.class, String.class);
+            ccfg1.setName(ATOMIC_CACHE_NAME);
+            ccfg1.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+            CacheConfiguration<Integer, String> ccfg2 = new CacheConfiguration<>();
+
+            ccfg2.setIndexedTypes(Integer.class, String.class);
+            ccfg2.setName(TX_CACHE_NAME);
+            ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+            cfg.setCacheConfiguration(ccfg1, ccfg2);
+        }
+        else {
+            GridQueryProcessor.idxCls = FailedIndexing.class;
+
+            cfg.setClientMode(true);
+        }
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicCacheInitialization() throws Exception {
+        checkCacheInitialization(ATOMIC_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTransactionalCacheInitialization() throws Exception {
+        checkCacheInitialization(TX_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicNearCacheInitialization() throws Exception {
+        checkCacheInitialization(NEAR_ATOMIC_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTransactionalNearCacheInitialization() throws Exception {
+        checkCacheInitialization(NEAR_TX_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkCacheInitialization(final String cacheName) throws Exception {
+        Ignite client = grid("client");
+
+        checkFailedCache(client, cacheName);
+
+        checkFineCache(client, CACHE_NAME + 1);
+
+        assertNull(client.cache(cacheName));
+        assertNull(client.getOrCreateCache(cacheName));
+
+        checkFineCache(client, CACHE_NAME + 2);
+    }
+
+    /**
+     * @param client Client.
+     * @param cacheName Cache name.
+     */
+    private void checkFineCache(Ignite client, String cacheName) {
+        IgniteCache<Integer, String> cache = client.getOrCreateCache(cacheName);
+
+        cache.put(1, "1");
+
+        assertEquals("1", cache.get(1));
+    }
+
+    /**
+     * @param client Client.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    private void checkFailedCache(final Ignite client, final String cacheName) {
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                IgniteCache<Integer, String> cache;
+
+                // Start cache with near enabled.
+                if (NEAR_ATOMIC_CACHE_NAME.equals(cacheName) || NEAR_TX_CACHE_NAME.equals(cacheName)) {
+                    CacheConfiguration<Integer, String> ccfg = new CacheConfiguration<Integer, String>(cacheName)
+                        .setNearConfiguration(new NearCacheConfiguration<Integer, String>());
+
+                    if (NEAR_TX_CACHE_NAME.equals(cacheName))
+                        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+                    cache = client.getOrCreateCache(ccfg);
+                }
+                else
+                    cache = client.cache(cacheName);
+
+
+                cache.put(1, "1");
+
+                assertEquals("1", cache.get(1));
+
+                return null;
+            }
+        }, CacheException.class, null);
+    }
+
+    /**
+     * To fail on cache start.
+     */
+    private static class FailedIndexing implements GridQueryIndexing {
+        /** {@inheritDoc} */
+        @Override public void start(GridKernalContext ctx, GridSpinBusyLock busyLock) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void stop() throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public QueryCursor<List<?>> queryTwoStep(GridCacheContext<?, ?> cctx, SqlFieldsQuery qry,
+            GridQueryCancel cancel) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public <K, V> QueryCursor<Cache.Entry<K, V>> queryTwoStep(GridCacheContext<?, ?> cctx,
+            SqlQuery qry) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public GridQueryFieldsResult queryLocalSqlFields(@Nullable String spaceName, String qry,
+            Collection<Object> params, IndexingQueryFilter filter, boolean enforceJoinOrder, int timeout,
+            GridQueryCancel cancel) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable String spaceName,
+            String qry, String alias, Collection<Object> params, GridQueryTypeDescriptor type,
+            IndexingQueryFilter filter) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalText(@Nullable String spaceName,
+            String qry, GridQueryTypeDescriptor type, IndexingQueryFilter filter) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void registerCache(GridCacheContext<?, ?> cctx,
+            CacheConfiguration<?, ?> ccfg) throws IgniteCheckedException {
+            if (FAILED_CACHES.contains(cctx.name()) && cctx.kernalContext().clientNode())
+                throw new IgniteCheckedException("Test query exception " + cctx.name());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void unregisterCache(CacheConfiguration<?, ?> ccfg) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean registerType(@Nullable String spaceName,
+            GridQueryTypeDescriptor desc) throws IgniteCheckedException {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void unregisterType(@Nullable String spaceName,
+            GridQueryTypeDescriptor type) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void store(@Nullable String spaceName, GridQueryTypeDescriptor type, CacheObject key,
+            CacheObject val, byte[] ver, long expirationTime) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void remove(@Nullable String spaceName, CacheObject key,
+            CacheObject val) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onSwap(@Nullable String spaceName, CacheObject key) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onUnswap(@Nullable String spaceName, CacheObject key,
+            CacheObject val) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void rebuildIndexes(@Nullable String spaceName, GridQueryTypeDescriptor type) {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public IndexingQueryFilter backupFilter(AffinityTopologyVersion topVer, int[] parts) {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onDisconnected(IgniteFuture<?> reconnectFut) {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public PreparedStatement prepareNativeStatement(String schema, String sql) throws SQLException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancelQueries(Collection<Long> queries) {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancelAllQueries() {
+
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 9fcf31a322f2a..914aedb40764a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -81,6 +81,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxPreloadNoWriteTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxReplicatedPeekModesTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxStoreValueTest;
+import org.apache.ignite.internal.processors.cache.IgniteClientCacheInitializationFailTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheFilterTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartNoExchangeTimeoutTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartSelfTest;
@@ -235,6 +236,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheAffinityEarlyTest.class);
         suite.addTestSuite(IgniteCacheCreatePutMultiNodeSelfTest.class);
         suite.addTestSuite(IgniteCacheCreatePutTest.class);
+        suite.addTestSuite(IgniteClientCacheInitializationFailTest.class);
 
         suite.addTestSuite(GridCacheTxLoadFromStoreOnLockSelfTest.class);
 

From ba6227be49c8a395a5632e9841a6acb65ae340b6 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 10 Apr 2017 11:40:17 +0300
Subject: [PATCH 230/446] IGNITE-2466 - Disable back-pressure for sender data
 nodes to avoid deadlock.

---
 .../dht/atomic/GridDhtAtomicCache.java           |  5 ++++-
 .../CacheAtomicPrimarySyncBackPressureTest.java  | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 5b9b1416d8f84..8a8221b428f13 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1914,7 +1914,10 @@ private void updateAllAsyncInternal0(
                             req.cleanup(!node.isLocal());
 
                         if (dhtFut != null) {
-                            if (req.writeSynchronizationMode() == PRIMARY_SYNC && !dhtFut.isDone()) {
+                            if (req.writeSynchronizationMode() == PRIMARY_SYNC
+                                // To avoid deadlock disable back-pressure for sender data node.
+                                && !ctx.discovery().cacheAffinityNode(ctx.discovery().node(nodeId), ctx.name())
+                                && !dhtFut.isDone()) {
                                 final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
 
                                 if (tracker != null && tracker instanceof GridNioMessageTracker) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
index 49e3e5ca07a15..30436f559f4ec 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
@@ -53,6 +54,8 @@ public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTe
         ccfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED);
         ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
 
+        ccfg.setNodeFilter(new NodeFilter());
+
         TestCommunicationSpi spi = new TestCommunicationSpi();
 
         spi.setMessageQueueLimit(100);
@@ -148,4 +151,17 @@ private static void sleep(long millis) {
             throw new IgniteSpiException(e);
         }
     }
+
+    /**
+     * Filters out server node producer.
+     */
+    private static class NodeFilter implements IgnitePredicate<ClusterNode> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode node) {
+            return !("server3".equals(node.attribute("org.apache.ignite.ignite.name")));
+        }
+    }
 }

From 315ff38eeef96f12954d6ff39c84d58b2b959667 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 6 Apr 2017 14:43:50 +0300
Subject: [PATCH 231/446] IGNITE-4879: Fixed System pool starvation while
 partition evicting.

---
 .../cache/distributed/dht/GridDhtLocalPartition.java          | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 3ce1dd83ae61e..a0ccc28c33b0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -294,8 +294,8 @@ void onRemoved(GridDhtCacheEntry entry) {
         // Make sure to remove exactly this entry.
         map.removeEntry(entry);
 
-        // Attempt to evict.
-        tryEvict();
+        // Attempt to evict in async way as multiple-threads can compete for same partition.
+        tryEvictAsync(true);
     }
 
     /**

From 89e9dbe484312c251f02c9fbe9698c3ac2e03df8 Mon Sep 17 00:00:00 2001
From: Alexander Fedotov <alexander.fedotoff@gmail.com>
Date: Mon, 10 Apr 2017 16:36:33 +0300
Subject: [PATCH 232/446] Fix
 org.apache.ignite.internal.processors.cache.expiry    
 .IgniteCacheExpiryPolicyAbstractTest#testNearExpiresWithCacheStore

---
 .../cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java      | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index 3339f65671c3e..41c915b774628 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -1061,6 +1061,9 @@ public void testNearExpiresOnClient() throws Exception {
      * @throws Exception If failed.
      */
     public void testNearExpiresWithCacheStore() throws Exception {
+        if(cacheMode() != PARTITIONED)
+            return;
+
         factory = CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 1));
 
         nearCache = true;

From 02b194268071b179d291b28472cef5d587e7558a Mon Sep 17 00:00:00 2001
From: Alexander Fedotov <alexander.fedotoff@gmail.com>
Date: Tue, 11 Apr 2017 12:00:59 +0300
Subject: [PATCH 233/446] Fix missing test resource directory for
 org.apache.ignite.spi.discovery.tcp    
 .TcpDiscoveryNodeAttributesUpdateOnReconnectTest.testReconnect

---
 modules/core/pom.xml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index f12cdfa3d510d..f786a2792523a 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -234,6 +234,9 @@
                     <exclude>**/*.java</exclude>
                 </excludes>
             </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+            </testResource>
         </testResources>
 
         <plugins>

From 20016a20f780eb3c21f249d3cb74d08018c4eea5 Mon Sep 17 00:00:00 2001
From: Alexander Fedotov <alexander.fedotoff@gmail.com>
Date: Tue, 11 Apr 2017 14:54:06 +0300
Subject: [PATCH 234/446] Fix
 org.apache.ignite.internal.processors.cache.expiry    
 .IgniteCacheExpiryPolicyAbstractTest#testNearExpiresWithCacheStore

---
 .../cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index 41c915b774628..e6737e43ce042 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -1087,9 +1087,11 @@ public void testNearExpiresWithCacheStore() throws Exception {
 
         cache.put(key, 1);
 
-        assertEquals(1, cache.localPeek(key, CachePeekMode.NEAR));
+        // Make entry cached in client NearCache.
         assertEquals(1, cache.get(key));
 
+        assertEquals(1, cache.localPeek(key, CachePeekMode.NEAR));
+
         waitExpired(key);
 
         for(int i = 0; i < gridCount(); i++)

From 465084da5b00dcfc056d338f5d0a24875ca2af08 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 12 Apr 2017 13:01:25 +0300
Subject: [PATCH 235/446] IGNITE-4907: Fixed excessive service instances can be
 started with dynamic deployment. This closes #1766.

(cherry picked from commit 0f7ef74)
---
 .../service/GridServiceProcessor.java         |  2 +-
 ...rviceProcessorMultiNodeConfigSelfTest.java | 95 ++++++++++++++++---
 ...GridServiceProcessorMultiNodeSelfTest.java | 61 ++++++++++++
 3 files changed, 146 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index d0b2733473333..37bffc3130a73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -972,7 +972,7 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
                         int perNodeCnt = totalCnt != 0 ? totalCnt / size : maxPerNodeCnt;
                         int remainder = totalCnt != 0 ? totalCnt % size : 0;
 
-                        if (perNodeCnt > maxPerNodeCnt && maxPerNodeCnt != 0) {
+                        if (perNodeCnt >= maxPerNodeCnt && maxPerNodeCnt != 0) {
                             perNodeCnt = maxPerNodeCnt;
                             remainder = 0;
                         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index 1bd3b035a446c..9da62c0abb702 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -17,11 +17,13 @@
 
 package org.apache.ignite.internal.processors.service;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
-import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -38,6 +40,9 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
     /** Node singleton name. */
     private static final String NODE_SINGLE_BUT_CLIENT = "serviceConfigEachNodeButClient";
 
+    /** Node singleton name. */
+    private static final String NODE_SINGLE_WITH_LIMIT = "serviceConfigWithLimit";
+
     /** Affinity service name. */
     private static final String AFFINITY = "serviceConfigAffinity";
 
@@ -51,7 +56,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        ServiceConfiguration[] arr = new ServiceConfiguration[4];
+        List<ServiceConfiguration> cfgs = new ArrayList<>();
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -60,7 +65,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new DummyService());
 
-        arr[0] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -68,7 +73,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setMaxPerNodeCount(1);
         cfg.setService(new DummyService());
 
-        arr[1] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -79,7 +84,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new AffinityService(AFFINITY_KEY));
 
-        arr[2] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -88,9 +93,18 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
         cfg.setService(new DummyService());
 
-        arr[3] = cfg;
+        cfgs.add(cfg);
+
+        cfg = new ServiceConfiguration();
 
-        return arr;
+        cfg.setName(NODE_SINGLE_WITH_LIMIT);
+        cfg.setMaxPerNodeCount(1);
+        cfg.setTotalCount(nodeCount() + 1);
+        cfg.setService(new DummyService());
+
+        cfgs.add(cfg);
+
+        return cfgs.toArray(new ServiceConfiguration[cfgs.size()]);
     }
 
     /** {@inheritDoc} */
@@ -107,6 +121,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
                         DummyService.cancelled(NODE_SINGLE) == 0 &&
                         DummyService.started(NODE_SINGLE_BUT_CLIENT) == nodeCount() &&
                         DummyService.cancelled(NODE_SINGLE_BUT_CLIENT) == 0 &&
+                        DummyService.started(NODE_SINGLE_WITH_LIMIT) >= nodeCount() &&
+                        DummyService.cancelled(NODE_SINGLE_WITH_LIMIT) == 0 &&
                         actualCount(AFFINITY, randomGrid().services().serviceDescriptors()) == 1;
                 }
             },
@@ -170,6 +186,59 @@ public void testAffinityUpdateTopology() throws Exception {
         finally {
             stopExtraNodes(nodeCnt);
         }
+
+        checkCount(AFFINITY, g.services().serviceDescriptors(), 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployLimits() throws Exception {
+        final Ignite g = randomGrid();
+
+        final String name = NODE_SINGLE_WITH_LIMIT;
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
+
+        int extraNodes = 2;
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        DummyService.exeLatch(name, latch);
+
+        startExtraNodes(extraNodes);
+
+        try {
+            latch.await();
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount() + 1);
+        }
+        finally {
+            stopExtraNodes(extraNodes);
+        }
+
+        assertEquals(name, 1, DummyService.cancelled(name));
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
+    }
+
+    /**
+     * @param srvcName Service name
+     * @param expectedDeps Expected number of service deployments
+     *
+     */
+    private boolean waitForDeployment(final String srvcName, final int expectedDeps) throws IgniteInterruptedCheckedException {
+        final Ignite g = randomGrid();
+
+        return GridTestUtils.waitForCondition(new GridAbsPredicateX() {
+            @Override public boolean applyx() {
+                return actualCount(srvcName, g.services().serviceDescriptors())  == expectedDeps;
+            }
+        }, 1500);
     }
 
     /**
@@ -212,10 +281,6 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         try {
             latch.await();
 
-            // Ensure service is deployed.
-            assertNotNull(grid(nodeCount() + newNodes - 1).services()
-                .serviceProxy(NODE_SINGLE_BUT_CLIENT, Service.class, false, 2000));
-
             assertEquals(name, newNodes, DummyService.started(name));
             assertEquals(name, 0, DummyService.cancelled(name));
 
@@ -224,6 +289,10 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         finally {
             stopExtraNodes(newNodes);
         }
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 
     /**
@@ -253,5 +322,9 @@ private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Ex
         finally {
             stopExtraNodes(servers + clients);
         }
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index f7403dcc820ca..d133cf299a756 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -264,4 +264,65 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
             stopGrid("client");
         }
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployLimits() throws Exception {
+            String name = "serviceWithLimitsUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            final int totalInstances = nodeCount() + 1;
+
+            CountDownLatch latch = new CountDownLatch(nodeCount());
+
+            DummyService.exeLatch(name, latch);
+
+            ServiceConfiguration srvcCfg = new ServiceConfiguration();
+
+            srvcCfg.setName(name);
+            srvcCfg.setMaxPerNodeCount(1);
+            srvcCfg.setTotalCount(totalInstances);
+            srvcCfg.setService(new DummyService());
+
+            IgniteServices svcs = g.services().withAsync();
+
+            svcs.deploy(srvcCfg);
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
+            latch.await();
+
+            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
+            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount());
+
+            int extraNodes = 2;
+
+            latch = new CountDownLatch(1);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(2);
+
+            try {
+                latch.await();
+
+                TestCase.assertEquals(name, totalInstances, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), totalInstances);
+            }
+            finally {
+                stopExtraNodes(extraNodes);
+            }
+    }
 }
\ No newline at end of file

From a20c307df1dd000309a273ef93231fdc41a2a81c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 13 Apr 2017 09:31:17 +0300
Subject: [PATCH 236/446] IGNITE-4891 - Fix. Key is deserialized during
 transactional get() even if withKeepBinary is set

(Backport from master)
---
 .../transactions/IgniteTxLocalAdapter.java    |   5 +-
 .../cache/CacheKeepBinaryTransactionTest.java | 121 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |   2 +
 3 files changed, 126 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index ea816fe6ce5f0..7f161859da101 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1823,8 +1823,9 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                         for (Iterator<KeyCacheObject> it = missed.keySet().iterator(); it.hasNext(); ) {
                             KeyCacheObject cacheKey = it.next();
 
-                            K keyVal =
-                                (K)(keepCacheObjects ? cacheKey : cacheKey.value(cacheCtx.cacheObjectContext(), false));
+                            K keyVal = (K)(keepCacheObjects ? cacheKey
+                                : cacheCtx.cacheObjectContext()
+                                    .unwrapBinaryIfNeeded(cacheKey, !deserializeBinary, false));
 
                             if (retMap.containsKey(keyVal))
                                 it.remove();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java
new file mode 100644
index 0000000000000..27d7c7f004330
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteBinary;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.TransactionConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+/**
+ * Test that no deserialization happens with binary objects and keepBinary set flag.
+ */
+public class CacheKeepBinaryTransactionTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TransactionConfiguration txCfg = new TransactionConfiguration();
+        txCfg.setDefaultTxConcurrency(TransactionConcurrency.OPTIMISTIC);
+        txCfg.setDefaultTxIsolation(TransactionIsolation.REPEATABLE_READ);
+
+        cfg.setTransactionConfiguration(txCfg);
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        CacheConfiguration ccfg = new CacheConfiguration("tx-cache");
+        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid(0);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBinaryGet() throws Exception {
+        IgniteEx ignite = grid(0);
+        IgniteCache<Object, Object> cache = ignite.cache("tx-cache").withKeepBinary();
+
+        try (Transaction tx = ignite.transactions().txStart()) {
+            BinaryObject key = ignite.binary().builder("test1")
+                .setField("id", 1).hashCode(1).build();
+
+            assertNull(cache.get(key));
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBinaryContains() throws Exception {
+        IgniteEx ignite = grid(0);
+        IgniteCache<Object, Object> cache = ignite.cache("tx-cache").withKeepBinary();
+
+        try (Transaction tx = ignite.transactions().txStart()) {
+            BinaryObject key = ignite.binary().builder("test2")
+                .setField("id", 1).hashCode(1).build();
+
+            assertFalse(cache.containsKey(key));
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBinaryPutGetContains() throws Exception {
+        IgniteEx ignite = grid(0);
+        IgniteCache<Object, Object> cache = ignite.cache("tx-cache").withKeepBinary();
+
+        try (Transaction tx = ignite.transactions().txStart()) {
+            IgniteBinary binary = ignite.binary();
+
+            BinaryObject key = binary.builder("test-key").setField("id", 1).hashCode(1).build();
+            BinaryObject val = binary.builder("test-val").setField("id", 22).build();
+
+            cache.put(key, val);
+
+            assertTrue(cache.containsKey(key));
+            assertEquals(val, cache.get(key));
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 7f0e23c29aa19..421676654e7d5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.CacheKeepBinaryTransactionTest;
 import org.apache.ignite.internal.processors.cache.CacheNearReaderUpdateTest;
 import org.apache.ignite.internal.processors.cache.CacheSerializableTransactionsTest;
 import org.apache.ignite.internal.processors.cache.GridCacheSwapSpaceSpiConsistencySelfTest;
@@ -50,6 +51,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheWriteBehindNoUpdateSelfTest.class);
         suite.addTestSuite(IgniteCachePutStackOverflowSelfTest.class);
         suite.addTestSuite(GridCacheSwapSpaceSpiConsistencySelfTest.class);
+        suite.addTestSuite(CacheKeepBinaryTransactionTest.class);
 
         suite.addTestSuite(CacheLateAffinityAssignmentTest.class);
         suite.addTestSuite(CacheLateAffinityAssignmentFairAffinityTest.class);

From 630558dfeb373f237057e394e8f2f63230d59dab Mon Sep 17 00:00:00 2001
From: vladisav <vladisavj@gmail.com>
Date: Thu, 13 Apr 2017 13:24:42 +0300
Subject: [PATCH 237/446] ignite-4173 IgniteSemaphore with failoverSafe enabled
 doesn't release permits in case permits owner node left topology

Backport from master.

(cherry picked from commit 76485fc)
---
 .../datastructures/GridCacheSemaphoreImpl.java       | 12 ++++++++++++
 .../SemaphoreFailoverSafeReleasePermitsTest.java     |  9 ++++-----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
index a11c79db6282c..e3cfd778da4e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
@@ -532,6 +532,18 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public void onNodeRemoved(UUID nodeId) {
+        try {
+            initializeSemaphore();
+        }
+        catch (IgniteCheckedException e) {
+            U.error(log, "Failed to recover from failover because distributed semaphore cannot be initialized " +
+                "(Ignore if this node is failing also)." );
+
+            // Degrade gracefully, no exception is thrown
+            // because other semaphores might also attempt to recover from failover.
+            return;
+        }
+
         int numPermits = sync.getPermitsForNode(nodeId);
 
         if (numPermits > 0) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/SemaphoreFailoverSafeReleasePermitsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/SemaphoreFailoverSafeReleasePermitsTest.java
index 241253d3de8bd..9a588e7c419f7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/SemaphoreFailoverSafeReleasePermitsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/SemaphoreFailoverSafeReleasePermitsTest.java
@@ -92,6 +92,9 @@ private void doTest() throws Exception {
 
             IgniteSemaphore sem = ignite.semaphore("sem", 1, true, true);
 
+            // Initialize second semaphore before the first one is broken.
+            IgniteSemaphore sem2 = grid(1).semaphore("sem", 1, true, true);
+
             assertEquals(1, sem.availablePermits());
 
             sem.acquire(1);
@@ -102,11 +105,7 @@ private void doTest() throws Exception {
 
             awaitPartitionMapExchange();
 
-            ignite = grid(1);
-
-            sem = ignite.semaphore("sem", 1, true, true);
-
-            assertTrue(sem.tryAcquire(1, 5000, TimeUnit.MILLISECONDS));
+            assertTrue(sem2.tryAcquire(1, 5000, TimeUnit.MILLISECONDS));
         }
         finally {
             stopAllGrids();

From 870b752c095ed3776e91a65b99763142b9f2ebc0 Mon Sep 17 00:00:00 2001
From: Vladisav Jelisavcic <vladisavj@gmail.com>
Date: Tue, 11 Apr 2017 14:09:12 +0300
Subject: [PATCH 238/446] ignite-1977 - fixed IgniteSemaphore fault tolerance.

Backport from master.

(cherry picked from commit 902bf42)
---
 .../GridCacheSemaphoreImpl.java               | 74 ++++++++++++++++---
 .../GridCacheSemaphoreState.java              | 22 ++++++
 ...bstractDataStructuresFailoverSelfTest.java | 21 ++++--
 3 files changed, 102 insertions(+), 15 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
index e3cfd778da4e1..c7426300db5e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
@@ -232,6 +232,10 @@ final int nonfairTryAcquireShared(int acquires) {
 
         /** {@inheritDoc} */
         @Override protected final boolean tryReleaseShared(int releases) {
+            // Fail-fast path.
+            if(broken)
+                return true;
+
             // Check if some other node updated the state.
             // This method is called with release==0 only when trying to wake through update.
             if (releases == 0)
@@ -295,6 +299,13 @@ protected boolean compareAndSetGlobalState(final int expVal, final int newVal, f
                                     throw new IgniteCheckedException("Failed to find semaphore with given name: " +
                                         name);
 
+                                // Abort if state is already broken.
+                                if (val.isBroken()) {
+                                    tx.rollback();
+
+                                    return true;
+                                }
+
                                 boolean retVal = val.getCount() == expVal;
 
                                 if (retVal) {
@@ -349,11 +360,13 @@ protected boolean compareAndSetGlobalState(final int expVal, final int newVal, f
 
         /**
          * This method is used for releasing the permits acquired by failing node.
+         * In case the semaphore is broken, no permits are released and semaphore is set (globally) to broken state.
          *
          * @param nodeId ID of the failing node.
+         * @param broken Flag indicating that this semaphore is broken.
          * @return True if this is the call that succeeded to change the global state.
          */
-        protected boolean releaseFailedNode(final UUID nodeId) {
+        protected boolean releaseFailedNode(final UUID nodeId, final boolean broken) {
             try {
                 return CU.outTx(
                     retryTopologySafe(new Callable<Boolean>() {
@@ -369,6 +382,25 @@ protected boolean releaseFailedNode(final UUID nodeId) {
                                     throw new IgniteCheckedException("Failed to find semaphore with given name: " +
                                         name);
 
+                                // Quit early if semaphore is already broken.
+                                if( val.isBroken()) {
+                                    tx.rollback();
+
+                                    return false;
+                                }
+
+                                // Mark semaphore as broken. No permits are released,
+                                // since semaphore is useless from now on.
+                                if (broken) {
+                                    val.setBroken(true);
+
+                                    semView.put(key, val);
+
+                                    tx.commit();
+
+                                    return true;
+                                }
+
                                 Map<UUID, Integer> map = val.getWaiters();
 
                                 if (!map.containsKey(nodeId)) {
@@ -473,7 +505,11 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
                                 tx.commit();
 
-                                return new Sync(count, waiters, failoverSafe);
+                                Sync sync = new Sync(count, waiters, failoverSafe);
+
+                                sync.setBroken(val.isBroken());
+
+                                return sync;
                             }
                         }
                     }),
@@ -520,6 +556,9 @@ private void initializeSemaphore() throws IgniteCheckedException {
         if (sync == null)
             return;
 
+        // Update broken flag.
+        sync.setBroken(val.isBroken());
+
         // Update permission count.
         sync.setPermits(val.getCount());
 
@@ -547,10 +586,13 @@ private void initializeSemaphore() throws IgniteCheckedException {
         int numPermits = sync.getPermitsForNode(nodeId);
 
         if (numPermits > 0) {
-            if (sync.failoverSafe)
-                // Release permits acquired by threads on failing node.
-                sync.releaseFailedNode(nodeId);
-            else {
+            // Semaphore is broken if reaches this point in non-failover safe mode.
+            boolean broken = !sync.failoverSafe;
+
+            // Release permits acquired by threads on failing node.
+            sync.releaseFailedNode(nodeId, broken);
+
+            if (broken) {
                 // Interrupt every waiting thread if this semaphore is not failover safe.
                 sync.setBroken(true);
 
@@ -626,8 +668,11 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
             sync.acquireSharedInterruptibly(permits);
 
-            if (isBroken())
+            if (isBroken()) {
+                Thread.interrupted(); // Clear interrupt flag.
+
                 throw new InterruptedException();
+            }
         }
         catch (IgniteCheckedException e) {
             throw U.convertException(e);
@@ -743,8 +788,11 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
             boolean result = sync.nonfairTryAcquireShared(1) >= 0;
 
-            if (isBroken())
+            if (isBroken()) {
+                Thread.interrupted(); // Clear interrupt flag.
+
                 throw new InterruptedException();
+            }
 
             return result;
         }
@@ -768,8 +816,11 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
             boolean result = sync.tryAcquireSharedNanos(1, unit.toNanos(timeout));
 
-            if (isBroken())
+            if (isBroken()) {
+                Thread.interrupted(); // Clear interrupt flag.
+
                 throw new InterruptedException();
+            }
 
             return result;
         }
@@ -837,8 +888,11 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
             boolean result = sync.tryAcquireSharedNanos(permits, unit.toNanos(timeout));
 
-            if (isBroken())
+            if (isBroken()) {
+                Thread.interrupted();
+
                 throw new InterruptedException();
+            }
 
             return result;
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreState.java
index 50cdf102e0331..cdff9c51d7a35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreState.java
@@ -46,6 +46,9 @@ public class GridCacheSemaphoreState implements GridCacheInternal, Externalizabl
     /** FailoverSafe flag. */
     private boolean failoverSafe;
 
+    /** Flag indicating that semaphore is no longer safe to use. */
+    private boolean broken;
+
     /**
      * Constructor.
      *
@@ -101,6 +104,21 @@ public boolean isFailoverSafe() {
         return failoverSafe;
     }
 
+    /**
+     * @return broken flag.
+     */
+    public boolean isBroken() {
+        return broken;
+    }
+
+    /**
+     *
+     * @param broken Flag indicating that this semaphore should be no longer used.
+     */
+    public void setBroken(boolean broken) {
+        this.broken = broken;
+    }
+
     /** {@inheritDoc} */
     @Override public Object clone() throws CloneNotSupportedException {
         return super.clone();
@@ -120,6 +138,8 @@ public boolean isFailoverSafe() {
                 out.writeInt(e.getValue());
             }
         }
+
+        out.writeBoolean(broken);
     }
 
     /** {@inheritDoc} */
@@ -135,6 +155,8 @@ public boolean isFailoverSafe() {
             for (int i = 0; i < size; i++)
                 waiters.put(U.readUuid(in), in.readInt());
         }
+
+        broken = in.readBoolean();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
index 6ba65abf5dec9..9295770796381 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
@@ -529,8 +529,6 @@ public void testSemaphoreMultipleTopologyChangeNonFailoverSafe() throws Exceptio
      * @throws Exception If failed.
      */
     private void doTestSemaphore(ConstantTopologyChangeWorker topWorker, final boolean failoverSafe) throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-1977");
-
         final int permits = topWorker instanceof MultipleTopologyChangeWorker ||
             topWorker instanceof PartitionedMultipleTopologyChangeWorker ? TOP_CHANGE_THREAD_CNT * 3 :
             TOP_CHANGE_CNT;
@@ -547,9 +545,14 @@ private void doTestSemaphore(ConstantTopologyChangeWorker topWorker, final boole
                             break;
                         }
                         catch (IgniteInterruptedException e) {
-                            // Exception may happen in non failover safe mode.
+                           // Exception may happen in non failover safe mode.
                             if (failoverSafe)
                                 throw e;
+                            else {
+                                // In non-failoverSafe mode semaphore is not safe to be reused,
+                                // and should always be discarded after exception is caught.
+                                break;
+                            }
                         }
                     }
 
@@ -568,6 +571,11 @@ private void doTestSemaphore(ConstantTopologyChangeWorker topWorker, final boole
                         // Exception may happen in non failover safe mode.
                         if (failoverSafe)
                             throw e;
+                        else {
+                            // In non-failoverSafe mode semaphore is not safe to be reused,
+                            // and should always be discarded after exception is caught.
+                            break;
+                        }
                     }
                 }
 
@@ -580,8 +588,11 @@ private void doTestSemaphore(ConstantTopologyChangeWorker topWorker, final boole
 
             fut.get();
 
-            for (Ignite g : G.allGrids())
-                assertEquals(permits, g.semaphore(STRUCTURE_NAME, permits, false, false).availablePermits());
+            // Semaphore is left in proper state only if failoverSafe mode is used.
+            if (failoverSafe) {
+                for (Ignite g : G.allGrids())
+                    assertEquals(permits, g.semaphore(STRUCTURE_NAME, permits, false, false).availablePermits());
+            }
         }
     }
 

From cd0b92950c6691c6fc1a26cb4f7e55f5ee459298 Mon Sep 17 00:00:00 2001
From: Yakov Zhdanov <yzhdanov@gridgain.com>
Date: Thu, 13 Apr 2017 15:52:20 +0300
Subject: [PATCH 239/446] ignite-4946 GridCacheP2PUndeploySelfTest became
 failed

(cherry picked from commit d298e75)
---
 .../eventstorage/GridEventStorageManager.java | 341 ++++++++++--------
 1 file changed, 187 insertions(+), 154 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
index a2c64ba33d8b3..406da2d284b63 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
@@ -21,6 +21,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EventListener;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
@@ -33,6 +34,7 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteEvents;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.DiscoveryEvent;
@@ -78,10 +80,7 @@
  */
 public class GridEventStorageManager extends GridManagerAdapter<EventStorageSpi> {
     /** Local event listeners. */
-    private final ConcurrentMap<Integer, Set<GridLocalEventListener>> lsnrs = new ConcurrentHashMap8<>();
-
-    /** Internal discovery listeners. */
-    private final ConcurrentMap<Integer, Set<DiscoveryEventListener>> discoLsnrs = new ConcurrentHashMap8<>();
+    private final ConcurrentMap<Integer, Set<EventListener>> lsnrs = new ConcurrentHashMap8<>();
 
     /** Busy lock to control activity of threads. */
     private final ReadWriteLock busyLock = new ReentrantReadWriteLock();
@@ -200,7 +199,7 @@ public GridEventStorageManager(GridKernalContext ctx) {
     @Override public void printMemoryStats() {
         int lsnrsCnt = 0;
 
-        for (Set<GridLocalEventListener> lsnrs0 : lsnrs.values())
+        for (Set<EventListener> lsnrs0 : lsnrs.values())
             lsnrsCnt += lsnrs0.size();
 
         X.println(">>>");
@@ -238,7 +237,6 @@ private void leaveBusy() {
         msgLsnr = null;
 
         lsnrs.clear();
-        discoLsnrs.clear();
     }
 
     /** {@inheritDoc} */
@@ -274,6 +272,26 @@ private void leaveBusy() {
      * @param evt Event to record.
      */
     public void record(Event evt) {
+        record0(evt);
+    }
+
+    /**
+     * Records discovery events.
+     *
+     * @param evt Event to record.
+     * @param discoCache Discovery cache.
+     */
+    public void record(DiscoveryEvent evt, DiscoCache discoCache) {
+        record0(evt, discoCache);
+    }
+
+    /**
+     * Records event if it's recordable.
+     *
+     * @param evt Event to record.
+     * @param params Additional parameters.
+     */
+    private void record0(Event evt, Object... params) {
         assert evt != null;
 
         if (!enterBusy())
@@ -297,31 +315,7 @@ public void record(Event evt) {
                 }
 
             if (isRecordable(type))
-                notifyListeners(evt);
-        }
-        finally {
-            leaveBusy();
-        }
-    }
-
-    /**
-     * Records discovery events.
-     *
-     * @param evt Event to record.
-     * @param discoCache Discovery cache.
-     */
-    public void record(DiscoveryEvent evt, DiscoCache discoCache) {
-        assert evt != null;
-
-        if (!enterBusy())
-            return;
-
-        try {
-            // Notify internal discovery listeners first.
-            notifyDiscoveryListeners(evt, discoCache);
-
-            // Notify all other registered listeners.
-            record(evt);
+                notifyListeners(lsnrs.get(evt.type()), evt, params);
         }
         finally {
             leaveBusy();
@@ -571,6 +565,8 @@ private boolean binarySearch(@Nullable int[] arr, int val) {
      * @param types Event types to subscribe listener for.
      */
     public void addLocalEventListener(IgnitePredicate<? extends Event> lsnr, int[] types) {
+        assert lsnr != null;
+
         try {
             ctx.resource().injectGeneric(lsnr);
         }
@@ -578,7 +574,7 @@ public void addLocalEventListener(IgnitePredicate<? extends Event> lsnr, int[] t
             throw new IgniteException("Failed to inject resources to event listener: " + lsnr, e);
         }
 
-        addLocalEventListener(new UserListenerWrapper(lsnr), types);
+        addEventListener(new UserListenerWrapper(lsnr), types);
     }
 
     /**
@@ -594,20 +590,7 @@ public void addLocalEventListener(GridLocalEventListener lsnr, int[] types) {
         assert types != null;
         assert types.length > 0;
 
-        if (!enterBusy())
-            return;
-
-        try {
-            for (int t : types) {
-                getOrCreate(lsnrs, t).add(lsnr);
-
-                if (!isRecordable(t))
-                    U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
-            }
-        }
-        finally {
-            leaveBusy();
-        }
+        addEventListener(new LocalListenerWrapper(lsnr), types);
     }
 
     /**
@@ -620,27 +603,7 @@ public void addLocalEventListener(GridLocalEventListener lsnr, int[] types) {
     public void addLocalEventListener(GridLocalEventListener lsnr, int type, @Nullable int... types) {
         assert lsnr != null;
 
-        if (!enterBusy())
-            return;
-
-        try {
-            getOrCreate(lsnrs, type).add(lsnr);
-
-            if (!isRecordable(type))
-                U.warn(log, "Added listener for disabled event type: " + U.gridEventName(type));
-
-            if (types != null) {
-                for (int t : types) {
-                    getOrCreate(lsnrs, t).add(lsnr);
-
-                    if (!isRecordable(t))
-                        U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
-                }
-            }
-        }
-        finally {
-            leaveBusy();
-        }
+        addEventListener(new LocalListenerWrapper(lsnr), type, types);
     }
 
     /**
@@ -656,12 +619,40 @@ public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int[] types)
         assert types != null;
         assert types.length > 0;
 
+        addEventListener(new DiscoveryListenerWrapper(lsnr), types);
+    }
+
+    /**
+     * Adds discovery event listener.
+     *
+     * @param lsnr Listener to add.
+     * @param type Event type to subscribe listener for.
+     * @param types Additional event types to subscribe listener for.
+     */
+    public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int type, @Nullable int... types) {
+        assert lsnr != null;
+
+        addEventListener(new DiscoveryListenerWrapper(lsnr), type, types);
+    }
+
+    /**
+     * Adds local event listener. Note that this method specifically disallow an empty
+     * array of event type to prevent accidental subscription for all system event that
+     * may lead to a drastic performance decrease.
+     *
+     * @param lsnr Listener to add.
+     * @param types Event types to subscribe listener for.
+     */
+    private void addEventListener(EventListener lsnr, int[] types) {
         if (!enterBusy())
             return;
 
         try {
             for (int t : types) {
-                getOrCreate(discoLsnrs, t).add(lsnr);
+                getOrCreate(lsnrs, t).add(lsnr);
+
+                if (!isRecordable(t))
+                    U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
             }
         }
         finally {
@@ -670,24 +661,28 @@ public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int[] types)
     }
 
     /**
-     * Adds discovery event listener.
+     * Adds local event listener.
      *
      * @param lsnr Listener to add.
      * @param type Event type to subscribe listener for.
      * @param types Additional event types to subscribe listener for.
      */
-    public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int type, @Nullable int... types) {
-        assert lsnr != null;
-
+    private void addEventListener(EventListener lsnr, int type, @Nullable int... types) {
         if (!enterBusy())
             return;
 
         try {
-            getOrCreate(discoLsnrs, type).add(lsnr);
+            getOrCreate(lsnrs, type).add(lsnr);
+
+            if (!isRecordable(type))
+                U.warn(log, "Added listener for disabled event type: " + U.gridEventName(type));
 
             if (types != null) {
                 for (int t : types) {
-                    getOrCreate(discoLsnrs, t).add(lsnr);
+                    getOrCreate(lsnrs, t).add(lsnr);
+
+                    if (!isRecordable(t))
+                        U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
                 }
             }
         }
@@ -696,6 +691,7 @@ public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int type, @Nu
         }
     }
 
+
     /**
      * @param lsnrs Listeners map.
      * @param type Event type.
@@ -727,7 +723,9 @@ private <T> Collection<T> getOrCreate(ConcurrentMap<Integer, Set<T>> lsnrs, Inte
      * @return Returns {@code true} if removed.
      */
     public boolean removeLocalEventListener(IgnitePredicate<? extends Event> lsnr, @Nullable int... types) {
-        return removeLocalEventListener(new UserListenerWrapper(lsnr), types);
+        assert lsnr != null;
+
+        return removeEventListener(new UserListenerWrapper(lsnr), types);
     }
 
     /**
@@ -741,33 +739,21 @@ public boolean removeLocalEventListener(IgnitePredicate<? extends Event> lsnr, @
     public boolean removeLocalEventListener(GridLocalEventListener lsnr, @Nullable int... types) {
         assert lsnr != null;
 
-        boolean found = false;
-
-        if (F.isEmpty(types)) {
-            for (Set<GridLocalEventListener> set : lsnrs.values())
-                if (set.remove(lsnr))
-                    found = true;
-        }
-        else {
-            assert types != null;
-
-            for (int type : types) {
-                Set<GridLocalEventListener> set = lsnrs.get(type);
-
-                if (set != null && set.remove(lsnr))
-                    found = true;
-            }
-        }
-
-        if (lsnr instanceof UserListenerWrapper)
-        {
-            IgnitePredicate p = ((UserListenerWrapper)lsnr).listener();
+        return removeEventListener(new LocalListenerWrapper(lsnr), types);
+    }
 
-            if (p instanceof PlatformEventFilterListener)
-                ((PlatformEventFilterListener)p).onClose();
-        }
+    /**
+     * Removes listener for specified events, if any. If no event types provided - it
+     * remove the listener for all its registered events.
+     *
+     * @param lsnr Listener.
+     * @param types Event types.
+     * @return Returns {@code true} if removed.
+     */
+    public boolean removeDiscoveryEventListener(DiscoveryEventListener lsnr, @Nullable int... types) {
+        assert lsnr != null;
 
-        return found;
+        return removeEventListener(new DiscoveryListenerWrapper(lsnr), types);
     }
 
     /**
@@ -778,13 +764,13 @@ public boolean removeLocalEventListener(GridLocalEventListener lsnr, @Nullable i
      * @param types Event types.
      * @return Returns {@code true} if removed.
      */
-    public boolean removeDiscoveryEventListener(DiscoveryEventListener lsnr, @Nullable int... types) {
+    private boolean removeEventListener(EventListener lsnr, @Nullable int[] types) {
         assert lsnr != null;
 
         boolean found = false;
 
         if (F.isEmpty(types)) {
-            for (Set<DiscoveryEventListener> set : discoLsnrs.values())
+            for (Set<EventListener> set : lsnrs.values())
                 if (set.remove(lsnr))
                     found = true;
         }
@@ -792,13 +778,21 @@ public boolean removeDiscoveryEventListener(DiscoveryEventListener lsnr, @Nullab
             assert types != null;
 
             for (int type : types) {
-                Set<DiscoveryEventListener> set = discoLsnrs.get(type);
+                Set<EventListener> set = lsnrs.get(type);
 
                 if (set != null && set.remove(lsnr))
                     found = true;
             }
         }
 
+        if (lsnr instanceof UserListenerWrapper)
+        {
+            IgnitePredicate p = ((UserListenerWrapper)lsnr).listener();
+
+            if (p instanceof PlatformEventFilterListener)
+                ((PlatformEventFilterListener)p).onClose();
+        }
+
         return found;
     }
 
@@ -861,63 +855,19 @@ public Event waitForEvent(long timeout, @Nullable Runnable c,
         return fut.get(timeout);
     }
 
-    /**
-     * @param evt Event to notify about.
-     */
-    private void notifyListeners(Event evt) {
-        assert evt != null;
-
-        notifyListeners(lsnrs.get(evt.type()), evt);
-    }
-
     /**
      * @param set Set of listeners.
      * @param evt Grid event.
      */
-    private void notifyListeners(@Nullable Collection<GridLocalEventListener> set, Event evt) {
-        assert evt != null;
-
-        if (!F.isEmpty(set)) {
-            assert set != null;
-
-            for (GridLocalEventListener lsnr : set) {
-                try {
-                    lsnr.onEvent(evt);
-                }
-                catch (Throwable e) {
-                    U.error(log, "Unexpected exception in listener notification for event: " + evt, e);
-
-                    if (e instanceof Error)
-                        throw (Error)e;
-                }
-            }
-        }
-    }
-
-    /**
-     * @param evt Discovery event
-     * @param cache Discovery cache.
-     */
-    private void notifyDiscoveryListeners(DiscoveryEvent evt, DiscoCache cache) {
-        assert evt != null;
-
-        notifyDiscoveryListeners(discoLsnrs.get(evt.type()), evt, cache);
-    }
-
-    /**
-     * @param set Set of listeners.
-     * @param evt Discovery event.
-     * @param cache Discovery cache.
-     */
-    private void notifyDiscoveryListeners(@Nullable Collection<DiscoveryEventListener> set, DiscoveryEvent evt, DiscoCache cache) {
+    private void notifyListeners(@Nullable Collection<EventListener> set, Event evt, Object[] params) {
         assert evt != null;
 
         if (!F.isEmpty(set)) {
             assert set != null;
 
-            for (DiscoveryEventListener lsnr : set) {
+            for (EventListener lsnr : set) {
                 try {
-                    lsnr.onEvent(evt, cache);
+                    ((ListenerWrapper)lsnr).onEvent(evt, params);
                 }
                 catch (Throwable e) {
                     U.error(log, "Unexpected exception in listener notification for event: " + evt, e);
@@ -1307,10 +1257,93 @@ private class RequestListener implements GridMessageListener {
         }
     }
 
+    /** */
+    private abstract static class ListenerWrapper implements EventListener {
+        abstract void onEvent(Event evt, Object[] params);
+    }
+
+    /**
+     * Wraps local listener
+     */
+    private static final class LocalListenerWrapper extends ListenerWrapper {
+        /** */
+        private final GridLocalEventListener lsnr;
+
+        /**
+         * @param lsnr Listener.
+         */
+        private LocalListenerWrapper(GridLocalEventListener lsnr) {
+            this.lsnr = lsnr;
+        }
+
+        /** {@inheritDoc} */
+        @Override void onEvent(Event evt, Object[] params) {
+            lsnr.onEvent(evt);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            LocalListenerWrapper wrapper = (LocalListenerWrapper)o;
+
+            return lsnr.equals(wrapper.lsnr);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return lsnr.hashCode();
+        }
+    }
+
+    /**
+     * Wraps discovery local listener
+     */
+    private static final class DiscoveryListenerWrapper extends ListenerWrapper {
+        /** */
+        private final DiscoveryEventListener lsnr;
+
+        /**
+         * @param lsnr Listener.
+         */
+        private DiscoveryListenerWrapper(DiscoveryEventListener lsnr) {
+            this.lsnr = lsnr;
+        }
+
+        /** {@inheritDoc} */
+        @Override void onEvent(Event evt, Object[] params) {
+            // No checks there since only DiscoveryManager produses DiscoveryEvents
+            // and it uses an overloaded method with additional parameters
+            lsnr.onEvent((DiscoveryEvent)evt, (DiscoCache)params[0]);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            DiscoveryListenerWrapper wrapper = (DiscoveryListenerWrapper)o;
+
+            return lsnr.equals(wrapper.lsnr);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return lsnr.hashCode();
+        }
+    }
+
     /**
-     * Wraps user listener predicate provided via {@link org.apache.ignite.IgniteEvents#localListen(org.apache.ignite.lang.IgnitePredicate, int...)}.
+     * Wraps user listener predicate provided via {@link IgniteEvents#localListen(IgnitePredicate, int...)}.
      */
-    private class UserListenerWrapper implements GridLocalEventListener {
+    private final class UserListenerWrapper extends ListenerWrapper {
         /** */
         private final IgnitePredicate<Event> lsnr;
 
@@ -1329,9 +1362,9 @@ private IgnitePredicate<? extends Event> listener() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onEvent(Event evt) {
+        @Override void onEvent(Event evt, Object[] params) {
             if (!lsnr.apply(evt))
-                removeLocalEventListener(this);
+                removeEventListener(this, null);
         }
 
         /** {@inheritDoc} */

From 405ce563fb7c35627c6e1bb0b68f423ba089c6f2 Mon Sep 17 00:00:00 2001
From: Dmitriy Shabalin <dmitriyff@gmail.com>
Date: Fri, 14 Apr 2017 17:55:38 +0700
Subject: [PATCH 240/446] IGNITE-4068 Added common primitive for buttons group.
 Refactored existing button groups. (cherry picked from commit e5200c2)

---
 .../frontend/app/helpers/jade/mixins.pug      | 12 ++++---
 .../app/modules/sql/sql.controller.js         |  6 ----
 .../app/primitives/btn-group/index.pug        | 35 +++++++++++++++++++
 .../views/configuration/domains.tpl.pug       |  1 -
 .../frontend/views/sql/sql.tpl.pug            | 12 +++----
 5 files changed, 46 insertions(+), 20 deletions(-)
 create mode 100644 modules/web-console/frontend/app/primitives/btn-group/index.pug

diff --git a/modules/web-console/frontend/app/helpers/jade/mixins.pug b/modules/web-console/frontend/app/helpers/jade/mixins.pug
index db175a2146ef1..b8cbfe2100b7d 100644
--- a/modules/web-console/frontend/app/helpers/jade/mixins.pug
+++ b/modules/web-console/frontend/app/helpers/jade/mixins.pug
@@ -15,6 +15,7 @@
     limitations under the License.
 
 include ./form
+include ../../primitives/btn-group/index
 
 //- Mixin for advanced options toggle.
 mixin advanced-options-toggle(click, cond, showMessage, hideMessage)
@@ -49,7 +50,7 @@ mixin main-table(title, rows, focusId, click, rowTemplate, searchField)
 
 //- Mixin with save, remove, clone and undo buttons.
 mixin save-remove-clone-undo-buttons(objectName)
-    -var removeTip = '"Remove current ' + objectName + '"'
+    -var removeTip = 'Remove current ' + objectName
     -var cloneTip = '"Clone current ' + objectName + '"'
     -var undoTip = '"Undo all changes for current ' + objectName + '"'
 
@@ -60,10 +61,11 @@ mixin save-remove-clone-undo-buttons(objectName)
             a.btn.btn-primary(id='save-item' ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && saveItem()' bs-tooltip='' data-title=`{{saveBtnTipText(ui.inputForm.$dirty, '${objectName}')}}` data-placement='bottom' data-trigger='hover') Save
         .panel-tip-container(ng-show='backupItem._id')
             a.btn.btn-primary(id='clone-item' ng-click='cloneItem()' bs-tooltip=cloneTip data-placement='bottom' data-trigger='hover') Clone
-        .btn-group.panel-tip-container(ng-show='backupItem._id')
-            button.btn.btn-primary(id='remove-item' ng-click='removeItem()' bs-tooltip=removeTip data-placement='bottom' data-trigger='hover') Remove
-            button.btn.dropdown-toggle.btn-primary(id='remove-item-dropdown' data-toggle='dropdown' data-container='body' bs-dropdown='[{ text: "Remove All", click: "removeAllItems()" }]' data-placement='bottom-right')
-                span.caret
+
+        -var actions = [{ text: "Remove", click: "removeItem()" }, { text: "Remove All", click: "removeAllItems()" }]
+
+        +btn-group(actions, removeTip)(ng-show='backupItem._id')
+
         .panel-tip-container(ng-show='backupItem')
             i.btn.btn-primary.fa.fa-undo(id='undo-item' ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && resetAll()' bs-tooltip=undoTip data-placement='bottom' data-trigger='hover')
 
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 075bd559ee1e2..b1cd3d079c89c 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -271,12 +271,6 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
             {value: 3600000, label: 'hours', short: 'h'}
         ];
 
-        $scope.exportDropdown = [
-            { text: 'Export all', click: 'exportCsvAll(paragraph)' }
-            // { 'text': 'Export all to CSV', 'click': 'exportCsvAll(paragraph)' },
-            // { 'text': 'Export all to PDF', 'click': 'exportPdfAll(paragraph)' }
-        ];
-
         $scope.metadata = [];
 
         $scope.metaFilter = '';
diff --git a/modules/web-console/frontend/app/primitives/btn-group/index.pug b/modules/web-console/frontend/app/primitives/btn-group/index.pug
new file mode 100644
index 0000000000000..c047b255bc3cd
--- /dev/null
+++ b/modules/web-console/frontend/app/primitives/btn-group/index.pug
@@ -0,0 +1,35 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+mixin btn-group(actions, tip)
+    .btn-group.panel-tip-container&attributes(attributes)
+        button.btn.btn-primary(
+            ng-click=`${actions[0].click}`
+
+            bs-tooltip=''
+            data-title=tip
+
+            data-trigger='hover'
+            data-placement='bottom'
+        ) #{actions[0].text}
+        button.btn.dropdown-toggle.btn-primary(
+            bs-dropdown=`${JSON.stringify(actions)}`
+
+            data-toggle='dropdown'
+            data-container='body'
+            data-placement='bottom-right'
+        )
+            span.caret
diff --git a/modules/web-console/frontend/views/configuration/domains.tpl.pug b/modules/web-console/frontend/views/configuration/domains.tpl.pug
index 33528c7c8dd68..a8f9b5ce19c1a 100644
--- a/modules/web-console/frontend/views/configuration/domains.tpl.pug
+++ b/modules/web-console/frontend/views/configuration/domains.tpl.pug
@@ -55,7 +55,6 @@ include /app/helpers/jade/mixins
                 .panel-tip-container(bs-tooltip='' data-title='Import domain models from database' data-placement='bottom')
                     button.btn.btn-primary(ng-click='showImportDomainModal()') Import from database
                 +save-remove-clone-undo-buttons('domain model')
-                .btn-group.panel-tip-container.pull-right(bs-tooltip='' data-title='Import domain models from demo database' data-placement='bottom')
                 hr
             .bs-affix-fix
             div(bs-collapse='' data-allow-multiple='true' ng-model='ui.activePanels')
diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug
index dcfc53137f616..6b14555a68ebf 100644
--- a/modules/web-console/frontend/views/sql/sql.tpl.pug
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -134,10 +134,8 @@ mixin table-result-heading-query
                 +result-toolbar
         .col-xs-4
             .pull-right
-                .btn-group(ng-disabled='paragraph.loading')
-                    button.btn.btn-primary(ng-click='exportCsv(paragraph)' bs-tooltip data-title='{{actionTooltip(paragraph, "export", false)}}') Export
-                    button.btn.btn-primary.dropdown-toggle(id='export-item-dropdown' data-toggle='dropdown' data-container='body' bs-dropdown='exportDropdown' data-placement='bottom-right')
-                        span.caret
+                -var actions = [{ text: "Export", click: 'exportCsv(paragraph)' }, { text: 'Export all', click: 'exportCsvAll(paragraph)' }]
+                +btn-group(actions, '{{ actionTooltip(paragraph, "export", false) }}')(ng-disabled='paragraph.loading')
 
 mixin table-result-heading-scan
     .total.row
@@ -151,10 +149,8 @@ mixin table-result-heading-scan
                 +result-toolbar
         .col-xs-4
             .pull-right
-                .btn-group(ng-disabled='paragraph.loading')
-                    button.btn.btn-primary(ng-click='exportCsv(paragraph)' bs-tooltip data-title='{{actionTooltip(paragraph, "export", false)}}') Export
-                    button.btn.btn-primary.dropdown-toggle(id='export-item-dropdown' data-toggle='dropdown' data-container='body' bs-dropdown='exportDropdown' data-placement='bottom-right')
-                        span.caret
+                -var actions = [{ text: "Export", click: 'exportCsv(paragraph)' }, { text: 'Export all', click: 'exportCsvAll(paragraph)' }]
+                +btn-group(actions, '{{ actionTooltip(paragraph, "export", false) }}')(ng-disabled='paragraph.loading')
 
 mixin table-result-body
     .grid(ui-grid='paragraph.gridOptions' ui-grid-resize-columns ui-grid-exporter)

From 60cf48dc175fa288cd74d1189f0e992c9a16dc99 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Fri, 14 Apr 2017 18:00:47 +0700
Subject: [PATCH 241/446] IGNITE-4886 Catch all errors. (cherry picked from
 commit 7e8d9e8)

---
 .../internal/visor/service/VisorServiceDescriptor.java   | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java
index 83ec77d5c22e0..26f5c10dd57b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java
@@ -68,7 +68,14 @@ public VisorServiceDescriptor() {
      */
     public VisorServiceDescriptor(ServiceDescriptor srvc) {
         name = srvc.name();
-        srvcCls = VisorTaskUtils.compactClass(srvc.serviceClass());
+
+        try {
+            srvcCls = VisorTaskUtils.compactClass(srvc.serviceClass());
+        }
+        catch (Throwable e) {
+            srvcCls = e.getClass().getName() + ": " + e.getMessage();
+        }
+
         totalCnt = srvc.totalCount();
         maxPerNodeCnt = srvc.maxPerNodeCount();
         cacheName = srvc.cacheName();

From 81c3ed4c0511841f7056677db6063b4eb8d2def0 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 14 Apr 2017 18:18:23 +0700
Subject: [PATCH 242/446] IGNITE-4896 Rewored GridClientNodeBean serialization.
 (cherry picked from commit a025268)

---
 .../client/message/GridClientNodeBean.java    |   2 +-
 .../top/GridTopologyCommandHandler.java       |   4 +
 ...ConfigurationCustomSerializerSelfTest.java | 147 ++++++++++++++++++
 .../IgniteBinaryObjectsTestSuite.java         |   2 +
 4 files changed, 154 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryConfigurationCustomSerializerSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/client/message/GridClientNodeBean.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/client/message/GridClientNodeBean.java
index ca8260865d9dc..f1848dfc6d0e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/client/message/GridClientNodeBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/client/message/GridClientNodeBean.java
@@ -250,7 +250,7 @@ public void setTcpPort(int tcpPort) {
 
         U.writeUuid(out, nodeId);
 
-        out.writeObject(consistentId);
+        out.writeObject(String.valueOf(consistentId));
         out.writeObject(metrics);
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
index 297785ea760f2..536ec88815471 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
@@ -52,7 +52,9 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.IgnitePortProtocol;
 
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_BINARY_CONFIGURATION;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CACHE;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_TCP_ADDRS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_TCP_HOST_NAMES;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_TCP_PORT;
@@ -291,6 +293,8 @@ private GridClientNodeBean createNodeBean(ClusterNode node, boolean mtr, boolean
             attrs.remove(ATTR_TX_CONFIG);
             attrs.remove(ATTR_SECURITY_SUBJECT);
             attrs.remove(ATTR_SECURITY_CREDENTIALS);
+            attrs.remove(ATTR_BINARY_CONFIGURATION);
+            attrs.remove(ATTR_NODE_CONSISTENT_ID);
 
             for (Iterator<Map.Entry<String, Object>> i = attrs.entrySet().iterator(); i.hasNext();) {
                 Map.Entry<String, Object> e = i.next();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryConfigurationCustomSerializerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryConfigurationCustomSerializerSelfTest.java
new file mode 100644
index 0000000000000..1da2967612c21
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryConfigurationCustomSerializerSelfTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.UUID;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinarySerializer;
+import org.apache.ignite.binary.BinaryTypeConfiguration;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.client.GridClient;
+import org.apache.ignite.internal.client.GridClientConfiguration;
+import org.apache.ignite.internal.client.GridClientFactory;
+import org.apache.ignite.internal.client.GridClientProtocol;
+import org.apache.ignite.internal.client.balancer.GridClientRoundRobinBalancer;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.node.VisorNodePingTask;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests that node will start with custom binary serializer and thin client will connect to such node.
+ */
+public class BinaryConfigurationCustomSerializerSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        BinaryConfiguration binaryCfg = new BinaryConfiguration();
+
+        BinaryTypeConfiguration btc = new BinaryTypeConfiguration("org.MyClass");
+
+        btc.setIdMapper(BinaryContext.defaultIdMapper());
+        btc.setEnum(false);
+
+        // Set custom serializer that is unknown for Optimized marshaller.
+        btc.setSerializer(new MyBinarySerializer());
+
+        binaryCfg.setTypeConfigurations(Collections.singletonList(btc));
+
+        cfg.setBinaryConfiguration(binaryCfg);
+
+        // Set custom consistent ID that unknown for Optimized marshaller.
+        cfg.setConsistentId(new MyConsistentId("test"));
+
+        cfg.setCacheConfiguration(new CacheConfiguration("TEST"));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        startGrids(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * Test that thin client will be able to connect to node with custom binary serializer and custom consistent ID.
+     *
+     * @throws Exception If failed.
+     */
+    public void testThinClientConnected() throws Exception {
+        UUID nid = ignite(0).cluster().localNode().id();
+
+        GridClientConfiguration clnCfg = new GridClientConfiguration();
+
+        clnCfg.setProtocol(GridClientProtocol.TCP);
+        clnCfg.setServers(Collections.singleton("127.0.0.1:11211"));
+        clnCfg.setBalancer(new GridClientRoundRobinBalancer());
+
+        // Start client.
+        GridClient client = GridClientFactory.start(clnCfg);
+
+        // Execute some task.
+        client.compute().execute(VisorNodePingTask.class.getName(), new VisorTaskArgument<>(nid, nid, false));
+
+        GridClientFactory.stop(client.id(), false);
+    }
+
+    /**
+     * Custom consistent ID.
+     */
+    private static class MyConsistentId implements Serializable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** Actual ID. */
+        private String id;
+
+        /**
+         * @param id Actual ID.
+         */
+        MyConsistentId(String id) {
+            this.id = id;
+        }
+
+        /**
+         * @return Consistent ID.
+         */
+        public String getId() {
+            return id;
+        }
+    }
+
+    /**
+     * Custom BinarySerializer.
+     */
+    private static class MyBinarySerializer implements BinarySerializer {
+        /** {@inheritDoc} */
+        @Override public void writeBinary(Object obj, BinaryWriter writer) throws BinaryObjectException {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readBinary(Object obj, BinaryReader reader) throws BinaryObjectException {
+            // No-op.
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
index 3496dbf381ac8..4aa22278d1d9d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.internal.binary.BinaryBasicIdMapperSelfTest;
 import org.apache.ignite.internal.binary.BinaryBasicNameMapperSelfTest;
 import org.apache.ignite.internal.binary.BinaryConfigurationConsistencySelfTest;
+import org.apache.ignite.internal.binary.BinaryConfigurationCustomSerializerSelfTest;
 import org.apache.ignite.internal.binary.BinaryEnumsSelfTest;
 import org.apache.ignite.internal.binary.BinaryFieldIdentityResolverSelfTest;
 import org.apache.ignite.internal.binary.BinaryFieldsHeapSelfTest;
@@ -100,6 +101,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(BinaryIdentityResolverConfigurationSelfTest.class);
 
         suite.addTestSuite(BinaryConfigurationConsistencySelfTest.class);
+        suite.addTestSuite(BinaryConfigurationCustomSerializerSelfTest.class);
         suite.addTestSuite(GridBinaryMarshallerCtxDisabledSelfTest.class);
         suite.addTestSuite(BinaryObjectBuilderDefaultMappersSelfTest.class);
         suite.addTestSuite(BinaryObjectBuilderSimpleNameLowerCaseMappersSelfTest.class);

From 4a1415ad01ff9fde30d5c7c02e6d938f1515178d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 12 Apr 2017 13:01:25 +0300
Subject: [PATCH 243/446] IGNITE-4907: Fixed excessive service instances can be
 started with dynamic deployment. This closes #1766.

(cherry picked from commit 0f7ef74)
---
 .../service/GridServiceProcessor.java         |  2 +-
 ...rviceProcessorMultiNodeConfigSelfTest.java | 95 ++++++++++++++++---
 ...GridServiceProcessorMultiNodeSelfTest.java | 61 ++++++++++++
 3 files changed, 146 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index a8af9832e9355..2a363e21e8ef5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -972,7 +972,7 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
                         int perNodeCnt = totalCnt != 0 ? totalCnt / size : maxPerNodeCnt;
                         int remainder = totalCnt != 0 ? totalCnt % size : 0;
 
-                        if (perNodeCnt > maxPerNodeCnt && maxPerNodeCnt != 0) {
+                        if (perNodeCnt >= maxPerNodeCnt && maxPerNodeCnt != 0) {
                             perNodeCnt = maxPerNodeCnt;
                             remainder = 0;
                         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index 1bd3b035a446c..9da62c0abb702 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -17,11 +17,13 @@
 
 package org.apache.ignite.internal.processors.service;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
-import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -38,6 +40,9 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
     /** Node singleton name. */
     private static final String NODE_SINGLE_BUT_CLIENT = "serviceConfigEachNodeButClient";
 
+    /** Node singleton name. */
+    private static final String NODE_SINGLE_WITH_LIMIT = "serviceConfigWithLimit";
+
     /** Affinity service name. */
     private static final String AFFINITY = "serviceConfigAffinity";
 
@@ -51,7 +56,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        ServiceConfiguration[] arr = new ServiceConfiguration[4];
+        List<ServiceConfiguration> cfgs = new ArrayList<>();
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -60,7 +65,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new DummyService());
 
-        arr[0] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -68,7 +73,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setMaxPerNodeCount(1);
         cfg.setService(new DummyService());
 
-        arr[1] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -79,7 +84,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new AffinityService(AFFINITY_KEY));
 
-        arr[2] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -88,9 +93,18 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
         cfg.setService(new DummyService());
 
-        arr[3] = cfg;
+        cfgs.add(cfg);
+
+        cfg = new ServiceConfiguration();
 
-        return arr;
+        cfg.setName(NODE_SINGLE_WITH_LIMIT);
+        cfg.setMaxPerNodeCount(1);
+        cfg.setTotalCount(nodeCount() + 1);
+        cfg.setService(new DummyService());
+
+        cfgs.add(cfg);
+
+        return cfgs.toArray(new ServiceConfiguration[cfgs.size()]);
     }
 
     /** {@inheritDoc} */
@@ -107,6 +121,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
                         DummyService.cancelled(NODE_SINGLE) == 0 &&
                         DummyService.started(NODE_SINGLE_BUT_CLIENT) == nodeCount() &&
                         DummyService.cancelled(NODE_SINGLE_BUT_CLIENT) == 0 &&
+                        DummyService.started(NODE_SINGLE_WITH_LIMIT) >= nodeCount() &&
+                        DummyService.cancelled(NODE_SINGLE_WITH_LIMIT) == 0 &&
                         actualCount(AFFINITY, randomGrid().services().serviceDescriptors()) == 1;
                 }
             },
@@ -170,6 +186,59 @@ public void testAffinityUpdateTopology() throws Exception {
         finally {
             stopExtraNodes(nodeCnt);
         }
+
+        checkCount(AFFINITY, g.services().serviceDescriptors(), 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployLimits() throws Exception {
+        final Ignite g = randomGrid();
+
+        final String name = NODE_SINGLE_WITH_LIMIT;
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
+
+        int extraNodes = 2;
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        DummyService.exeLatch(name, latch);
+
+        startExtraNodes(extraNodes);
+
+        try {
+            latch.await();
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount() + 1);
+        }
+        finally {
+            stopExtraNodes(extraNodes);
+        }
+
+        assertEquals(name, 1, DummyService.cancelled(name));
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
+    }
+
+    /**
+     * @param srvcName Service name
+     * @param expectedDeps Expected number of service deployments
+     *
+     */
+    private boolean waitForDeployment(final String srvcName, final int expectedDeps) throws IgniteInterruptedCheckedException {
+        final Ignite g = randomGrid();
+
+        return GridTestUtils.waitForCondition(new GridAbsPredicateX() {
+            @Override public boolean applyx() {
+                return actualCount(srvcName, g.services().serviceDescriptors())  == expectedDeps;
+            }
+        }, 1500);
     }
 
     /**
@@ -212,10 +281,6 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         try {
             latch.await();
 
-            // Ensure service is deployed.
-            assertNotNull(grid(nodeCount() + newNodes - 1).services()
-                .serviceProxy(NODE_SINGLE_BUT_CLIENT, Service.class, false, 2000));
-
             assertEquals(name, newNodes, DummyService.started(name));
             assertEquals(name, 0, DummyService.cancelled(name));
 
@@ -224,6 +289,10 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         finally {
             stopExtraNodes(newNodes);
         }
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 
     /**
@@ -253,5 +322,9 @@ private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Ex
         finally {
             stopExtraNodes(servers + clients);
         }
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index f7403dcc820ca..d133cf299a756 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -264,4 +264,65 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
             stopGrid("client");
         }
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployLimits() throws Exception {
+            String name = "serviceWithLimitsUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            final int totalInstances = nodeCount() + 1;
+
+            CountDownLatch latch = new CountDownLatch(nodeCount());
+
+            DummyService.exeLatch(name, latch);
+
+            ServiceConfiguration srvcCfg = new ServiceConfiguration();
+
+            srvcCfg.setName(name);
+            srvcCfg.setMaxPerNodeCount(1);
+            srvcCfg.setTotalCount(totalInstances);
+            srvcCfg.setService(new DummyService());
+
+            IgniteServices svcs = g.services().withAsync();
+
+            svcs.deploy(srvcCfg);
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
+            latch.await();
+
+            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
+            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount());
+
+            int extraNodes = 2;
+
+            latch = new CountDownLatch(1);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(2);
+
+            try {
+                latch.await();
+
+                TestCase.assertEquals(name, totalInstances, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), totalInstances);
+            }
+            finally {
+                stopExtraNodes(extraNodes);
+            }
+    }
 }
\ No newline at end of file

From e206b9f1fd3dbf927f940d558144a4796479ed5d Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgain.com>
Date: Fri, 14 Apr 2017 18:32:30 +0700
Subject: [PATCH 244/446] IGNITE-4871 Added Kubernetes IP finder to Cluster
 configuration screen. (cherry picked from commit f978ff2)

---
 modules/web-console/backend/app/mongo.js      |  8 +++-
 .../frontend/app/data/pom-dependencies.json   |  1 +
 .../app/helpers/jade/form/form-field-text.pug | 40 +++++++++++--------
 .../frontend/app/helpers/jade/mixins.pug      |  8 +++-
 .../generator/ConfigurationGenerator.js       | 12 ++++++
 .../generator/defaults/Cluster.service.js     |  6 +++
 .../states/configuration/clusters/general.pug |  3 ++
 .../clusters/general/discovery/kubernetes.pug | 37 +++++++++++++++++
 .../summary/summary.controller.js             |  3 ++
 .../configuration/summary/summary.worker.js   | 22 ++++++++++
 .../controllers/clusters-controller.js        |  3 +-
 11 files changed, 124 insertions(+), 19 deletions(-)
 create mode 100644 modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/kubernetes.pug

diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index 2d252b9b8eccc..f4a62b1fe2558 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -407,7 +407,7 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
             authenticator: String,
             forceServerMode: Boolean,
             clientReconnectDisabled: Boolean,
-            kind: {type: String, enum: ['Vm', 'Multicast', 'S3', 'Cloud', 'GoogleStorage', 'Jdbc', 'SharedFs', 'ZooKeeper']},
+            kind: {type: String, enum: ['Vm', 'Multicast', 'S3', 'Cloud', 'GoogleStorage', 'Jdbc', 'SharedFs', 'ZooKeeper', 'Kubernetes']},
             Vm: {
                 addresses: [String]
             },
@@ -528,6 +528,12 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
                 basePath: String,
                 serviceName: String,
                 allowDuplicateRegistrations: Boolean
+            },
+            Kubernetes: {
+                serviceName: String,
+                namespace: String,
+                masterUrl: String,
+                accountToken: String
             }
         },
         atomicConfiguration: {
diff --git a/modules/web-console/frontend/app/data/pom-dependencies.json b/modules/web-console/frontend/app/data/pom-dependencies.json
index 7d2bed0de8172..3e0543bb64861 100644
--- a/modules/web-console/frontend/app/data/pom-dependencies.json
+++ b/modules/web-console/frontend/app/data/pom-dependencies.json
@@ -3,6 +3,7 @@
     "S3": {"artifactId": "ignite-aws"},
     "GoogleStorage": {"artifactId": "ignite-gce"},
     "ZooKeeper": {"artifactId": "ignite-zookeeper"},
+    "Kubernetes": {"artifactId": "ignite-kubernetes"},
 
     "Log4j": {"artifactId": "ignite-log4j"},
     "Log4j2": {"artifactId": "ignite-log4j2"},
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
index 76ea6e609d6e1..63feaf85f13a3 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
@@ -14,12 +14,12 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin ignite-form-field-input(name, model, disabled, required, placeholder)
+mixin ignite-field-input(type, name, model, disabled, required, placeholder)
     input.form-control(
         id=`{{ ${name} }}Input`
         name=`{{ ${name} }}`
         placeholder=placeholder
-        type='text'
+        type=type
 
         data-ng-model=model
 
@@ -30,21 +30,11 @@ mixin ignite-form-field-input(name, model, disabled, required, placeholder)
         data-ignite-form-panel-field=''
     )&attributes(attributes ? attributes.attributes ? attributes.attributes : attributes: {})
 
-mixin ignite-form-field-url-input(name, model, disabled, required, placeholder)
-    input.form-control(
-        id=`{{ ${name} }}Input`
-        name=`{{ ${name} }}`
-        placeholder=placeholder
-        type='url'
-
-        data-ng-model=model
-
-        data-ng-required=required && `${required}`
-        data-ng-disabled=disabled && `${disabled}`
-        data-ng-focus='tableReset()'
+mixin ignite-form-field-input(name, model, disabled, required, placeholder)
+    +ignite-field-input('text', name, model, disabled, required, placeholder)
 
-        data-ignite-form-panel-field=''
-    )&attributes(attributes ? attributes.attributes ? attributes.attributes : attributes: {})
+mixin ignite-form-field-url-input(name, model, disabled, required, placeholder)
+    +ignite-field-input('url', name, model, disabled, required, placeholder)
 
 mixin ignite-form-field-text(label, model, name, disabled, required, placeholder, tip)
     -var errLbl = label.substring(0, label.length - 1)
@@ -62,3 +52,21 @@ mixin ignite-form-field-text(label, model, name, disabled, required, placeholder
 
             .input-tip
                 +ignite-form-field-input(name, model, disabled, required, placeholder)(attributes=attributes)
+
+mixin ignite-form-field-url(label, model, name, required, placeholder, tip)
+    -var errLbl = label.substring(0, label.length - 1)
+
+    .ignite-form-field
+        +ignite-form-field__label(label, name, required)
+        .ignite-form-field__control
+            if tip
+                i.tipField.icon-help(bs-tooltip='' data-title=tip)
+
+            if block
+                block
+
+            +form-field-feedback(name, 'required', errLbl + ' could not be empty!')
+            +form-field-feedback(name, 'url', errLbl + ' should be a valid URL!')
+
+            .input-tip
+                +ignite-form-field-url-input(name, model, false, required, placeholder)(attributes=attributes)
diff --git a/modules/web-console/frontend/app/helpers/jade/mixins.pug b/modules/web-console/frontend/app/helpers/jade/mixins.pug
index b8cbfe2100b7d..472d30414b31e 100644
--- a/modules/web-console/frontend/app/helpers/jade/mixins.pug
+++ b/modules/web-console/frontend/app/helpers/jade/mixins.pug
@@ -199,6 +199,12 @@ mixin text(lbl, model, name, required, placeholder, tip)
         if  block
             block
 
+//- Mixin for text field.
+mixin url(lbl, model, name, required, placeholder, tip)
+    +ignite-form-field-url(lbl, model, name, required, placeholder, tip)
+        if  block
+            block
+
 //- Mixin for password field.
 mixin password(lbl, model, name, required, placeholder, tip)
     +ignite-form-field-password(lbl, model, name, false, required, placeholder, tip)
@@ -350,7 +356,7 @@ mixin table-java-package-field(name, model, items, valid, save, newItem)
                 ignite-on-escape=onEscape
             )
 
-//- Mixin for table java package field.
+//- Mixin for table url field.
 mixin table-url-field(name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
     -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 670b8284f14c1..7cd910da43f93 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -320,6 +320,18 @@ export default class IgniteConfigurationGenerator {
                     .boolProperty('allowDuplicateRegistrations');
 
                 break;
+
+            case 'Kubernetes':
+                ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder',
+                    'ipFinder', cluster.discovery.Kubernetes, clusterDflts.discovery.Kubernetes);
+
+                ipFinder.stringProperty('serviceName')
+                    .stringProperty('namespace')
+                    .stringProperty('masterUrl')
+                    .pathProperty('accountToken');
+
+                break;
+
             default:
                 // No-op.
         }
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
index 6333ef96afcd1..b2e91c8836fe8 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
@@ -75,6 +75,12 @@ const DFLT_CLUSTER = {
             Forever: {
                 retryIntervalMs: 1000
             }
+        },
+        Kubernetes: {
+            serviceName: 'ignite',
+            namespace: 'default',
+            masterUrl: 'https://kubernetes.default.svc.cluster.local:443',
+            accountToken: '/var/run/secrets/kubernetes.io/serviceaccount/token'
         }
     },
     atomics: {
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
index dfc49d61ef4c2..62cc23db0045a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
@@ -52,6 +52,7 @@ include /app/helpers/jade/mixins
                             <li>JDBC - JDBC based IP finder that use database to store node IP address</li>\
                             <li>Shared filesystem - Shared filesystem based IP finder that use file to store node IP address</li>\
                             <li>Apache ZooKeeper - Apache ZooKeeper based IP finder when you use ZooKeeper to coordinate your distributed environment</li>\
+                            <li>Kubernetes - IP finder for automatic lookup of Ignite nodes running in Kubernetes environment</li>\
                         </ul>')
                 .settings-row
                     .panel-details
@@ -71,6 +72,8 @@ include /app/helpers/jade/mixins
                             include ./general/discovery/vm
                         div(ng-if=`${modelDiscoveryKind} === 'ZooKeeper'`)
                             include ./general/discovery/zookeeper
+                        div(ng-if=`${modelDiscoveryKind} === 'Kubernetes'`)
+                            include ./general/discovery/kubernetes
             .col-sm-6
                 -var model = 'backupItem'
                 +preview-xml-java(model, 'clusterCaches', 'caches')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/kubernetes.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/kubernetes.pug
new file mode 100644
index 0000000000000..c4e639816cd58
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/kubernetes.pug
@@ -0,0 +1,37 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins
+
+-var discoveryKind = 'Kubernetes'
+-var model = 'backupItem.discovery.Kubernetes'
+
+div
+    .details-row
+        +text('Service name:', `${model}.serviceName`, `'${discoveryKind}ServiceName'`, 'false', 'ignite',
+            "The name of Kubernetes service for Ignite pods' IP addresses lookup.<br/>\
+            The name of the service must be equal to the name set in service's Kubernetes configuration.<br/>\
+            If this parameter is not changed then the name of the service has to be set to 'ignite' in the corresponding Kubernetes configuration.")
+    .details-row
+        +text('Namespace:', `${model}.namespace`, `'${discoveryKind}Namespace'`, 'false', 'default',
+            "The namespace the Kubernetes service belongs to.<br/>\
+            By default, it's supposed that the service is running under Kubernetes `default` namespace.")
+    .details-row
+        +url('Kubernetes server:', `${model}.masterUrl`, `'${discoveryKind}MasterUrl'`, 'false', 'https://kubernetes.default.svc.cluster.local:443',
+            'The host name of the Kubernetes API server')
+    .details-row
+        +text('Service token file:', `${model}.accountToken`, `'${discoveryKind}AccountToken'`, 'false', '/var/run/secrets/kubernetes.io/serviceaccount/token',
+            'The path to the service token file')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
index 25203b30d1a27..0089528140f53 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
@@ -264,6 +264,9 @@ export default [
             if (cluster.discovery.kind === 'Jdbc' && cluster.discovery.Jdbc.dialect)
                 $scope.dialects[cluster.discovery.Jdbc.dialect] = true;
 
+            if (cluster.discovery.kind === 'Kubernetes')
+                resourcesFolder.children.push({ type: 'file', name: 'ignite-service.yaml' });
+
             _.forEach(cluster.caches, (cache) => {
                 if (cache.cacheStoreFactory) {
                     const store = cache.cacheStoreFactory[cache.cacheStoreFactory.kind];
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
index 7ea1d5ad5beb0..5933f6caa6c53 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
@@ -38,6 +38,25 @@ const generator = IgniteConfigurationGenerator;
 
 const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
 
+const kubernetesConfig = (cluster) => {
+    if (!cluster.discovery.Kubernetes)
+        cluster.discovery.Kubernetes = { serviceName: 'ignite' };
+
+    return `apiVersion: v1\n\
+kind: Service\n\
+metadata:\n\
+  # Name of Ignite Service used by Kubernetes IP finder for IP addresses lookup.\n\
+  name: ${ cluster.discovery.Kubernetes.serviceName || 'ignite' }\n\
+spec:\n\
+  clusterIP: None # custom value.\n\
+  ports:\n\
+    - port: 9042 # custom value.\n\
+  selector:\n\
+    # Must be equal to one of the labels set in Ignite pods'\n\
+    # deployement configuration.\n\
+    app: ${ cluster.discovery.Kubernetes.serviceName || 'ignite' }`;
+};
+
 // eslint-disable-next-line no-undef
 onmessage = function(e) {
     const {cluster, data, demo} = e.data;
@@ -67,6 +86,9 @@ onmessage = function(e) {
 
     const metaPath = `${resourcesPath}/META-INF`;
 
+    if (cluster.discovery.kind === 'Kubernetes')
+        zip.file(`${metaPath}/ignite-service.yaml`, kubernetesConfig(cluster));
+
     zip.file(`${metaPath}/${serverXml}`, spring.igniteConfiguration(cfg).asString());
     zip.file(`${metaPath}/${clientXml}`, spring.igniteConfiguration(clientCfg, clientNearCaches).asString());
 
diff --git a/modules/web-console/frontend/controllers/clusters-controller.js b/modules/web-console/frontend/controllers/clusters-controller.js
index c8392cf7ef0aa..e936955e05054 100644
--- a/modules/web-console/frontend/controllers/clusters-controller.js
+++ b/modules/web-console/frontend/controllers/clusters-controller.js
@@ -228,7 +228,8 @@ export default ['clustersController', [
             {value: 'GoogleStorage', label: 'Google cloud storage'},
             {value: 'Jdbc', label: 'JDBC'},
             {value: 'SharedFs', label: 'Shared filesystem'},
-            {value: 'ZooKeeper', label: 'Apache ZooKeeper'}
+            {value: 'ZooKeeper', label: 'Apache ZooKeeper'},
+            {value: 'Kubernetes', label: 'Kubernetes'}
         ];
 
         $scope.swapSpaceSpis = [

From b22738080101536a8af1ed60e70d693897e9bc7c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 14 Apr 2017 17:54:02 +0300
Subject: [PATCH 245/446] ignite-4173 Fix test. Permits must be released on
 node fail.

(cherry picked from commit 1f867c6)
---
 .../GridCacheAbstractDataStructuresFailoverSelfTest.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
index 9295770796381..4ab7b6780d2aa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
@@ -418,7 +418,7 @@ public void testSemaphoreFailoverSafe() throws Exception {
 
             stopGrid(NEW_GRID_NAME);
 
-            assertEquals(10, semaphore.availablePermits());
+            assertEquals(20, semaphore.availablePermits());
         }
     }
 

From 1985496ea98f4d7112a0b99727297427f343e9b2 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 17 Apr 2017 10:20:41 +0300
Subject: [PATCH 246/446] Add missed license headers.

---
 modules/kubernetes/config/Dockerfile            | 17 +++++++++++++++++
 .../kubernetes/config/ignite-deployment.yaml    | 17 +++++++++++++++++
 modules/kubernetes/config/ignite-service.yaml   | 17 +++++++++++++++++
 3 files changed, 51 insertions(+)

diff --git a/modules/kubernetes/config/Dockerfile b/modules/kubernetes/config/Dockerfile
index 4e08ce80e2147..2274535b5ccb2 100644
--- a/modules/kubernetes/config/Dockerfile
+++ b/modules/kubernetes/config/Dockerfile
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # Use Java 8 image as default one.
 FROM java:8
 
diff --git a/modules/kubernetes/config/ignite-deployment.yaml b/modules/kubernetes/config/ignite-deployment.yaml
index ed5c102a666a6..1e30f075e63bb 100644
--- a/modules/kubernetes/config/ignite-deployment.yaml
+++ b/modules/kubernetes/config/ignite-deployment.yaml
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # An example of a Kubernetes configuration for Ignite pods deployment.
 apiVersion: extensions/v1beta1
 kind: Deployment
diff --git a/modules/kubernetes/config/ignite-service.yaml b/modules/kubernetes/config/ignite-service.yaml
index 07b751624cbbf..7a7343ab522b3 100644
--- a/modules/kubernetes/config/ignite-service.yaml
+++ b/modules/kubernetes/config/ignite-service.yaml
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # An example of a Kubernetes configuration for Ignite lookup service deployment.
 apiVersion: v1
 kind: Service

From ae794ab45b22abe1e570d5d29661396ea7c0f951 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 17 Apr 2017 13:40:28 +0300
Subject: [PATCH 247/446] IGNITE-4159 Fix version.

---
 modules/kubernetes/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/kubernetes/pom.xml b/modules/kubernetes/pom.xml
index 5d4e5f0c8b509..7a632c617934f 100644
--- a/modules/kubernetes/pom.xml
+++ b/modules/kubernetes/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kubernetes</artifactId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>

From bf1049741f7a64728bd433f78262ba273f969848 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 17 Apr 2017 19:00:30 +0300
Subject: [PATCH 248/446] IGNITE-4954 - Configurable expiration timeout for
 Cassandra session. This closes #1785.

---
 .../cassandra/datasource/DataSource.java      | 26 ++++++++++++++++++-
 .../session/CassandraSessionImpl.java         | 23 +++++++++-------
 .../cassandra/session/pool/SessionPool.java   |  6 ++---
 .../session/pool/SessionWrapper.java          | 15 ++++++-----
 4 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
index 915eebde73f01..1fa2a1df9faa4 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
@@ -46,6 +46,9 @@
  * Data source abstraction to specify configuration of the Cassandra session to be used.
  */
 public class DataSource {
+    /** Default expiration timeout for Cassandra driver session. */
+    public static final long DFLT_SESSION_EXPIRATION_TIMEOUT = 300000; // 5 minutes.
+
     /** Number of rows to immediately fetch in CQL statement execution. */
     private Integer fetchSize;
 
@@ -123,6 +126,9 @@ public class DataSource {
     /** Netty options to use for connection. */
     private NettyOptions nettyOptions;
 
+    /** Expiration timeout for Cassandra driver session. */
+    private long sessionExpirationTimeout = DFLT_SESSION_EXPIRATION_TIMEOUT;
+
     /** Cassandra session wrapper instance. */
     private volatile CassandraSession ses;
 
@@ -441,6 +447,23 @@ public void setNettyOptions(NettyOptions options) {
         invalidate();
     }
 
+    /**
+     * Sets expiration timeout for Cassandra driver session. Idle sessions that are not
+     * used during this timeout value will be automatically closed and recreated later
+     * on demand.
+     * <p>
+     * If set to {@code 0}, timeout is disabled.
+     * <p>
+     * Default value is {@link #DFLT_SESSION_EXPIRATION_TIMEOUT}.
+     *
+     * @param sessionExpirationTimeout Expiration timeout for Cassandra driver session.
+     */
+    public void setSessionExpirationTimeout(long sessionExpirationTimeout) {
+        this.sessionExpirationTimeout = sessionExpirationTimeout;
+
+        invalidate();
+    }
+
     /**
      * Creates Cassandra session wrapper if it wasn't created yet and returns it
      *
@@ -523,7 +546,8 @@ public synchronized CassandraSession session(IgniteLogger log) {
         if (nettyOptions != null)
             builder = builder.withNettyOptions(nettyOptions);
 
-        return ses = new CassandraSessionImpl(builder, fetchSize, readConsistency, writeConsistency, log);
+        return ses = new CassandraSessionImpl(
+            builder, fetchSize, readConsistency, writeConsistency, sessionExpirationTimeout, log);
     }
 
     /**
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java
index 95b8581c5c793..cee776bb27363 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java
@@ -17,6 +17,13 @@
 
 package org.apache.ignite.cache.store.cassandra.session;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.Cache;
 import com.datastax.driver.core.BatchStatement;
 import com.datastax.driver.core.BoundStatement;
 import com.datastax.driver.core.Cluster;
@@ -30,13 +37,6 @@
 import com.datastax.driver.core.exceptions.AlreadyExistsException;
 import com.datastax.driver.core.exceptions.InvalidQueryException;
 import com.datastax.driver.core.querybuilder.Batch;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.cache.Cache;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.store.cassandra.common.CassandraHelper;
@@ -82,6 +82,9 @@ public class CassandraSessionImpl implements CassandraSession {
     /** Consistency level for Cassandra WRITE operations (insert/update/delete). */
     private ConsistencyLevel writeConsistency;
 
+    /** Expiration timeout. */
+    private long expirationTimeout;
+
     /** Logger. */
     private IgniteLogger log;
 
@@ -101,11 +104,12 @@ public class CassandraSessionImpl implements CassandraSession {
      * @param log Logger.
      */
     public CassandraSessionImpl(Cluster.Builder builder, Integer fetchSize, ConsistencyLevel readConsistency,
-        ConsistencyLevel writeConsistency, IgniteLogger log) {
+        ConsistencyLevel writeConsistency, long expirationTimeout, IgniteLogger log) {
         this.builder = builder;
         this.fetchSize = fetchSize;
         this.readConsistency = readConsistency;
         this.writeConsistency = writeConsistency;
+        this.expirationTimeout = expirationTimeout;
         this.log = log;
     }
 
@@ -404,7 +408,8 @@ else if (CassandraHelper.isPreparedStatementClusterError(e))
     /** {@inheritDoc} */
     @Override public synchronized void close() throws IOException {
         if (decrementSessionRefs() == 0 && ses != null) {
-            SessionPool.put(this, ses);
+            SessionPool.put(this, ses, expirationTimeout);
+
             ses = null;
         }
     }
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java
index fc4a907e6e06e..86db713482160 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java
@@ -17,13 +17,13 @@
 
 package org.apache.ignite.cache.store.cassandra.session.pool;
 
-import com.datastax.driver.core.Session;
 import java.lang.Thread.State;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import com.datastax.driver.core.Session;
 import org.apache.ignite.cache.store.cassandra.session.CassandraSessionImpl;
 
 /**
@@ -98,14 +98,14 @@ private static class SessionMonitor extends Thread {
      * @param cassandraSes Session wrapper.
      * @param driverSes Driver session.
      */
-    public static void put(CassandraSessionImpl cassandraSes, Session driverSes) {
+    public static void put(CassandraSessionImpl cassandraSes, Session driverSes, long expirationTimeout) {
         if (cassandraSes == null || driverSes == null)
             return;
 
         SessionWrapper old;
 
         synchronized (sessions) {
-            old = sessions.put(cassandraSes, new SessionWrapper(driverSes));
+            old = sessions.put(cassandraSes, new SessionWrapper(driverSes, expirationTimeout));
 
             if (monitorSingleton == null || State.TERMINATED.equals(monitorSingleton.getState())) {
                 monitorSingleton = new SessionMonitor();
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java
index 7c5722bb8f21f..68b9dd480f8b8 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java
@@ -24,12 +24,12 @@
  * Wrapper for Cassandra driver session, responsible for monitoring session expiration and its closing.
  */
 public class SessionWrapper {
-    /** Expiration timeout for Cassandra driver session. */
-    public static final long DFLT_EXPIRATION_TIMEOUT = 300000;  // 5 minutes.
-
     /** Cassandra driver session. */
     private Session ses;
 
+    /** Expiration timeout. */
+    private long expirationTimeout;
+
     /** Wrapper creation time.  */
     private long time;
 
@@ -38,9 +38,11 @@ public class SessionWrapper {
      *
      * @param ses Cassandra driver session.
      */
-    public SessionWrapper(Session ses) {
+    public SessionWrapper(Session ses, long expirationTimeout) {
         this.ses = ses;
-        this.time = System.currentTimeMillis();
+        this.expirationTimeout = expirationTimeout;
+
+        time = System.currentTimeMillis();
     }
 
     /**
@@ -49,7 +51,7 @@ public SessionWrapper(Session ses) {
      * @return true if session expired.
      */
     public boolean expired() {
-        return System.currentTimeMillis() - time > DFLT_EXPIRATION_TIMEOUT;
+        return expirationTimeout > 0 && System.currentTimeMillis() - time > expirationTimeout;
     }
 
     /**
@@ -66,6 +68,7 @@ public Session driverSession() {
      */
     public void release() {
         CassandraHelper.closeSession(ses);
+
         ses = null;
     }
 }

From a4c70f12305c15cbf15a6e5eb7c500f883a8b317 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Thu, 20 Apr 2017 20:45:58 +0300
Subject: [PATCH 249/446] ignite-5041 NPE in deadlock detection fixed

---
 .../cache/DynamicCacheDescriptor.java         |  24 ++
 .../cache/GridCacheSharedContext.java         |  16 ++
 .../cache/transactions/IgniteTxManager.java   |  21 +-
 .../cache/transactions/TxDeadlock.java        |  19 +-
 .../cache/transactions/TxLocksResponse.java   |  37 +--
 ...adlockDetectionMessageMarshallingTest.java | 116 +++++++++
 ...xDeadlockDetectionUnmasrhalErrorsTest.java | 225 ++++++++++++++++++
 .../transactions/TxDeadlockNpeClientTest.java | 220 +++++++++++++++++
 .../TxDeadlockDetectionTestSuite.java         |   4 +
 9 files changed, 655 insertions(+), 27 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionUnmasrhalErrorsTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
index 8b62f03cb8649..1d0346a04e42a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
@@ -20,9 +20,11 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.plugin.CachePluginManager;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -81,6 +83,12 @@ public class DynamicCacheDescriptor {
     /** */
     private AffinityTopologyVersion rcvdFromVer;
 
+    /** Mutex. */
+    private final Object mux = new Object();
+
+    /** Cached object context for marshalling issues when cache isn't started. */
+    private volatile CacheObjectContext objCtx;
+
     /**
      * @param ctx Context.
      * @param cacheCfg Cache configuration.
@@ -209,6 +217,22 @@ public CacheConfiguration cacheConfiguration() {
         return cacheCfg;
     }
 
+    /**
+     * Creates and caches cache object context if needed.
+     *
+     * @param proc Object processor.
+     */
+    public CacheObjectContext cacheObjectContext(IgniteCacheObjectProcessor proc) throws IgniteCheckedException {
+        if (objCtx == null) {
+            synchronized (mux) {
+                if (objCtx == null)
+                    objCtx = proc.contextForCache(cacheCfg);
+            }
+        }
+
+        return objCtx;
+    }
+
     /**
      * @return Cache plugin manager.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index 117a5c33189d7..da5ff2e2714fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -392,6 +392,22 @@ public GridCacheContext<K, V> cacheContext(int cacheId) {
         return ctxMap.get(cacheId);
     }
 
+    /**
+     * Returns cache object context if created or creates new and caches it until cache started.
+     *
+     * @param cacheId Cache id.
+     */
+    public @Nullable CacheObjectContext cacheObjectContext(int cacheId) throws IgniteCheckedException {
+        GridCacheContext<K, V> ctx = ctxMap.get(cacheId);
+
+        if (ctx != null)
+            return ctx.cacheObjectContext();
+
+        DynamicCacheDescriptor desc = cache().cacheDescriptor(cacheId);
+
+        return desc != null ? desc.cacheObjectContext(kernalContext().cacheObjects()) : null;
+    }
+
     /**
      * @return Grid name.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 24f8ea918b07a..1ba0102408eb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -31,7 +31,6 @@
 import java.util.concurrent.ConcurrentMap;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteClientDisconnectedException;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryObjectException;
 import org.apache.ignite.cluster.ClusterNode;
@@ -2479,11 +2478,18 @@ private class DeadlockDetectionListener implements GridMessageListener {
         @Override public void onMessage(UUID nodeId, Object msg) {
             GridCacheMessage cacheMsg = (GridCacheMessage)msg;
 
-            unmarshall(nodeId, cacheMsg);
+            Throwable err = null;
 
-            if (cacheMsg.classError() != null) {
+            try {
+                unmarshall(nodeId, cacheMsg);
+            }
+            catch (Exception e) {
+                err = e;
+            }
+
+            if (err != null || cacheMsg.classError() != null) {
                 try {
-                    processFailedMessage(nodeId, cacheMsg);
+                    processFailedMessage(nodeId, cacheMsg, err);
                 }
                 catch(Throwable e){
                     U.error(log, "Failed to process message [senderId=" + nodeId +
@@ -2536,7 +2542,7 @@ else if (msg instanceof TxLocksResponse) {
          * @param nodeId Node ID.
          * @param msg Message.
          */
-        private void processFailedMessage(UUID nodeId, GridCacheMessage msg) throws IgniteCheckedException {
+        private void processFailedMessage(UUID nodeId, GridCacheMessage msg, Throwable err) throws IgniteCheckedException {
             switch (msg.directType()) {
                 case -24: {
                     TxLocksRequest req = (TxLocksRequest)msg;
@@ -2568,7 +2574,10 @@ private void processFailedMessage(UUID nodeId, GridCacheMessage msg) throws Igni
                         return;
                     }
 
-                    fut.onResult(nodeId, res);
+                    if (err == null)
+                        fut.onResult(nodeId, res);
+                    else
+                        fut.onDone(null, err);
                 }
 
                 break;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlock.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlock.java
index f8130e11a2abd..97db6988052c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlock.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlock.java
@@ -21,11 +21,10 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
@@ -133,11 +132,21 @@ public String toString(GridCacheSharedContext ctx) {
         for (Map.Entry<IgniteTxKey, String> e : keyLabels.entrySet()) {
             IgniteTxKey txKey = e.getKey();
 
-            GridCacheContext cctx = ctx.cacheContext(txKey.cacheId());
+            try {
+                CacheObjectContext objCtx = ctx.cacheObjectContext(txKey.cacheId());
 
-            Object val = CU.value(txKey.key(), cctx, true);
+                Object val = txKey.key().value(objCtx, true);
 
-            sb.append(e.getValue()).append(" [key=").append(val).append(", cache=").append(cctx.namexx()).append("]\n");
+                sb.append(e.getValue())
+                    .append(" [key=")
+                    .append(val)
+                    .append(", cache=")
+                    .append(objCtx.cacheName())
+                    .append("]\n");
+            }
+            catch (Exception ex) {
+                sb.append("Unable to unmarshall deadlock information for key [key=").append(e.getValue()).append("]\n");
+            }
         }
 
         return sb.toString();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java
index fa6afdd0b8a8e..a2f3037afaa08 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java
@@ -181,31 +181,36 @@ public void addKey(IgniteTxKey key) {
 
     /** {@inheritDoc} */
     @Override public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr) throws IgniteCheckedException {
-        super.finishUnmarshal(ctx, ldr);
+        try {
+            super.finishUnmarshal(ctx, ldr);
 
-        if (nearTxKeysArr != null) {
-            for (int i = 0; i < nearTxKeysArr.length; i++) {
-                IgniteTxKey key = nearTxKeysArr[i];
+            if (nearTxKeysArr != null) {
+                for (int i = 0; i < nearTxKeysArr.length; i++) {
+                    IgniteTxKey txKey = nearTxKeysArr[i];
 
-                key.finishUnmarshal(ctx.cacheContext(key.cacheId()), ldr);
+                    txKey.key().finishUnmarshal(ctx.cacheObjectContext(txKey.cacheId()), ldr);
 
-                txLocks().put(key, locksArr[i]);
+                    txLocks().put(txKey, locksArr[i]);
+                }
+
+                nearTxKeysArr = null;
+                locksArr = null;
             }
 
-            nearTxKeysArr = null;
-            locksArr = null;
-        }
+            if (txKeysArr != null) {
+                txKeys = U.newHashSet(txKeysArr.length);
 
-        if (txKeysArr != null) {
-            txKeys = U.newHashSet(txKeysArr.length);
+                for (IgniteTxKey txKey : txKeysArr) {
+                    txKey.key().finishUnmarshal(ctx.cacheObjectContext(txKey.cacheId()), ldr);
 
-            for (IgniteTxKey key : txKeysArr) {
-                key.finishUnmarshal(ctx.cacheContext(key.cacheId()), ldr);
+                    txKeys.add(txKey);
+                }
 
-                txKeys.add(key);
+                txKeysArr = null;
             }
-
-            txKeysArr = null;
+        }
+        catch (Exception e) {
+            throw new IgniteCheckedException(e);
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
new file mode 100644
index 0000000000000..eafd09f2fdbf9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.managers.communication.GridMessageListener;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class TxDeadlockDetectionMessageMarshallingTest extends GridCommonAbstractTest {
+    /** Topic. */
+    private static final String TOPIC = "mytopic";
+
+    /** Client mode. */
+    private static boolean clientMode;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(clientMode);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMessageUnmarshallWithoutCacheContext() throws Exception {
+        try {
+            Ignite ignite = startGrid(0);
+
+            CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>();
+
+            IgniteCache<Integer, Integer> cache = ignite.getOrCreateCache(ccfg);
+
+            clientMode = true;
+
+            Ignite client = startGrid(1);
+
+            final GridCacheSharedContext<Object, Object> clientCtx = ((IgniteKernal)client).context().cache().context();
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            final AtomicBoolean res = new AtomicBoolean();
+
+            clientCtx.gridIO().addMessageListener(TOPIC, new GridMessageListener() {
+                @Override public void onMessage(UUID nodeId, Object msg) {
+                    if (msg instanceof TxLocksResponse) {
+                        try {
+                            ((TxLocksResponse)msg).finishUnmarshal(clientCtx, clientCtx.deploy().globalLoader());
+
+                            res.set(true);
+                        }
+                        catch (Exception e) {
+                            log.error("Message unmarshal failed", e);
+                        }
+                        finally {
+                            latch.countDown();
+                        }
+                    }
+                }
+            });
+
+            GridCacheContext cctx = ((IgniteCacheProxy)cache).context();
+
+            KeyCacheObject key = cctx.toCacheKeyObject(1);
+
+            TxLocksResponse msg = new TxLocksResponse();
+            msg.addKey(cctx.txKey(key));
+
+            msg.prepareMarshal(cctx.shared());
+
+            ((IgniteKernal)ignite).context().cache().context().gridIO().send(
+                ((IgniteKernal)client).getLocalNodeId(), TOPIC, msg, GridIoPolicy.PUBLIC_POOL);
+
+            boolean await = latch.await(1, TimeUnit.SECONDS);
+
+            assertTrue(await && res.get());
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionUnmasrhalErrorsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionUnmasrhalErrorsTest.java
new file mode 100644
index 0000000000000..598725e328f5f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionUnmasrhalErrorsTest.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionDeadlockException;
+import org.apache.ignite.transactions.TransactionTimeoutException;
+
+import static org.apache.ignite.internal.util.typedef.X.hasCause;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ *
+ */
+public class TxDeadlockDetectionUnmasrhalErrorsTest extends GridCommonAbstractTest {
+    /** Nodes count. */
+    private static final int NODES_CNT = 2;
+
+    /** Client. */
+    private static boolean client;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(client);
+
+        if (isDebug()) {
+            TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+            discoSpi.failureDetectionTimeoutEnabled(false);
+
+            cfg.setDiscoverySpi(discoSpi);
+        }
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid(0);
+
+        client = true;
+
+        startGrid(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeadlockCacheObjectContext() throws Exception {
+        IgniteCache<Integer, Integer> cache0 = null;
+        IgniteCache<Integer, Integer> cache1 = null;
+        try {
+            cache0 = getCache(ignite(0), "cache0");
+            cache1 = getCache(ignite(0), "cache1");
+
+            IgniteCache<Integer, Integer> clientCache0 = grid(1).cache("cache0");
+
+            awaitPartitionMapExchange();
+
+            final CyclicBarrier barrier = new CyclicBarrier(2);
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            final AtomicInteger threadCnt = new AtomicInteger();
+
+            final AtomicBoolean deadlock = new AtomicBoolean();
+
+            IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+                @Override public void run() {
+                    int threadNum = threadCnt.getAndIncrement();
+
+                    Ignite ignite = ignite(0);
+
+                    IgniteCache<Integer, Integer> cache1 = ignite.cache("cache" + (threadNum == 0 ? 0 : 1));
+
+                    IgniteCache<Integer, Integer> cache2 = ignite.cache("cache" + (threadNum == 0 ? 1 : 0));
+
+                    try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 1000, 0)) {
+                        int key1 = threadNum == 0 ? 0 : 1;
+
+                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
+                            ", tx=" + tx + ", key=" + key1 + ", cache=" + cache1.getName() + ']');
+
+                        cache1.put(key1, 0);
+
+                        barrier.await();
+
+                        int key2 = threadNum == 0 ? 1 : 0;
+
+                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
+                            ", tx=" + tx + ", key=" + key2 + ", cache=" + cache2.getName() + ']');
+
+                        latch.countDown();
+
+                        cache2.put(key2, 1);
+
+                        tx.commit();
+
+                        log.info(">>> Commit done");
+                    }
+                    catch (Throwable e) {
+                        // At least one stack trace should contain TransactionDeadlockException.
+                        if (hasCause(e, TransactionTimeoutException.class) &&
+                            hasCause(e, TransactionDeadlockException.class)
+                            ) {
+                            if (deadlock.compareAndSet(false, true))
+                                U.error(log, "At least one stack trace should contain " +
+                                    TransactionDeadlockException.class.getSimpleName(), e);
+                        }
+                    }
+                }
+            }, 2, "tx-thread");
+
+            latch.await();
+
+            Ignite client = grid(1);
+
+            try (Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 500, 0)) {
+                clientCache0.put(0, 3);
+                clientCache0.put(1, 3);
+
+                tx.commit();
+
+                log.info(">>> Commit done");
+            }
+            catch (CacheException e) {
+                assertTrue(X.hasCause(e, TransactionTimeoutException.class));
+            }
+            catch (Throwable e) {
+                log.error("Unexpected exception occurred", e);
+
+                fail();
+            }
+
+            fut.get();
+
+            assertTrue(deadlock.get());
+
+            for (int i = 0; i < NODES_CNT ; i++) {
+                Ignite ignite = ignite(i);
+
+                IgniteTxManager txMgr = ((IgniteKernal)ignite).context().cache().context().tm();
+
+                Collection<IgniteInternalFuture<?>> futs = txMgr.deadlockDetectionFutures();
+
+                assertTrue(futs.isEmpty());
+            }
+
+            //assertNotNull(grid(1).context().cache().context().cacheContext(cacheId));
+        }
+        finally {
+            if (cache0 != null)
+                cache0.destroy();
+
+            if (cache1 != null)
+                cache1.destroy();
+        }
+    }
+
+
+
+    /**
+     * @param ignite Ignite.
+     * @param name Name.
+     */
+    private IgniteCache<Integer, Integer> getCache(Ignite ignite, String name) {
+        CacheConfiguration ccfg = defaultCacheConfiguration();
+
+        ccfg.setName(name);
+        ccfg.setCacheMode(CacheMode.PARTITIONED);
+        ccfg.setBackups(0);
+        ccfg.setNearConfiguration(null);
+
+        return ignite.getOrCreateCache(ccfg);
+    }
+
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java
new file mode 100644
index 0000000000000..f75750916728c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionDeadlockException;
+import org.apache.ignite.transactions.TransactionTimeoutException;
+
+import static org.apache.ignite.internal.util.typedef.X.hasCause;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ *
+ */
+public class TxDeadlockNpeClientTest extends GridCommonAbstractTest {
+    /** Nodes count. */
+    private static final int NODES_CNT = 2;
+
+    /** Client. */
+    private static boolean client;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(client);
+
+        if (isDebug()) {
+            TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+            discoSpi.failureDetectionTimeoutEnabled(false);
+
+            cfg.setDiscoverySpi(discoSpi);
+        }
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid(0);
+
+        client = true;
+
+        startGrid(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeadlockCacheObjectContext() throws Exception {
+        IgniteCache<Integer, Integer> cache0 = null;
+        IgniteCache<Integer, Integer> cache1 = null;
+
+        try {
+            cache0 = getCache(ignite(0), "cache0");
+            cache1 = getCache(ignite(0), "cache1");
+
+            IgniteCache<Integer, Integer> clientCache0 = grid(1).cache("cache0");
+
+            awaitPartitionMapExchange();
+
+            final CyclicBarrier barrier = new CyclicBarrier(2);
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            final AtomicInteger threadCnt = new AtomicInteger();
+
+            final AtomicBoolean deadlock = new AtomicBoolean();
+
+            IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+                @Override public void run() {
+                    int threadNum = threadCnt.getAndIncrement();
+
+                    Ignite ignite = ignite(0);
+
+                    IgniteCache<Integer, Integer> cache1 = ignite.cache("cache" + (threadNum == 0 ? 0 : 1));
+
+                    IgniteCache<Integer, Integer> cache2 = ignite.cache("cache" + (threadNum == 0 ? 1 : 0));
+
+                    try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 1000, 0)) {
+                        int key1 = threadNum == 0 ? 0 : 1;
+
+                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
+                            ", tx=" + tx + ", key=" + key1 + ", cache=" + cache1.getName() + ']');
+
+                        cache1.put(key1, 0);
+
+                        barrier.await();
+
+                        int key2 = threadNum == 0 ? 1 : 0;
+
+                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
+                            ", tx=" + tx + ", key=" + key2 + ", cache=" + cache2.getName() + ']');
+
+                        latch.countDown();
+
+                        cache2.put(key2, 1);
+
+                        tx.commit();
+
+                        log.info(">>> Commit done");
+                    }
+                    catch (Throwable e) {
+                        // At least one stack trace should contain TransactionDeadlockException.
+                        if (hasCause(e, TransactionTimeoutException.class) &&
+                            hasCause(e, TransactionDeadlockException.class)
+                            ) {
+                            if (deadlock.compareAndSet(false, true))
+                                U.error(log, "At least one stack trace should contain " +
+                                    TransactionDeadlockException.class.getSimpleName(), e);
+                        }
+                    }
+                }
+            }, 2, "tx-thread");
+
+            latch.await();
+
+            Ignite client = grid(1);
+
+            try (Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 500, 0)) {
+                clientCache0.put(0, 3);
+                clientCache0.put(1, 3);
+
+                tx.commit();
+
+                log.info(">>> Commit done");
+            }
+            catch (CacheException e) {
+                assertTrue(X.hasCause(e, TransactionTimeoutException.class));
+            }
+            catch (Throwable e) {
+                fail();
+            }
+
+            fut.get();
+
+            assertTrue(deadlock.get());
+
+            for (int i = 0; i < NODES_CNT ; i++) {
+                Ignite ignite = ignite(i);
+
+                IgniteTxManager txMgr = ((IgniteKernal)ignite).context().cache().context().tm();
+
+                Collection<IgniteInternalFuture<?>> futs = txMgr.deadlockDetectionFutures();
+
+                assertTrue(futs.isEmpty());
+            }
+        }
+        finally {
+            if (cache0 != null)
+                cache0.destroy();
+
+            if (cache1 != null)
+                cache1.destroy();
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @param name Name.
+     */
+    private IgniteCache<Integer, Integer> getCache(Ignite ignite, String name) {
+        CacheConfiguration ccfg = defaultCacheConfiguration();
+
+        ccfg.setName(name);
+        ccfg.setCacheMode(CacheMode.PARTITIONED);
+        ccfg.setBackups(0);
+        ccfg.setNearConfiguration(null);
+
+        return ignite.getOrCreateCache(ccfg);
+    }
+
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java
index 5a1b1ad37168a..337d4d46a06d0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java
@@ -19,8 +19,10 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.transactions.DepthFirstSearchTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionMessageMarshallingTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionNoHangsTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionUnmasrhalErrorsTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticDeadlockDetectionCrossCacheTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticDeadlockDetectionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxPessimisticDeadlockDetectionCrossCacheTest;
@@ -44,6 +46,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TxPessimisticDeadlockDetectionCrossCacheTest.class);
         suite.addTestSuite(TxDeadlockDetectionTest.class);
         suite.addTestSuite(TxDeadlockDetectionNoHangsTest.class);
+        suite.addTestSuite(TxDeadlockDetectionUnmasrhalErrorsTest.class);
+        suite.addTestSuite(TxDeadlockDetectionMessageMarshallingTest.class);
 
         return suite;
     }

From d94172e7bb7ff4ebab422997e5c70f8cd03c0ae4 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 26 Apr 2017 15:16:46 +0300
Subject: [PATCH 250/446] Redundant test removed

---
 .../transactions/TxDeadlockNpeClientTest.java | 220 ------------------
 1 file changed, 220 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java
deleted file mode 100644
index f75750916728c..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.transactions;
-
-import java.util.Collection;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.cache.CacheException;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.transactions.Transaction;
-import org.apache.ignite.transactions.TransactionDeadlockException;
-import org.apache.ignite.transactions.TransactionTimeoutException;
-
-import static org.apache.ignite.internal.util.typedef.X.hasCause;
-import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
-import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
-import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
-
-/**
- *
- */
-public class TxDeadlockNpeClientTest extends GridCommonAbstractTest {
-    /** Nodes count. */
-    private static final int NODES_CNT = 2;
-
-    /** Client. */
-    private static boolean client;
-
-    /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        cfg.setClientMode(client);
-
-        if (isDebug()) {
-            TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
-            discoSpi.failureDetectionTimeoutEnabled(false);
-
-            cfg.setDiscoverySpi(discoSpi);
-        }
-
-        return cfg;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        super.beforeTestsStarted();
-
-        startGrid(0);
-
-        client = true;
-
-        startGrid(1);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        super.afterTestsStopped();
-
-        stopAllGrids();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testDeadlockCacheObjectContext() throws Exception {
-        IgniteCache<Integer, Integer> cache0 = null;
-        IgniteCache<Integer, Integer> cache1 = null;
-
-        try {
-            cache0 = getCache(ignite(0), "cache0");
-            cache1 = getCache(ignite(0), "cache1");
-
-            IgniteCache<Integer, Integer> clientCache0 = grid(1).cache("cache0");
-
-            awaitPartitionMapExchange();
-
-            final CyclicBarrier barrier = new CyclicBarrier(2);
-
-            final CountDownLatch latch = new CountDownLatch(1);
-
-            final AtomicInteger threadCnt = new AtomicInteger();
-
-            final AtomicBoolean deadlock = new AtomicBoolean();
-
-            IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
-                @Override public void run() {
-                    int threadNum = threadCnt.getAndIncrement();
-
-                    Ignite ignite = ignite(0);
-
-                    IgniteCache<Integer, Integer> cache1 = ignite.cache("cache" + (threadNum == 0 ? 0 : 1));
-
-                    IgniteCache<Integer, Integer> cache2 = ignite.cache("cache" + (threadNum == 0 ? 1 : 0));
-
-                    try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 1000, 0)) {
-                        int key1 = threadNum == 0 ? 0 : 1;
-
-                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
-                            ", tx=" + tx + ", key=" + key1 + ", cache=" + cache1.getName() + ']');
-
-                        cache1.put(key1, 0);
-
-                        barrier.await();
-
-                        int key2 = threadNum == 0 ? 1 : 0;
-
-                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
-                            ", tx=" + tx + ", key=" + key2 + ", cache=" + cache2.getName() + ']');
-
-                        latch.countDown();
-
-                        cache2.put(key2, 1);
-
-                        tx.commit();
-
-                        log.info(">>> Commit done");
-                    }
-                    catch (Throwable e) {
-                        // At least one stack trace should contain TransactionDeadlockException.
-                        if (hasCause(e, TransactionTimeoutException.class) &&
-                            hasCause(e, TransactionDeadlockException.class)
-                            ) {
-                            if (deadlock.compareAndSet(false, true))
-                                U.error(log, "At least one stack trace should contain " +
-                                    TransactionDeadlockException.class.getSimpleName(), e);
-                        }
-                    }
-                }
-            }, 2, "tx-thread");
-
-            latch.await();
-
-            Ignite client = grid(1);
-
-            try (Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 500, 0)) {
-                clientCache0.put(0, 3);
-                clientCache0.put(1, 3);
-
-                tx.commit();
-
-                log.info(">>> Commit done");
-            }
-            catch (CacheException e) {
-                assertTrue(X.hasCause(e, TransactionTimeoutException.class));
-            }
-            catch (Throwable e) {
-                fail();
-            }
-
-            fut.get();
-
-            assertTrue(deadlock.get());
-
-            for (int i = 0; i < NODES_CNT ; i++) {
-                Ignite ignite = ignite(i);
-
-                IgniteTxManager txMgr = ((IgniteKernal)ignite).context().cache().context().tm();
-
-                Collection<IgniteInternalFuture<?>> futs = txMgr.deadlockDetectionFutures();
-
-                assertTrue(futs.isEmpty());
-            }
-        }
-        finally {
-            if (cache0 != null)
-                cache0.destroy();
-
-            if (cache1 != null)
-                cache1.destroy();
-        }
-    }
-
-    /**
-     * @param ignite Ignite.
-     * @param name Name.
-     */
-    private IgniteCache<Integer, Integer> getCache(Ignite ignite, String name) {
-        CacheConfiguration ccfg = defaultCacheConfiguration();
-
-        ccfg.setName(name);
-        ccfg.setCacheMode(CacheMode.PARTITIONED);
-        ccfg.setBackups(0);
-        ccfg.setNearConfiguration(null);
-
-        return ignite.getOrCreateCache(ccfg);
-    }
-
-
-}

From f9ecacc625b458539775e6550bd9b7613ed38f21 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 28 Apr 2017 11:46:23 +0300
Subject: [PATCH 251/446] IGNITE-5077 - Support service security permissions

backport from master
(cherry picked from commit 6236b5f)
---
 .../processors/security/SecurityContext.java  |  9 ++++++
 .../service/GridServiceProcessor.java         | 11 +++++++
 .../security/SecurityBasicPermissionSet.java  | 17 ++++++++++
 .../plugin/security/SecurityPermission.java   | 13 ++++++--
 .../security/SecurityPermissionSet.java       |  8 +++++
 .../SecurityPermissionSetBuilder.java         | 19 +++++++++++
 .../SecurityPermissionSetBuilderTest.java     | 32 +++++++++++++++----
 .../junits/spi/GridSpiAbstractTest.java       |  5 +++
 8 files changed, 106 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContext.java
index ef46713a72f63..bf5894ef3f48c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContext.java
@@ -47,6 +47,15 @@ public interface SecurityContext {
      */
     public boolean cacheOperationAllowed(String cacheName, SecurityPermission perm);
 
+    /**
+     * Checks whether service operation is allowed.
+     *
+     * @param srvcName Service name.
+     * @param perm Permission to check.
+     * @return {@code True} if task operation is allowed.
+     */
+    public boolean serviceOperationAllowed(String srvcName, SecurityPermission perm);
+
     /**
      * Checks whether system-wide permission is allowed (excluding Visor task operations).
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 2a363e21e8ef5..d7b9abc7fc1bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -92,6 +92,7 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
@@ -496,6 +497,8 @@ public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
 
         validate(cfg);
 
+        ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
+
         if (!state.srvcCompatibility) {
             Marshaller marsh = ctx.config().getMarshaller();
 
@@ -632,6 +635,8 @@ private ServicesCompatibilityState markCompatibilityStateAsUsed() {
      * @return Future.
      */
     public IgniteInternalFuture<?> cancel(String name) {
+        ctx.security().authorize(name, SecurityPermission.SERVICE_CANCEL, null);
+
         while (true) {
             try {
                 GridFutureAdapter<?> fut = new GridFutureAdapter<>();
@@ -780,6 +785,8 @@ public Collection<ServiceDescriptor> serviceDescriptors() {
      */
     @SuppressWarnings("unchecked")
     public <T> T service(String name) {
+        ctx.security().authorize(name, SecurityPermission.SERVICE_INVOKE, null);
+
         Collection<ServiceContextImpl> ctxs;
 
         synchronized (locSvcs) {
@@ -844,6 +851,8 @@ public ServiceContextImpl serviceContext(String name) {
     @SuppressWarnings("unchecked")
     public <T> T serviceProxy(ClusterGroup prj, String name, Class<? super T> svcItf, boolean sticky, long timeout)
         throws IgniteException {
+        ctx.security().authorize(name, SecurityPermission.SERVICE_INVOKE, null);
+
         if (hasLocalNode(prj)) {
             ServiceContextImpl ctx = serviceContext(name);
 
@@ -883,6 +892,8 @@ private boolean hasLocalNode(ClusterGroup prj) {
      */
     @SuppressWarnings("unchecked")
     public <T> Collection<T> services(String name) {
+        ctx.security().authorize(name, SecurityPermission.SERVICE_INVOKE, null);
+
         Collection<ServiceContextImpl> ctxs;
 
         synchronized (locSvcs) {
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
index 5b50c56321377..7521dff84df4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
@@ -38,6 +38,9 @@ public class SecurityBasicPermissionSet implements SecurityPermissionSet {
     /** Task permissions. */
     private Map<String, Collection<SecurityPermission>> taskPerms = new HashMap<>();
 
+    /** Service permissions. */
+    private Map<String, Collection<SecurityPermission>> srvcPerms = new HashMap<>();
+
     /** System permissions. */
     private Collection<SecurityPermission> sysPerms = new ArrayList<>();
 
@@ -62,6 +65,15 @@ public void setTaskPermissions(Map<String, Collection<SecurityPermission>> taskP
         this.taskPerms = taskPerms;
     }
 
+    /**
+     * Setter for set service permission map.
+     *
+     * @param srvcPerms Service permissions.
+     */
+    public void setServicePermissions(Map<String, Collection<SecurityPermission>> srvcPerms) {
+        this.srvcPerms = srvcPerms;
+    }
+
     /**
      * Setter for set collection  system permission.
      *
@@ -90,6 +102,11 @@ public void setDefaultAllowAll(boolean dfltAllowAll) {
         return taskPerms;
     }
 
+    /** {@inheritDoc} */
+    @Override public Map<String, Collection<SecurityPermission>> servicePermissions() {
+        return srvcPerms;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public Collection<SecurityPermission> systemPermissions() {
         return sysPerms;
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
index 9f63c1ed1d2a2..54361614e62a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
@@ -21,7 +21,7 @@
 
 /**
  * Supported security permissions within grid. Permissions
- * are specified on per-cache or per-task level.
+ * are specified on per-cache, per-task or per-service level.
  */
 public enum SecurityPermission {
     /** Cache {@code read} permission. */
@@ -55,7 +55,16 @@ public enum SecurityPermission {
     ADMIN_CACHE,
 
     /** Visor admin operations permissions. */
-    ADMIN_OPS;
+    ADMIN_OPS,
+
+    /** Service deploy permission. */
+    SERVICE_DEPLOY,
+
+    /** Service cancel permission. */
+    SERVICE_CANCEL,
+
+    /** Service invoke permission. */
+    SERVICE_INVOKE;
 
     /** Enumerated values. */
     private static final SecurityPermission[] VALS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSet.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSet.java
index 99615019c10d2..5e07e421eaf78 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSet.java
@@ -58,6 +58,14 @@ public interface SecurityPermissionSet extends Serializable, LessNamingBean {
      */
     public Map<String, Collection<SecurityPermission>> cachePermissions();
 
+    /**
+     * Map of service names to service permissions. Wildcards are allowed at the
+     * end of service names.
+     *
+     * @return Map of service names to service permissions.
+     */
+    public Map<String, Collection<SecurityPermission>> servicePermissions();
+
     /**
      * Collection of system-wide permissions (events enable/disable, Visor task execution).
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
index 61ad77ce08f5e..cf38c0f0e7e90 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
@@ -57,6 +57,9 @@ public class SecurityPermissionSetBuilder {
     /** Task permissions.*/
     private Map<String, Collection<SecurityPermission>> taskPerms = new HashMap<>();
 
+    /** Service permissions.*/
+    private Map<String, Collection<SecurityPermission>> srvcPerms = new HashMap<>();
+
     /** System permissions.*/
     private List<SecurityPermission> sysPerms = new ArrayList<>();
 
@@ -99,6 +102,21 @@ public SecurityPermissionSetBuilder appendTaskPermissions(String name, SecurityP
         return this;
     }
 
+    /**
+     * Append permission set form {@link org.apache.ignite.IgniteServices service} with {@code name}.
+     *
+     * @param name  String for map some service to permission set.
+     * @param perms Permissions.
+     * @return SecurityPermissionSetBuilder refer to same permission builder.
+     */
+    public SecurityPermissionSetBuilder appendServicePermissions(String name, SecurityPermission... perms) {
+        validate(toCollection("SERVICE_"), perms);
+
+        append(srvcPerms, name, toCollection(perms));
+
+        return this;
+    }
+
     /**
      * Append permission set form {@link org.apache.ignite.IgniteCache cache} with {@code name}.
      *
@@ -215,6 +233,7 @@ public SecurityPermissionSet build() {
         permSet.setDefaultAllowAll(dfltAllowAll);
         permSet.setCachePermissions(unmodifiableMap(cachePerms));
         permSet.setTaskPermissions(unmodifiableMap(taskPerms));
+        permSet.setServicePermissions(unmodifiableMap(srvcPerms));
         permSet.setSystemPermissions(unmodifiableList(sysPerms));
 
         return permSet;
diff --git a/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java b/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
index f63f9a7da0a01..5443cfd77fd5c 100644
--- a/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
@@ -28,6 +28,8 @@
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_PUT;
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_READ;
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_REMOVE;
+import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_DEPLOY;
+import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_INVOKE;
 import static org.apache.ignite.plugin.security.SecurityPermission.TASK_CANCEL;
 import static org.apache.ignite.plugin.security.SecurityPermission.TASK_EXECUTE;
 import static org.apache.ignite.plugin.security.SecurityPermission.EVENTS_ENABLE;
@@ -41,6 +43,7 @@ public class SecurityPermissionSetBuilderTest extends GridCommonAbstractTest {
     /**
      *
      */
+    @SuppressWarnings({"ThrowableNotThrown", "ArraysAsListWithZeroOrOneArgument"})
     public void testPermissionBuilder() {
         SecurityBasicPermissionSet exp = new SecurityBasicPermissionSet();
 
@@ -56,13 +59,18 @@ public void testPermissionBuilder() {
 
         exp.setTaskPermissions(permTask);
 
+        Map<String, Collection<SecurityPermission>> permSrvc = new HashMap<>();
+        permSrvc.put("service1", Arrays.asList(SERVICE_DEPLOY));
+        permSrvc.put("service2", Arrays.asList(SERVICE_INVOKE));
+
+        exp.setServicePermissions(permSrvc);
+
         exp.setSystemPermissions(Arrays.asList(ADMIN_VIEW, EVENTS_ENABLE));
 
         final SecurityPermissionSetBuilder permsBuilder = new SecurityPermissionSetBuilder();
 
         assertThrows(log, new Callable<Object>() {
-                    @Override
-                    public Object call() throws Exception {
+                    @Override public Object call() throws Exception {
                         permsBuilder.appendCachePermissions("cache", ADMIN_VIEW);
                         return null;
                     }
@@ -71,8 +79,7 @@ public Object call() throws Exception {
         );
 
         assertThrows(log, new Callable<Object>() {
-                    @Override
-                    public Object call() throws Exception {
+                    @Override public Object call() throws Exception {
                         permsBuilder.appendTaskPermissions("task", CACHE_READ);
                         return null;
                     }
@@ -81,8 +88,7 @@ public Object call() throws Exception {
         );
 
         assertThrows(log, new Callable<Object>() {
-                    @Override
-                    public Object call() throws Exception {
+                    @Override public Object call() throws Exception {
                         permsBuilder.appendSystemPermissions(TASK_EXECUTE, CACHE_PUT);
                         return null;
                     }
@@ -90,6 +96,15 @@ public Object call() throws Exception {
                 "you can assign permission only start with [EVENTS_, ADMIN_], but you try TASK_EXECUTE"
         );
 
+        assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    permsBuilder.appendSystemPermissions(SERVICE_INVOKE, CACHE_REMOVE);
+                    return null;
+                }
+            }, IgniteException.class,
+            "you can assign permission only start with [EVENTS_, ADMIN_], but you try SERVICE_INVOKE"
+        );
+
         permsBuilder.appendCachePermissions(
                 "cache1", CACHE_PUT, CACHE_REMOVE
         ).appendCachePermissions(
@@ -98,12 +113,17 @@ public Object call() throws Exception {
                 "task1", TASK_CANCEL
         ).appendTaskPermissions(
                 "task2", TASK_EXECUTE
+        ).appendServicePermissions(
+            "service1", SERVICE_DEPLOY
+        ).appendServicePermissions(
+            "service2", SERVICE_INVOKE
         ).appendSystemPermissions(ADMIN_VIEW, EVENTS_ENABLE);
 
         SecurityPermissionSet actual = permsBuilder.build();
 
         assertEquals(exp.cachePermissions(), actual.cachePermissions());
         assertEquals(exp.taskPermissions(), actual.taskPermissions());
+        assertEquals(exp.servicePermissions(), actual.servicePermissions());
         assertEquals(exp.systemPermissions(), actual.systemPermissions());
         assertEquals(exp.defaultAllowAll(), actual.defaultAllowAll());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java
index 20b3cf2fdb354..0aeff3c08b78a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java
@@ -717,6 +717,11 @@ private static class SecurityPermissionSetImpl implements SecurityPermissionSet
             return Collections.emptyMap();
         }
 
+        /** {@inheritDoc} */
+        @Override public Map<String, Collection<SecurityPermission>> servicePermissions() {
+            return Collections.emptyMap();
+        }
+
         /** {@inheritDoc} */
         @Nullable @Override public Collection<SecurityPermission> systemPermissions() {
             return null;

From 91c899b909383c78b78b9bf0c8f233b8c75ef29e Mon Sep 17 00:00:00 2001
From: Valentin Kulichenko <valentin.kulichenko@gmail.com>
Date: Fri, 28 Apr 2017 14:48:57 +0200
Subject: [PATCH 252/446] IGNITE-5081 - Removed redundant duplication of
 permissions in SecurityPermissionSetBuilder

---
 .../SecurityPermissionSetBuilder.java         | 17 ++---
 .../SecurityPermissionSetBuilderTest.java     | 63 +++++++++++--------
 2 files changed, 46 insertions(+), 34 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
index cf38c0f0e7e90..abac541ffdb68 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
@@ -17,16 +17,17 @@
 
 package org.apache.ignite.plugin.security;
 
-import java.util.Map;
-import java.util.List;
-import java.util.HashMap;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
-import static java.util.Collections.unmodifiableList;
 import static java.util.Collections.unmodifiableMap;
+import static java.util.Collections.unmodifiableSet;
 
 /**
  * Provides a convenient way to create a permission set.
@@ -61,7 +62,7 @@ public class SecurityPermissionSetBuilder {
     private Map<String, Collection<SecurityPermission>> srvcPerms = new HashMap<>();
 
     /** System permissions.*/
-    private List<SecurityPermission> sysPerms = new ArrayList<>();
+    private Set<SecurityPermission> sysPerms = new HashSet<>();
 
     /** Default allow all.*/
     private boolean dfltAllowAll;
@@ -193,7 +194,7 @@ private void validate(Collection<String> ptrns, SecurityPermission perm) {
     private final <T> Collection<T> toCollection(T... perms) {
         assert perms != null;
 
-        Collection<T> col = new ArrayList<>(perms.length);
+        Collection<T> col = U.newHashSet(perms.length);
 
         Collections.addAll(col, perms);
 
@@ -234,7 +235,7 @@ public SecurityPermissionSet build() {
         permSet.setCachePermissions(unmodifiableMap(cachePerms));
         permSet.setTaskPermissions(unmodifiableMap(taskPerms));
         permSet.setServicePermissions(unmodifiableMap(srvcPerms));
-        permSet.setSystemPermissions(unmodifiableList(sysPerms));
+        permSet.setSystemPermissions(unmodifiableSet(sysPerms));
 
         return permSet;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java b/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
index 5443cfd77fd5c..0ac7bc739f179 100644
--- a/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
@@ -17,23 +17,24 @@
 
 package org.apache.ignite.plugin.security;
 
-import java.util.Map;
-import java.util.Arrays;
-import java.util.HashMap;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.Callable;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_VIEW;
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_PUT;
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_READ;
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_REMOVE;
 import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_DEPLOY;
 import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_INVOKE;
+import static org.apache.ignite.plugin.security.SecurityPermission.EVENTS_ENABLE;
 import static org.apache.ignite.plugin.security.SecurityPermission.TASK_CANCEL;
 import static org.apache.ignite.plugin.security.SecurityPermission.TASK_EXECUTE;
-import static org.apache.ignite.plugin.security.SecurityPermission.EVENTS_ENABLE;
-import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_VIEW;
 import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
 
 /**
@@ -41,31 +42,30 @@
  */
 public class SecurityPermissionSetBuilderTest extends GridCommonAbstractTest {
     /**
-     *
      */
     @SuppressWarnings({"ThrowableNotThrown", "ArraysAsListWithZeroOrOneArgument"})
     public void testPermissionBuilder() {
         SecurityBasicPermissionSet exp = new SecurityBasicPermissionSet();
 
         Map<String, Collection<SecurityPermission>> permCache = new HashMap<>();
-        permCache.put("cache1", Arrays.asList(CACHE_PUT, CACHE_REMOVE));
-        permCache.put("cache2", Arrays.asList(CACHE_READ));
+        permCache.put("cache1", permissions(CACHE_PUT, CACHE_REMOVE));
+        permCache.put("cache2", permissions(CACHE_READ));
 
         exp.setCachePermissions(permCache);
 
         Map<String, Collection<SecurityPermission>> permTask = new HashMap<>();
-        permTask.put("task1", Arrays.asList(TASK_CANCEL));
-        permTask.put("task2", Arrays.asList(TASK_EXECUTE));
+        permTask.put("task1", permissions(TASK_CANCEL));
+        permTask.put("task2", permissions(TASK_EXECUTE));
 
         exp.setTaskPermissions(permTask);
 
         Map<String, Collection<SecurityPermission>> permSrvc = new HashMap<>();
-        permSrvc.put("service1", Arrays.asList(SERVICE_DEPLOY));
-        permSrvc.put("service2", Arrays.asList(SERVICE_INVOKE));
+        permSrvc.put("service1", permissions(SERVICE_DEPLOY));
+        permSrvc.put("service2", permissions(SERVICE_INVOKE));
 
         exp.setServicePermissions(permSrvc);
 
-        exp.setSystemPermissions(Arrays.asList(ADMIN_VIEW, EVENTS_ENABLE));
+        exp.setSystemPermissions(permissions(ADMIN_VIEW, EVENTS_ENABLE));
 
         final SecurityPermissionSetBuilder permsBuilder = new SecurityPermissionSetBuilder();
 
@@ -105,19 +105,18 @@ public void testPermissionBuilder() {
             "you can assign permission only start with [EVENTS_, ADMIN_], but you try SERVICE_INVOKE"
         );
 
-        permsBuilder.appendCachePermissions(
-                "cache1", CACHE_PUT, CACHE_REMOVE
-        ).appendCachePermissions(
-                "cache2", CACHE_READ
-        ).appendTaskPermissions(
-                "task1", TASK_CANCEL
-        ).appendTaskPermissions(
-                "task2", TASK_EXECUTE
-        ).appendServicePermissions(
-            "service1", SERVICE_DEPLOY
-        ).appendServicePermissions(
-            "service2", SERVICE_INVOKE
-        ).appendSystemPermissions(ADMIN_VIEW, EVENTS_ENABLE);
+        permsBuilder
+            .appendCachePermissions("cache1", CACHE_PUT)
+            .appendCachePermissions("cache1", CACHE_PUT, CACHE_REMOVE)
+            .appendCachePermissions("cache2", CACHE_READ)
+            .appendTaskPermissions("task1", TASK_CANCEL)
+            .appendTaskPermissions("task2", TASK_EXECUTE)
+            .appendTaskPermissions("task2", TASK_EXECUTE)
+            .appendServicePermissions("service1", SERVICE_DEPLOY)
+            .appendServicePermissions("service2", SERVICE_INVOKE)
+            .appendServicePermissions("service2", SERVICE_INVOKE)
+            .appendSystemPermissions(ADMIN_VIEW)
+            .appendSystemPermissions(ADMIN_VIEW, EVENTS_ENABLE);
 
         SecurityPermissionSet actual = permsBuilder.build();
 
@@ -127,4 +126,16 @@ public void testPermissionBuilder() {
         assertEquals(exp.systemPermissions(), actual.systemPermissions());
         assertEquals(exp.defaultAllowAll(), actual.defaultAllowAll());
     }
+
+    /**
+     * @param perms Permissions.
+     * @return Collection.
+     */
+    private static Collection<SecurityPermission> permissions(SecurityPermission... perms) {
+        Collection<SecurityPermission> col = U.newHashSet(perms.length);
+
+        Collections.addAll(col, perms);
+
+        return col;
+    }
 }

From b48a26b9b1e97fb8eb52c2a2f36005770922ac3d Mon Sep 17 00:00:00 2001
From: Valentin Kulichenko <valentin.kulichenko@gmail.com>
Date: Fri, 28 Apr 2017 14:53:33 +0200
Subject: [PATCH 253/446] IGNITE-5080 - Fixes in SecurityBasicPermissionSet

---
 .../security/SecurityBasicPermissionSet.java  | 89 ++++++++++++++-----
 1 file changed, 65 insertions(+), 24 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
index 7521dff84df4f..44166d936c9aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
@@ -17,10 +17,12 @@
 
 package org.apache.ignite.plugin.security;
 
-import java.util.Map;
-import java.util.HashMap;
-import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
 
@@ -33,16 +35,20 @@ public class SecurityBasicPermissionSet implements SecurityPermissionSet {
     private static final long serialVersionUID = 0L;
 
     /** Cache permissions. */
-    private Map<String, Collection<SecurityPermission>> cachePerms = new HashMap<>();
+    @GridToStringInclude
+    private Map<String, Collection<SecurityPermission>> cachePermissions = new HashMap<>();
 
     /** Task permissions. */
-    private Map<String, Collection<SecurityPermission>> taskPerms = new HashMap<>();
+    @GridToStringInclude
+    private Map<String, Collection<SecurityPermission>> taskPermissions = new HashMap<>();
 
     /** Service permissions. */
-    private Map<String, Collection<SecurityPermission>> srvcPerms = new HashMap<>();
+    @GridToStringInclude
+    private Map<String, Collection<SecurityPermission>> servicePermissions = new HashMap<>();
 
     /** System permissions. */
-    private Collection<SecurityPermission> sysPerms = new ArrayList<>();
+    @GridToStringInclude
+    private Collection<SecurityPermission> systemPermissions;
 
     /** Default allow all. */
     private boolean dfltAllowAll;
@@ -50,37 +56,43 @@ public class SecurityBasicPermissionSet implements SecurityPermissionSet {
     /**
      * Setter for set cache permission map.
      *
-     * @param cachePerms Cache permissions.
+     * @param cachePermissions Cache permissions.
      */
-    public void setCachePermissions(Map<String, Collection<SecurityPermission>> cachePerms) {
-        this.cachePerms = cachePerms;
+    public void setCachePermissions(Map<String, Collection<SecurityPermission>> cachePermissions) {
+        A.notNull(cachePermissions, "cachePermissions");
+
+        this.cachePermissions = cachePermissions;
     }
 
     /**
      * Setter for set task permission map.
      *
-     * @param taskPerms Task permissions.
+     * @param taskPermissions Task permissions.
      */
-    public void setTaskPermissions(Map<String, Collection<SecurityPermission>> taskPerms) {
-        this.taskPerms = taskPerms;
+    public void setTaskPermissions(Map<String, Collection<SecurityPermission>> taskPermissions) {
+        A.notNull(taskPermissions, "taskPermissions");
+
+        this.taskPermissions = taskPermissions;
     }
 
     /**
      * Setter for set service permission map.
      *
-     * @param srvcPerms Service permissions.
+     * @param servicePermissions Service permissions.
      */
-    public void setServicePermissions(Map<String, Collection<SecurityPermission>> srvcPerms) {
-        this.srvcPerms = srvcPerms;
+    public void setServicePermissions(Map<String, Collection<SecurityPermission>> servicePermissions) {
+        A.notNull(taskPermissions, "servicePermissions");
+
+        this.servicePermissions = servicePermissions;
     }
 
     /**
-     * Setter for set collection  system permission.
+     * Setter for set collection system permission.
      *
-     * @param sysPerms System permissions.
+     * @param systemPermissions System permissions.
      */
-    public void setSystemPermissions(Collection<SecurityPermission> sysPerms) {
-        this.sysPerms = sysPerms;
+    public void setSystemPermissions(Collection<SecurityPermission> systemPermissions) {
+        this.systemPermissions = systemPermissions;
     }
 
     /**
@@ -94,22 +106,22 @@ public void setDefaultAllowAll(boolean dfltAllowAll) {
 
     /** {@inheritDoc} */
     @Override public Map<String, Collection<SecurityPermission>> cachePermissions() {
-        return cachePerms;
+        return cachePermissions;
     }
 
     /** {@inheritDoc} */
     @Override public Map<String, Collection<SecurityPermission>> taskPermissions() {
-        return taskPerms;
+        return taskPermissions;
     }
 
     /** {@inheritDoc} */
     @Override public Map<String, Collection<SecurityPermission>> servicePermissions() {
-        return srvcPerms;
+        return servicePermissions;
     }
 
     /** {@inheritDoc} */
     @Nullable @Override public Collection<SecurityPermission> systemPermissions() {
-        return sysPerms;
+        return systemPermissions;
     }
 
     /** {@inheritDoc} */
@@ -117,6 +129,35 @@ public void setDefaultAllowAll(boolean dfltAllowAll) {
         return dfltAllowAll;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+
+        if (!(o instanceof SecurityBasicPermissionSet))
+            return false;
+
+        SecurityBasicPermissionSet other = (SecurityBasicPermissionSet)o;
+
+        return dfltAllowAll == other.dfltAllowAll &&
+            F.eq(cachePermissions, other.cachePermissions) &&
+            F.eq(taskPermissions, other.taskPermissions) &&
+            F.eq(servicePermissions, other.servicePermissions) &&
+            F.eq(systemPermissions, other.systemPermissions);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = (dfltAllowAll ? 1 : 0);
+
+        res = 31 * res + (cachePermissions != null ? cachePermissions.hashCode() : 0);
+        res = 31 * res + (taskPermissions != null ? taskPermissions.hashCode() : 0);
+        res = 31 * res + (servicePermissions != null ? servicePermissions.hashCode() : 0);
+        res = 31 * res + (systemPermissions != null ? systemPermissions.hashCode() : 0);
+
+        return res;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(SecurityBasicPermissionSet.class, this);

From f66c23cbb9a6f2c923ebf75c58f00afaf1c0b5f3 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Wed, 3 May 2017 17:47:45 +0300
Subject: [PATCH 254/446] IGNITE-4939 Receive event before cache initialized
 fix

---
 .../processors/cache/GridCacheProcessor.java  | 20 +++++++++++
 .../cache/IgniteCacheProxyStub.java           | 33 +++++++++++++++++++
 .../CacheContinuousQueryHandler.java          |  3 +-
 3 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyStub.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 0be2072e3766d..5a54e53dbd3f5 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3345,6 +3345,26 @@ public <K, V> IgniteCacheProxy<K, V> jcache(@Nullable String name) {
         return cache;
     }
 
+    /**
+     * @param name Cache name.
+     * @param id Cache id.
+     * @return Cache instance for given name or stub if cache not created yet, but cache descriptor exists.
+     */
+    @SuppressWarnings("unchecked")
+    public <K, V> IgniteCacheProxy<K, V> safeJcache(String name, int id) {
+        assert name != null;
+
+        IgniteCacheProxy<K, V> cache = (IgniteCacheProxy<K, V>)jCacheProxies.get(name);
+
+        if (cache == null)
+            if (cacheDescriptor(id) != null && CU.isSystemCache(name))
+                cache = new IgniteCacheProxyStub<K, V>();
+            else
+                throw new IllegalArgumentException("Cache is not configured: " + name);
+
+        return cache;
+    }
+
     /**
      * @return All configured public cache instances.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyStub.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyStub.java
new file mode 100644
index 0000000000000..75489edb98e74
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyStub.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+/**
+ * Cache proxy stub.
+ */
+public class IgniteCacheProxyStub<K, V> extends IgniteCacheProxy<K, V>{
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * Empty constructor.
+     */
+    public IgniteCacheProxyStub() {
+        // No-op.
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 83edab444f8c0..1a6577d241774 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -716,7 +716,8 @@ private void notifyCallback0(UUID nodeId,
 
         GridCacheContext<K, V> cctx = cacheContext(ctx);
 
-        final IgniteCache cache = cctx.kernalContext().cache().jcache(cctx.name());
+        //returns stub if system cache not created yet
+        final IgniteCache cache = cctx.kernalContext().cache().safeJcache(cctx.name(), cctx.cacheId());
 
         if (internal) {
             if (e.isFiltered())

From 45b4d6316145d0b4b46713409f5e8fbe55ff4c41 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 4 May 2017 12:11:37 +0300
Subject: [PATCH 255/446] IGNITE-4939 Receive event before cache initialized
 fix

---
 .../ignite/internal/processors/cache/GridCacheProcessor.java  | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 5a54e53dbd3f5..ccd7ae0bf500d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3352,9 +3352,7 @@ public <K, V> IgniteCacheProxy<K, V> jcache(@Nullable String name) {
      */
     @SuppressWarnings("unchecked")
     public <K, V> IgniteCacheProxy<K, V> safeJcache(String name, int id) {
-        assert name != null;
-
-        IgniteCacheProxy<K, V> cache = (IgniteCacheProxy<K, V>)jCacheProxies.get(name);
+        IgniteCacheProxy<K, V> cache = (IgniteCacheProxy<K, V>)jCacheProxies.get(maskNull(name));
 
         if (cache == null)
             if (cacheDescriptor(id) != null && CU.isSystemCache(name))

From 075bcfca0ea22633be13cd02647e359ad6fdca16 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 4 May 2017 12:21:04 +0300
Subject: [PATCH 256/446] Fix flacky service deployment tests.

---
 .../GridServiceProcessorAbstractSelfTest.java | 17 ++++++
 ...rviceProcessorMultiNodeConfigSelfTest.java | 46 +++++++--------
 ...GridServiceProcessorMultiNodeSelfTest.java | 56 ++++++++++---------
 3 files changed, 68 insertions(+), 51 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
index 0f79855518e3d..6d91f36aab00b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
@@ -30,7 +30,9 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
+import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
 import org.apache.ignite.internal.util.typedef.CA;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.lang.IgniteFuture;
@@ -519,6 +521,21 @@ protected int actualCount(String svcName, Iterable<ServiceDescriptor> descs) {
         return sum;
     }
 
+    /**
+     * @param srvcName Service name
+     * @param expectedDeps Expected number of service deployments
+     *
+     */
+    protected boolean waitForDeployment(final String srvcName, final int expectedDeps) throws IgniteInterruptedCheckedException {
+        final Ignite g = randomGrid();
+
+        return GridTestUtils.waitForCondition(new GridAbsPredicateX() {
+            @Override public boolean applyx() {
+                return actualCount(srvcName, g.services().serviceDescriptors())  == expectedDeps;
+            }
+        }, 1500);
+    }
+
     /**
      * Counter service.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index 9da62c0abb702..c78bcaae607ac 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -22,7 +22,6 @@
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -56,7 +55,6 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        List<ServiceConfiguration> cfgs = new ArrayList<>();
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -65,6 +63,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new DummyService());
 
+        List<ServiceConfiguration> cfgs = new ArrayList<>();
+
         cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
@@ -202,43 +202,31 @@ public void testDeployLimits() throws Exception {
 
         checkCount(name, g.services().serviceDescriptors(), nodeCount());
 
-        int extraNodes = 2;
-
         CountDownLatch latch = new CountDownLatch(1);
 
         DummyService.exeLatch(name, latch);
 
+        int extraNodes = 2;
+
         startExtraNodes(extraNodes);
 
         try {
             latch.await();
 
+            waitForDeployment(name, nodeCount() + 1);
+
             checkCount(name, g.services().serviceDescriptors(), nodeCount() + 1);
         }
         finally {
             stopExtraNodes(extraNodes);
         }
 
-        assertEquals(name, 1, DummyService.cancelled(name));
-
         waitForDeployment(name, nodeCount());
 
-        checkCount(name, g.services().serviceDescriptors(), nodeCount());
-    }
-
-    /**
-     * @param srvcName Service name
-     * @param expectedDeps Expected number of service deployments
-     *
-     */
-    private boolean waitForDeployment(final String srvcName, final int expectedDeps) throws IgniteInterruptedCheckedException {
-        final Ignite g = randomGrid();
+        // Service can be redeployed when nodes is stopping one-by-one.
+        assertEquals(0, DummyService.started(name) - DummyService.cancelled(name));
 
-        return GridTestUtils.waitForCondition(new GridAbsPredicateX() {
-            @Override public boolean applyx() {
-                return actualCount(srvcName, g.services().serviceDescriptors())  == expectedDeps;
-            }
-        }, 1500);
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 
     /**
@@ -281,8 +269,12 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         try {
             latch.await();
 
-            assertEquals(name, newNodes, DummyService.started(name));
-            assertEquals(name, 0, DummyService.cancelled(name));
+            waitForDeployment(name, nodeCount() + newNodes);
+
+            // Since we start extra nodes, there may be extra start and cancel events,
+            // so we check only the difference between start and cancel and
+            // not start and cancel events individually.
+            assertEquals(name, newNodes,  DummyService.started(name) - DummyService.cancelled(name));
 
             checkCount(name, g.services().serviceDescriptors(), nodeCount() + newNodes);
         }
@@ -314,8 +306,12 @@ private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Ex
         try {
             latch.await();
 
-            assertEquals(name, servers, DummyService.started(name));
-            assertEquals(name, 0, DummyService.cancelled(name));
+            waitForDeployment(name, nodeCount() + servers);
+
+            // Since we start extra nodes, there may be extra start and cancel events,
+            // so we check only the difference between start and cancel and
+            // not start and cancel events individually.
+            assertEquals(name, servers,  DummyService.started(name) - DummyService.cancelled(name));
 
             checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index d133cf299a756..8ecceb9a65d39 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -92,7 +92,7 @@ public void testAffinityDeployUpdateTopology() throws Exception {
         // Store a cache key.
         g.cache(CACHE_NAME).put(affKey, affKey.toString());
 
-        String name = "serviceAffinityUpdateTopology";
+        final String name = "serviceAffinityUpdateTopology";
 
         IgniteServices svcs = g.services().withAsync();
 
@@ -129,9 +129,7 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
         Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
         try {
-            final int prestartedNodes = nodeCount() + 1;
-
-            String name = "serviceOnEachNodeButClientUpdateTopology";
+            final String name = "serviceOnEachNodeButClientUpdateTopology";
 
             Ignite g = randomGrid();
 
@@ -156,8 +154,8 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
             // Ensure service is deployed
             assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
 
-            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
-            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            assertEquals(name, nodeCount(), DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
 
             int servers = 2;
             int clients = 2;
@@ -171,12 +169,12 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
             try {
                 latch.await();
 
-                // Ensure service is deployed
-                assertNotNull(grid(prestartedNodes + servers - 1)
-                    .services().serviceProxy(name, Service.class, false, 2000));
+                waitForDeployment(name, servers);
 
-                TestCase.assertEquals(name, nodeCount() + servers, DummyService.started(name));
-                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+                // Since we start extra nodes, there may be extra start and cancel events,
+                // so we check only the difference between start and cancel and
+                // not start and cancel events individually.
+                assertEquals(name, nodeCount() + servers,  DummyService.started(name) - DummyService.cancelled(name));
 
                 checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
             }
@@ -197,7 +195,7 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
         Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
         try {
-            String name = "serviceOnEachNodeUpdateTopology";
+            final String name = "serviceOnEachNodeUpdateTopology";
 
             Ignite g = randomGrid();
 
@@ -231,8 +229,8 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
             // Ensure service is deployed
             assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
 
-            TestCase.assertEquals(name, prestartedNodes, DummyService.started(name));
-            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            assertEquals(name, prestartedNodes, DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
 
             int servers = 2;
             int clients = 2;
@@ -248,11 +246,13 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
             try {
                 latch.await();
 
-                // Ensure service is deployed
-                assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+                waitForDeployment(name, prestartedNodes + extraNodes);
 
-                TestCase.assertEquals(name, prestartedNodes + extraNodes, DummyService.started(name));
-                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+                // Since we start extra nodes, there may be extra start and cancel events,
+                // so we check only the difference between start and cancel and
+                // not start and cancel events individually.
+                assertEquals(name, prestartedNodes + extraNodes,
+                    DummyService.started(name) - DummyService.cancelled(name));
 
                 checkCount(name, g.services().serviceDescriptors(), prestartedNodes + extraNodes);
             }
@@ -269,7 +269,7 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
      * @throws Exception If failed.
      */
     public void testDeployLimits() throws Exception {
-            String name = "serviceWithLimitsUpdateTopology";
+        final String name = "serviceWithLimitsUpdateTopology";
 
             Ignite g = randomGrid();
 
@@ -300,24 +300,28 @@ public void testDeployLimits() throws Exception {
 
             latch.await();
 
-            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
-            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+        assertEquals(name, nodeCount(), DummyService.started(name));
+        assertEquals(name, 0, DummyService.cancelled(name));
 
             checkCount(name, g.services().serviceDescriptors(), nodeCount());
 
-            int extraNodes = 2;
-
             latch = new CountDownLatch(1);
 
             DummyService.exeLatch(name, latch);
 
-            startExtraNodes(2);
+        int extraNodes = 2;
+
+        startExtraNodes(extraNodes);
 
             try {
                 latch.await();
 
-                TestCase.assertEquals(name, totalInstances, DummyService.started(name));
-                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            waitForDeployment(name, totalInstances);
+
+            // Since we start extra nodes, there may be extra start and cancel events,
+            // so we check only the difference between start and cancel and
+            // not start and cancel events individually.
+            assertEquals(name, totalInstances,  DummyService.started(name) - DummyService.cancelled(name));
 
                 checkCount(name, g.services().serviceDescriptors(), totalInstances);
             }

From 25c06b50d46937cb39534cdf4147b862217289a2 Mon Sep 17 00:00:00 2001
From: rfqu <rfq@list.ru>
Date: Tue, 2 May 2017 19:46:44 +0300
Subject: [PATCH 257/446] ignite-4220 Support statements for JDBC and Cassandra
 store

---
 .../store/cassandra/CassandraCacheStore.java  | 16 ++++-
 .../session/LoadCacheCustomQueryWorker.java   | 26 +++++--
 .../tests/IgnitePersistentStoreTest.java      | 23 ++++--
 .../store/jdbc/CacheAbstractJdbcStore.java    | 72 +++++++++++++------
 .../CacheJdbcPojoStoreAbstractSelfTest.java   | 49 +++++++++++++
 5 files changed, 152 insertions(+), 34 deletions(-)

diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
index e8da3a7c8e511..2e1d3ea70318d 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
@@ -20,6 +20,7 @@
 import com.datastax.driver.core.BoundStatement;
 import com.datastax.driver.core.PreparedStatement;
 import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Statement;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -103,10 +104,19 @@ public CassandraCacheStore(DataSource dataSrc, KeyValuePersistenceSettings setti
             CassandraSession ses = getCassandraSession();
 
             for (Object obj : args) {
-                if (obj == null || !(obj instanceof String) || !((String)obj).trim().toLowerCase().startsWith("select"))
-                    continue;
+                LoadCacheCustomQueryWorker<K, V> task = null;
 
-                futs.add(pool.submit(new LoadCacheCustomQueryWorker<>(ses, (String) obj, controller, log, clo)));
+                if (obj instanceof Statement)
+                    task = new LoadCacheCustomQueryWorker<>(ses, (Statement)obj, controller, log, clo);
+                else if (obj instanceof String) {
+                    String qry = ((String)obj).trim();
+
+                    if (qry.toLowerCase().startsWith("select"))
+                        task = new LoadCacheCustomQueryWorker<>(ses, (String) obj, controller, log, clo);
+                }
+
+                if (task != null)
+                    futs.add(pool.submit(task));
             }
 
             for (Future<?> fut : futs)
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/LoadCacheCustomQueryWorker.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/LoadCacheCustomQueryWorker.java
index d3ace7d436e81..d186b98cf72fa 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/LoadCacheCustomQueryWorker.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/LoadCacheCustomQueryWorker.java
@@ -36,8 +36,8 @@ public class LoadCacheCustomQueryWorker<K, V> implements Callable<Void> {
     /** Cassandra session to execute CQL query */
     private final CassandraSession ses;
 
-    /** User query. */
-    private final String qry;
+    /** Statement. */
+    private final Statement stmt;
 
     /** Persistence controller */
     private final PersistenceController ctrl;
@@ -49,12 +49,28 @@ public class LoadCacheCustomQueryWorker<K, V> implements Callable<Void> {
     private final IgniteBiInClosure<K, V> clo;
 
     /**
+     * @param ses Session.
+     * @param qry Query.
+     * @param ctrl Control.
+     * @param log Logger.
      * @param clo Closure for loaded values.
      */
     public LoadCacheCustomQueryWorker(CassandraSession ses, String qry, PersistenceController ctrl,
-        IgniteLogger log, IgniteBiInClosure<K, V> clo) {
+                                      IgniteLogger log, IgniteBiInClosure<K, V> clo) {
+        this(ses, new SimpleStatement(qry.trim().endsWith(";") ? qry : qry + ';'), ctrl, log, clo);
+    }
+
+    /**
+     * @param ses Session.
+     * @param stmt Statement.
+     * @param ctrl Control.
+     * @param log Logger.
+     * @param clo Closure for loaded values.
+     */
+    public LoadCacheCustomQueryWorker(CassandraSession ses, Statement stmt, PersistenceController ctrl,
+                                      IgniteLogger log, IgniteBiInClosure<K, V> clo) {
         this.ses = ses;
-        this.qry = qry.trim().endsWith(";") ? qry : qry + ";";
+        this.stmt = stmt;
         this.ctrl = ctrl;
         this.log = log;
         this.clo = clo;
@@ -70,7 +86,7 @@ public LoadCacheCustomQueryWorker(CassandraSession ses, String qry, PersistenceC
 
             /** {@inheritDoc} */
             @Override public Statement getStatement() {
-                return new SimpleStatement(qry);
+                return stmt;
             }
 
             /** {@inheritDoc} */
diff --git a/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java b/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
index 5da6ba25dabea..51d08855388c6 100644
--- a/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
+++ b/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.tests;
 
+import com.datastax.driver.core.SimpleStatement;
 import java.util.Collection;
 import java.util.Map;
 import org.apache.ignite.Ignite;
@@ -35,6 +36,7 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.Assert;
 import org.springframework.core.io.ClassPathResource;
 
 /**
@@ -346,20 +348,29 @@ public void loadCacheTest() {
         LOGGER.info("Running loadCache test");
 
         try (Ignite ignite = Ignition.start("org/apache/ignite/tests/persistence/pojo/ignite-config.xml")) {
-            IgniteCache<PersonId, Person> personCache3 = ignite.getOrCreateCache(new CacheConfiguration<PersonId, Person>("cache3"));
+            CacheConfiguration<PersonId, Person> ccfg = new CacheConfiguration<>("cache3");
+
+            IgniteCache<PersonId, Person> personCache3 = ignite.getOrCreateCache(ccfg);
+
             int size = personCache3.size(CachePeekMode.ALL);
 
             LOGGER.info("Initial cache size " + size);
 
             LOGGER.info("Loading cache data from Cassandra table");
 
-            personCache3.loadCache(null, new String[] {"select * from test1.pojo_test3 limit 3"});
+            String qry = "select * from test1.pojo_test3 limit 3";
+
+            personCache3.loadCache(null, qry);
 
             size = personCache3.size(CachePeekMode.ALL);
-            if (size != 3) {
-                throw new RuntimeException("Cache data was incorrectly loaded from Cassandra. " +
-                    "Expected number of records is 3, but loaded number of records is " + size);
-            }
+            Assert.assertEquals("Cache data was incorrectly loaded from Cassandra table by '" + qry + "'", 3, size);
+
+            personCache3.clear();
+
+            personCache3.loadCache(null, new SimpleStatement(qry));
+
+            size = personCache3.size(CachePeekMode.ALL);
+            Assert.assertEquals("Cache data was incorrectly loaded from Cassandra table by statement", 3, size);
 
             LOGGER.info("Cache data loaded from Cassandra table");
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
index e7ce52651a868..e211fadf2fb74 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
@@ -81,7 +81,6 @@
 import static org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory.DFLT_PARALLEL_LOAD_CACHE_MINIMUM_THRESHOLD;
 import static org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory.DFLT_WRITE_ATTEMPTS;
 import static org.apache.ignite.cache.store.jdbc.JdbcTypesTransformer.NUMERIC_TYPES;
-import static org.apache.ignite.cache.store.jdbc.JdbcTypesTransformer.NUMERIC_TYPES;
 
 /**
  * Implementation of {@link CacheStore} backed by JDBC.
@@ -753,17 +752,34 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
                         }
                     }))
                         throw new CacheLoaderException("Provided key type is not found in store or cache configuration " +
-                            "[cache=" + U.maskName(cacheName) + ", key=" + keyType + "]");
-
-                    String qry = args[i + 1].toString();
+                            "[cache=" + U.maskName(cacheName) + ", key=" + keyType + ']');
 
                     EntryMapping em = entryMapping(cacheName, typeIdForTypeName(kindForName(keyType), keyType));
 
-                    if (log.isInfoEnabled())
-                        log.info("Started load cache using custom query [cache=" + U.maskName(cacheName) +
-                            ", keyType=" + keyType + ", query=" + qry + "]");
+                    Object arg = args[i + 1];
+
+                    LoadCacheCustomQueryWorker<K, V> task;
+
+                    if (arg instanceof PreparedStatement) {
+                        PreparedStatement stmt = (PreparedStatement)arg;
+
+                        if (log.isInfoEnabled())
+                            log.info("Started load cache using custom statement [cache=" + U.maskName(cacheName) +
+                                      ", keyType=" + keyType + ", stmt=" + stmt + ']');
+
+                        task = new LoadCacheCustomQueryWorker<>(em, stmt, clo);
+                    }
+                    else {
+                        String qry = arg.toString();
+
+                        if (log.isInfoEnabled())
+                              log.info("Started load cache using custom query [cache=" + U.maskName(cacheName) +
+                                  ", keyType=" + keyType + ", query=" + qry + ']');
+
+                        task = new LoadCacheCustomQueryWorker<>(em, qry, clo);
+                    }
 
-                    futs.add(pool.submit(new LoadCacheCustomQueryWorker<>(em, qry, clo)));
+                    futs.add(pool.submit(task));
                 }
             }
             else {
@@ -778,7 +794,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
                     processedKeyTypes.add(keyType);
 
                     if (log.isInfoEnabled())
-                        log.info("Started load cache [cache=" + U.maskName(cacheName) + ", keyType=" + keyType + "]");
+                        log.info("Started load cache [cache=" + U.maskName(cacheName) + ", keyType=" + keyType + ']');
 
                     if (parallelLoadCacheMinThreshold > 0) {
                         Connection conn = null;
@@ -795,7 +811,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
                             if (rs.next()) {
                                 if (log.isDebugEnabled())
                                     log.debug("Multithread loading entries from db [cache=" + U.maskName(cacheName) +
-                                        ", keyType=" + keyType + "]");
+                                        ", keyType=" + keyType + ']');
 
                                 int keyCnt = em.keyCols.size();
 
@@ -824,7 +840,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
                         }
                         catch (SQLException e) {
                             log.warning("Failed to load entries from db in multithreaded mode, will try in single thread " +
-                                "[cache=" + U.maskName(cacheName) + ", keyType=" + keyType + " ]", e);
+                                "[cache=" + U.maskName(cacheName) + ", keyType=" + keyType + ']', e);
                         }
                         finally {
                             U.closeQuiet(conn);
@@ -833,7 +849,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
 
                     if (log.isDebugEnabled())
                         log.debug("Single thread loading entries from db [cache=" + U.maskName(cacheName) +
-                            ", keyType=" + keyType + "]");
+                            ", keyType=" + keyType + ']');
 
                     futs.add(pool.submit(loadCacheFull(em, clo)));
                 }
@@ -860,7 +876,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
         EntryMapping em = entryMapping(session().cacheName(), typeIdForObject(key));
 
         if (log.isDebugEnabled())
-            log.debug("Load value from db [table= " + em.fullTableName() + ", key=" + key + "]");
+            log.debug("Load value from db [table= " + em.fullTableName() + ", key=" + key + ']');
 
         Connection conn = null;
 
@@ -1954,12 +1970,26 @@ private class LoadCacheCustomQueryWorker<K1, V1> implements Callable<Void> {
         /** Entry mapping description. */
         private final EntryMapping em;
 
+        /** User statement. */
+        private PreparedStatement stmt;
+
         /** User query. */
-        private final String qry;
+        private String qry;
 
         /** Closure for loaded values. */
         private final IgniteBiInClosure<K1, V1> clo;
 
+        /**
+         * @param em Entry mapping description.
+         * @param stmt User statement.
+         * @param clo Closure for loaded values.
+         */
+        private LoadCacheCustomQueryWorker(EntryMapping em, PreparedStatement stmt, IgniteBiInClosure<K1, V1> clo) {
+            this.em = em;
+            this.stmt = stmt;
+            this.clo = clo;
+        }
+
         /**
          * @param em Entry mapping description.
          * @param qry User query.
@@ -1975,12 +2005,12 @@ private LoadCacheCustomQueryWorker(EntryMapping em, String qry, IgniteBiInClosur
         @Override public Void call() throws Exception {
             Connection conn = null;
 
-            PreparedStatement stmt = null;
-
             try {
-                conn = openConnection(true);
+                if (stmt == null) {
+                    conn = openConnection(true);
 
-                stmt = conn.prepareStatement(qry);
+                    stmt = conn.prepareStatement(qry);
+                }
 
                 stmt.setFetchSize(dialect.getFetchSize());
 
@@ -2006,9 +2036,11 @@ private LoadCacheCustomQueryWorker(EntryMapping em, String qry, IgniteBiInClosur
                 throw new CacheLoaderException("Failed to execute custom query for load cache", e);
             }
             finally {
-                U.closeQuiet(stmt);
+                if (conn != null) {
+                    U.closeQuiet(stmt);
 
-                U.closeQuiet(conn);
+                    U.closeQuiet(conn);
+                }
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
index 1de44f7243fbb..9e59769503f53 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
@@ -328,6 +328,55 @@ protected void checkCacheLoadWithSql() {
         assertEquals(PERSON_CNT, c1.size());
     }
 
+    /**
+     * Checks that data was loaded correctly with prepared statement.
+     */
+    protected void checkCacheLoadWithStatement() throws SQLException {
+        Connection conn = null;
+
+        PreparedStatement stmt = null;
+
+        try {
+            conn = getConnection();
+
+            conn.setAutoCommit(true);
+
+            String qry = "select id, org_id, name, birthday, gender from Person";
+
+            stmt = conn.prepareStatement(qry);
+
+            IgniteCache<Object, Object> c1 = grid().cache(CACHE_NAME);
+
+            c1.loadCache(null, "org.apache.ignite.cache.store.jdbc.model.PersonKey", stmt);
+
+            assertEquals(PERSON_CNT, c1.size());
+        }
+        finally {
+            U.closeQuiet(stmt);
+
+            U.closeQuiet(conn);
+        }
+
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLoadCacheWithStatement() throws Exception {
+        startTestGrid(false, false, false, false, 512);
+
+        checkCacheLoadWithStatement();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLoadCacheWithStatementTx() throws Exception {
+        startTestGrid(false, false, false, true, 512);
+
+        checkCacheLoadWithStatement();
+    }
+
     /**
      * @throws Exception If failed.
      */

From 987c182686962673e70398395cb27e94f894713b Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Mon, 15 May 2017 11:54:16 +0300
Subject: [PATCH 258/446] Fixed "IGNITE-5214 ConcurrentModificationException
 with enable DEBUG log level"

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../continuous/CacheContinuousQueryHandler.java | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 1a6577d241774..5f00d580f4231 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -998,6 +998,7 @@ private static class PartitionRecovery {
 
             synchronized (pendingEvts) {
                 if (log.isDebugEnabled()) {
+
                     log.debug("Handling event [lastFiredEvt=" + lastFiredEvt +
                         ", curTop=" + curTop +
                         ", entUpdCnt=" + entry.updateCounter() +
@@ -1126,15 +1127,15 @@ private static class PartitionRecovery {
                             break;
                     }
                 }
-            }
 
-            if (log.isDebugEnabled()) {
-                log.debug("Will send to listener the following events [entries=" + entries +
-                    ", lastFiredEvt=" + lastFiredEvt +
-                    ", curTop=" + curTop +
-                    ", entUpdCnt=" + entry.updateCounter() +
-                    ", partId=" + entry.partition() +
-                    ", pendingEvts=" + pendingEvts + ']');
+                if (log.isDebugEnabled()) {
+                    log.debug("Will send to listener the following events [entries=" + entries +
+                        ", lastFiredEvt=" + lastFiredEvt +
+                        ", curTop=" + curTop +
+                        ", entUpdCnt=" + entry.updateCounter() +
+                        ", partId=" + entry.partition() +
+                        ", pendingEvts=" + pendingEvts + ']');
+                }
             }
 
             return entries;

From a5d2b9ba3ed1e72ef3970ecfed499a97f925ff2c Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 15 May 2017 17:39:52 +0300
Subject: [PATCH 259/446] Fixed thread pools incorrect shutdown.

(cherry picked from commit dacf973)
---
 .../processors/cache/GridCacheAdapter.java    | 12 ++---
 .../ignite/internal/util/IgniteUtils.java     | 45 ++++++++++---------
 2 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index a50661e137600..7a35a2abaa303 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1132,14 +1132,14 @@ public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, bool
         if (!F.isEmpty(jobs)) {
             ExecutorService execSvc = null;
 
-            if (jobs.size() > 1) {
-                execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
+            try {
+                if (jobs.size() > 1) {
+                    execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
 
-                for (int i = 1; i < jobs.size(); i++)
-                    execSvc.execute(jobs.get(i));
-            }
+                    for (int i = 1; i < jobs.size(); i++)
+                        execSvc.execute(jobs.get(i));
+                }
 
-            try {
                 jobs.get(0).run();
             }
             finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index a204e9ac729a5..cc21b86db7c94 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1867,33 +1867,36 @@ public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
 
         ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, addrs.size()));
 
-        for (final InetAddress addr : addrs) {
-            futs.add(executor.submit(new Runnable() {
-                @Override public void run() {
-                    if (reachable(addr, reachTimeout)) {
-                        synchronized (res) {
-                            res.add(addr);
+        try {
+            for (final InetAddress addr : addrs) {
+                futs.add(executor.submit(new Runnable() {
+                    @Override public void run() {
+                        if (reachable(addr, reachTimeout)) {
+                            synchronized (res) {
+                                res.add(addr);
+                            }
                         }
                     }
-                }
-            }));
-        }
-
-        for (Future<?> fut : futs) {
-            try {
-                fut.get();
+                }));
             }
-            catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
 
-                throw new IgniteException("Thread has been interrupted.", e);
-            }
-            catch (ExecutionException e) {
-                throw new IgniteException(e);
+            for (Future<?> fut : futs) {
+                try {
+                    fut.get();
+                }
+                catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+
+                    throw new IgniteException("Thread has been interrupted.", e);
+                }
+                catch (ExecutionException e) {
+                    throw new IgniteException(e);
+                }
             }
         }
-
-        executor.shutdown();
+        finally {
+            executor.shutdown();
+        }
 
         return res;
     }

From 5e08c2951522a640bf84b6c60f03f29edd970ffc Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 15 May 2017 20:24:10 +0300
Subject: [PATCH 260/446] IGNITE-5225: Fix NPE caused by changes in
 IGNITE-4577.

(cherry picked from commit d463840)
---
 .../ignite/internal/util/IgniteUtils.java       |  4 ++--
 .../communication/tcp/TcpCommunicationSpi.java  | 17 +++++++++++------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index cc21b86db7c94..f8f1fa5bddfa0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1847,11 +1847,11 @@ public static synchronized boolean isLocalHostChanged() throws IOException {
      * @return List of reachable addresses.
      */
     public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
-        final int reachTimeout = 2000;
-
         if (addrs.isEmpty())
             return Collections.emptyList();
 
+        final int reachTimeout = 2000;
+
         if (addrs.size() == 1) {
             InetAddress addr = F.first(addrs);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 89ecc36d2ace1..60d97348d42cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2769,22 +2769,27 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
         Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
 
-        for (InetSocketAddress addr : addrs)
-            allInetAddrs.add(addr.getAddress());
+        for (InetSocketAddress addr : addrs) {
+            // Skip unresolved as addr.getAddress() can return null.
+            if(!addr.isUnresolved())
+                allInetAddrs.add(addr.getAddress());
+        }
 
         List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
 
         if (reachableInetAddrs.size() < allInetAddrs.size()) {
             LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
 
+            List<InetSocketAddress> unreachableInetAddr = new ArrayList<>(allInetAddrs.size() - reachableInetAddrs.size());
+
             for (InetSocketAddress addr : addrs) {
                 if (reachableInetAddrs.contains(addr.getAddress()))
                     addrs0.add(addr);
+                else
+                    unreachableInetAddr.add(addr);
             }
-            for (InetSocketAddress addr : addrs) {
-                if (!reachableInetAddrs.contains(addr.getAddress()))
-                    addrs0.add(addr);
-            }
+
+            addrs0.addAll(unreachableInetAddr);
 
             addrs = addrs0;
         }

From 62765b5b7598ab2a82b9c595e4a1276d9f2f5860 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 16 May 2017 11:30:29 +0300
Subject: [PATCH 261/446] DirectByteBufferStreamImpl: converted asserts into
 exceptions.

(cherry picked from commit 560ef60)
---
 .../v2/DirectByteBufferStreamImplV2.java      | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
index d7dc990848094..af45263e36a8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
@@ -1,4 +1,4 @@
-/*
+ /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -27,6 +27,7 @@
 import java.util.Map;
 import java.util.RandomAccess;
 import java.util.UUID;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.direct.stream.DirectByteBufferStream;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -80,7 +81,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<byte[]> BYTE_ARR_CREATOR = new ArrayCreator<byte[]>() {
         @Override public byte[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid byte array length: " + len);
 
             switch (len) {
                 case 0:
@@ -95,7 +97,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<short[]> SHORT_ARR_CREATOR = new ArrayCreator<short[]>() {
         @Override public short[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid short array length: " + len);
 
             switch (len) {
                 case 0:
@@ -110,7 +113,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<int[]> INT_ARR_CREATOR = new ArrayCreator<int[]>() {
         @Override public int[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid int array length: " + len);
 
             switch (len) {
                 case 0:
@@ -125,7 +129,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<long[]> LONG_ARR_CREATOR = new ArrayCreator<long[]>() {
         @Override public long[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid long array length: " + len);
 
             switch (len) {
                 case 0:
@@ -140,7 +145,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<float[]> FLOAT_ARR_CREATOR = new ArrayCreator<float[]>() {
         @Override public float[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid float array length: " + len);
 
             switch (len) {
                 case 0:
@@ -155,7 +161,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<double[]> DOUBLE_ARR_CREATOR = new ArrayCreator<double[]>() {
         @Override public double[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid double array length: " + len);
 
             switch (len) {
                 case 0:
@@ -170,7 +177,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<char[]> CHAR_ARR_CREATOR = new ArrayCreator<char[]>() {
         @Override public char[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid char array length: " + len);
 
             switch (len) {
                 case 0:
@@ -185,7 +193,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<boolean[]> BOOLEAN_ARR_CREATOR = new ArrayCreator<boolean[]>() {
         @Override public boolean[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid boolean array length: " + len);
 
             switch (len) {
                 case 0:

From ebc4a1648a80fbbd485e4c351fce9bee163318f9 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 16 May 2017 11:30:29 +0300
Subject: [PATCH 262/446] DirectByteBufferStreamImpl: converted asserts into
 exceptions.

(cherry picked from commit 560ef60)
---
 .../v2/DirectByteBufferStreamImplV2.java      | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
index d7dc990848094..af45263e36a8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
@@ -1,4 +1,4 @@
-/*
+ /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -27,6 +27,7 @@
 import java.util.Map;
 import java.util.RandomAccess;
 import java.util.UUID;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.direct.stream.DirectByteBufferStream;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -80,7 +81,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<byte[]> BYTE_ARR_CREATOR = new ArrayCreator<byte[]>() {
         @Override public byte[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid byte array length: " + len);
 
             switch (len) {
                 case 0:
@@ -95,7 +97,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<short[]> SHORT_ARR_CREATOR = new ArrayCreator<short[]>() {
         @Override public short[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid short array length: " + len);
 
             switch (len) {
                 case 0:
@@ -110,7 +113,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<int[]> INT_ARR_CREATOR = new ArrayCreator<int[]>() {
         @Override public int[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid int array length: " + len);
 
             switch (len) {
                 case 0:
@@ -125,7 +129,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<long[]> LONG_ARR_CREATOR = new ArrayCreator<long[]>() {
         @Override public long[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid long array length: " + len);
 
             switch (len) {
                 case 0:
@@ -140,7 +145,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<float[]> FLOAT_ARR_CREATOR = new ArrayCreator<float[]>() {
         @Override public float[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid float array length: " + len);
 
             switch (len) {
                 case 0:
@@ -155,7 +161,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<double[]> DOUBLE_ARR_CREATOR = new ArrayCreator<double[]>() {
         @Override public double[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid double array length: " + len);
 
             switch (len) {
                 case 0:
@@ -170,7 +177,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<char[]> CHAR_ARR_CREATOR = new ArrayCreator<char[]>() {
         @Override public char[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid char array length: " + len);
 
             switch (len) {
                 case 0:
@@ -185,7 +193,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<boolean[]> BOOLEAN_ARR_CREATOR = new ArrayCreator<boolean[]>() {
         @Override public boolean[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid boolean array length: " + len);
 
             switch (len) {
                 case 0:

From 9cd7e0f8d132f9b7c496fe64f75f271ef60da5eb Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Thu, 9 Feb 2017 16:44:41 +0700
Subject: [PATCH 263/446] IGNITE-4676 Fixed hang if closure executed nested
 internal task with continuation. Added test. (cherry picked from commit
 e7a5307)

---
 .../processors/job/GridJobWorker.java         |  4 +
 .../internal/GridContinuousTaskSelfTest.java  | 79 +++++++++++++++++++
 2 files changed, 83 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index 6a00d96a0abc2..acefde72eefab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -617,6 +617,10 @@ else if (X.hasCause(e, GridServiceNotFoundException.class) ||
                 // Finish here only if not held by this thread.
                 if (!HOLD.get())
                     finishJob(res, ex, sndRes);
+                else
+                    // Make sure flag is not set for current thread.
+                    // This may happen in case of nested internal task call with continuation.
+                    HOLD.set(false);
 
                 ctx.job().currentTaskSession(null);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
index 98e3c5afc8e2d..cec288714c100 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
@@ -21,10 +21,12 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.concurrent.Callable;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
@@ -43,7 +45,9 @@
 import org.apache.ignite.compute.ComputeTaskSessionAttributeListener;
 import org.apache.ignite.compute.ComputeTaskSessionFullSupport;
 import org.apache.ignite.compute.ComputeTaskSplitAdapter;
+import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.resources.TaskContinuousMapperResource;
@@ -51,6 +55,7 @@
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Continuous task test.
@@ -195,6 +200,80 @@ public void testMultipleHoldccCalls() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If test failed.
+     */
+    public void testClosureWithNestedInternalTask() throws Exception {
+        try {
+            IgniteEx ignite = startGrid(0);
+
+            ComputeTaskInternalFuture<String> fut = ignite.context().closure().callAsync(GridClosureCallMode.BALANCE, new Callable<String>() {
+                /** */
+                @IgniteInstanceResource
+                private IgniteEx g;
+
+                @Override public String call() throws Exception {
+                    return g.compute(g.cluster()).execute(NestedHoldccTask.class, null);
+                }
+            }, ignite.cluster().nodes());
+
+            assertEquals("DONE", fut.get(3000));
+        }
+        finally {
+            stopGrid(0, true);
+        }
+    }
+
+    /** Test task with continuation. */
+    @GridInternal
+    public static class NestedHoldccTask extends ComputeTaskAdapter<String, String> {
+        /** {@inheritDoc} */
+        @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
+            @Nullable String arg) throws IgniteException {
+            Map<ComputeJob, ClusterNode> map = new HashMap<>();
+
+            for (ClusterNode node : subgrid)
+                map.put(new NestedHoldccJob(), node);
+
+            return map;
+
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public String reduce(List<ComputeJobResult> results) throws IgniteException {
+            return results.get(0).getData();
+        }
+    }
+
+    /** Test job. */
+    public static class NestedHoldccJob extends ComputeJobAdapter {
+        /** */
+        @JobContextResource
+        private ComputeJobContext jobCtx;
+
+        /** */
+        private int cnt = 0;
+
+        /** {@inheritDoc} */
+        @Override public Object execute() throws IgniteException {
+            if (cnt < 1) {
+                cnt++;
+
+                jobCtx.holdcc();
+
+                new Timer().schedule(new TimerTask() {
+                    @Override public void run() {
+                        jobCtx.callcc();
+                    }
+                }, 500);
+
+                return "NOT DONE";
+            }
+
+            return "DONE";
+        }
+    }
+
     /** */
     @SuppressWarnings({"PublicInnerClass"})
     public static class TestMultipleHoldccCallsClosure implements IgniteClosure<Object, Boolean> {

From 43bcc15127bd3fd7ac4e277da6da9e5fb6a855c0 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Thu, 30 Mar 2017 11:08:10 +0700
Subject: [PATCH 264/446] IGNITE-4838 Fixed internal task detection logic.
 Added tests. (cherry picked from commit ba68c6c)

---
 .../processors/task/GridTaskProcessor.java    |  9 ++++-
 .../internal/GridTaskExecutionSelfTest.java   | 34 +++++++++++++++++++
 ...cutionWithoutPeerClassLoadingSelfTest.java | 31 +++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |  2 ++
 4 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index d32b51c0867ca..935686456e593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -608,6 +608,13 @@ else if (task != null) {
         if (subjId == null)
             subjId = ctx.localNodeId();
 
+        boolean internal = false;
+
+        if (dep == null || taskCls == null)
+            assert deployEx != null;
+        else
+            internal = dep.internalTask(task, taskCls);
+
         // Creates task session with task name and task version.
         GridTaskSessionImpl ses = ctx.session().createTaskSession(
             sesId,
@@ -621,7 +628,7 @@ else if (task != null) {
             Collections.<ComputeJobSibling>emptyList(),
             Collections.emptyMap(),
             fullSup,
-            dep != null && dep.internalTask(task, taskCls),
+            internal,
             subjId);
 
         ComputeTaskInternalFuture<R> fut = new ComputeTaskInternalFuture<>(ses, ctx);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
index 996acd686f3cd..cc6a1eae6b2c6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
@@ -22,8 +22,10 @@
 import org.apache.ignite.GridTestTask;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteDeploymentException;
 import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
@@ -45,6 +47,20 @@ public GridTaskExecutionSelfTest() {
         super(false);
     }
 
+    /** */
+    protected boolean peerClassLoadingEnabled() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setPeerClassLoadingEnabled(peerClassLoadingEnabled());
+
+        return cfg;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         startGrid(1);
@@ -144,4 +160,22 @@ public void testJobIdCollision() throws Exception {
         for (IgniteFuture<Object> fut : futs)
             fut.get();
     }
+
+    /**
+     * Test execution of non-existing task by name IGNITE-4838.
+     *
+     * @throws Exception If failed.
+     */
+    public void testExecuteTaskWithInvalidName() throws Exception {
+        try {
+            ComputeTaskFuture<?> fut = ignite.compute().execute("invalid.task.name", null);
+
+            fut.get();
+
+            assert false : "Should never be reached due to exception thrown.";
+        }
+        catch (IgniteDeploymentException e) {
+            info("Received correct exception: " + e);
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java
new file mode 100644
index 0000000000000..45e65cd9db017
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+
+/**
+ * Task execution test.
+ */
+@GridCommonTest(group = "Kernal Self")
+public class GridTaskExecutionWithoutPeerClassLoadingSelfTest extends GridTaskExecutionSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean peerClassLoadingEnabled() {
+        return false;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 8a501fdb801d6..ae64c95fb30b6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -51,6 +51,7 @@
 import org.apache.ignite.internal.GridStopWithWaitSelfTest;
 import org.apache.ignite.internal.GridTaskCancelSingleNodeSelfTest;
 import org.apache.ignite.internal.GridTaskExecutionSelfTest;
+import org.apache.ignite.internal.GridTaskExecutionWithoutPeerClassLoadingSelfTest;
 import org.apache.ignite.internal.GridTaskFailoverAffinityRunTest;
 import org.apache.ignite.internal.GridTaskFailoverSelfTest;
 import org.apache.ignite.internal.GridTaskFutureImplStopGridSelfTest;
@@ -100,6 +101,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCancelUnusedJobSelfTest.class);
         suite.addTestSuite(GridTaskJobRejectSelfTest.class);
         suite.addTestSuite(GridTaskExecutionSelfTest.class);
+        suite.addTestSuite(GridTaskExecutionWithoutPeerClassLoadingSelfTest.class);
         suite.addTestSuite(GridFailoverSelfTest.class);
         suite.addTestSuite(GridTaskListenerSelfTest.class);
         suite.addTestSuite(GridFailoverTopologySelfTest.class);

From 72882126c047b937bd5ed93c85e28262530e4977 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Thu, 30 Mar 2017 11:08:10 +0700
Subject: [PATCH 265/446] IGNITE-4838 Fixed internal task detection logic.
 Added tests. (cherry picked from commit ba68c6c)

---
 .../processors/task/GridTaskProcessor.java    |  9 ++++-
 .../internal/GridTaskExecutionSelfTest.java   | 34 +++++++++++++++++++
 ...cutionWithoutPeerClassLoadingSelfTest.java | 31 +++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |  2 ++
 4 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index d32b51c0867ca..935686456e593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -608,6 +608,13 @@ else if (task != null) {
         if (subjId == null)
             subjId = ctx.localNodeId();
 
+        boolean internal = false;
+
+        if (dep == null || taskCls == null)
+            assert deployEx != null;
+        else
+            internal = dep.internalTask(task, taskCls);
+
         // Creates task session with task name and task version.
         GridTaskSessionImpl ses = ctx.session().createTaskSession(
             sesId,
@@ -621,7 +628,7 @@ else if (task != null) {
             Collections.<ComputeJobSibling>emptyList(),
             Collections.emptyMap(),
             fullSup,
-            dep != null && dep.internalTask(task, taskCls),
+            internal,
             subjId);
 
         ComputeTaskInternalFuture<R> fut = new ComputeTaskInternalFuture<>(ses, ctx);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
index 996acd686f3cd..cc6a1eae6b2c6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
@@ -22,8 +22,10 @@
 import org.apache.ignite.GridTestTask;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteDeploymentException;
 import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
@@ -45,6 +47,20 @@ public GridTaskExecutionSelfTest() {
         super(false);
     }
 
+    /** */
+    protected boolean peerClassLoadingEnabled() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setPeerClassLoadingEnabled(peerClassLoadingEnabled());
+
+        return cfg;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         startGrid(1);
@@ -144,4 +160,22 @@ public void testJobIdCollision() throws Exception {
         for (IgniteFuture<Object> fut : futs)
             fut.get();
     }
+
+    /**
+     * Test execution of non-existing task by name IGNITE-4838.
+     *
+     * @throws Exception If failed.
+     */
+    public void testExecuteTaskWithInvalidName() throws Exception {
+        try {
+            ComputeTaskFuture<?> fut = ignite.compute().execute("invalid.task.name", null);
+
+            fut.get();
+
+            assert false : "Should never be reached due to exception thrown.";
+        }
+        catch (IgniteDeploymentException e) {
+            info("Received correct exception: " + e);
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java
new file mode 100644
index 0000000000000..45e65cd9db017
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+
+/**
+ * Task execution test.
+ */
+@GridCommonTest(group = "Kernal Self")
+public class GridTaskExecutionWithoutPeerClassLoadingSelfTest extends GridTaskExecutionSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean peerClassLoadingEnabled() {
+        return false;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 8a501fdb801d6..ae64c95fb30b6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -51,6 +51,7 @@
 import org.apache.ignite.internal.GridStopWithWaitSelfTest;
 import org.apache.ignite.internal.GridTaskCancelSingleNodeSelfTest;
 import org.apache.ignite.internal.GridTaskExecutionSelfTest;
+import org.apache.ignite.internal.GridTaskExecutionWithoutPeerClassLoadingSelfTest;
 import org.apache.ignite.internal.GridTaskFailoverAffinityRunTest;
 import org.apache.ignite.internal.GridTaskFailoverSelfTest;
 import org.apache.ignite.internal.GridTaskFutureImplStopGridSelfTest;
@@ -100,6 +101,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCancelUnusedJobSelfTest.class);
         suite.addTestSuite(GridTaskJobRejectSelfTest.class);
         suite.addTestSuite(GridTaskExecutionSelfTest.class);
+        suite.addTestSuite(GridTaskExecutionWithoutPeerClassLoadingSelfTest.class);
         suite.addTestSuite(GridFailoverSelfTest.class);
         suite.addTestSuite(GridTaskListenerSelfTest.class);
         suite.addTestSuite(GridFailoverTopologySelfTest.class);

From 0d3d93cd4eeb589f1b6a11b48e429defad01c82f Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 18 May 2017 19:11:08 +0300
Subject: [PATCH 266/446] IGNITE-4842 Now containsKey() respects
 isReadFromBackup() flag.

(cherry picked from commit d84fd29)
---
 .../processors/cache/GridCacheAdapter.java    |   4 +-
 .../IgniteCacheContainsKeyAtomicTest.java     | 103 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   3 +
 3 files changed, 108 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheContainsKeyAtomicTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index a50661e137600..da4893229799f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -642,7 +642,7 @@ public void onKernalStop() {
 
         return (IgniteInternalFuture)getAsync(
             key,
-            /*force primary*/false,
+            /*force primary*/ !ctx.config().isReadFromBackup(),
             /*skip tx*/false,
             /*subj id*/null,
             /*task name*/null,
@@ -669,7 +669,7 @@ public void onKernalStop() {
 
         return getAllAsync(
             keys,
-            /*force primary*/false,
+            /*force primary*/ !ctx.config().isReadFromBackup(),
             /*skip tx*/false,
             /*subj id*/null,
             /*task name*/null,
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheContainsKeyAtomicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheContainsKeyAtomicTest.java
new file mode 100644
index 0000000000000..981d245245a20
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheContainsKeyAtomicTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * Verifies that containsKey() works as expected on atomic cache.
+ */
+public class IgniteCacheContainsKeyAtomicTest extends GridCacheAbstractSelfTest {
+    /** Cache name. */
+    public static final String CACHE_NAME = "replicated";
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 4;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        IgniteCache cache = ignite(0).cache(CACHE_NAME);
+
+        if (cache != null)
+            cache.clear();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testContainsPutIfAbsent() throws Exception {
+        checkPutIfAbsent(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testContainsPutIfAbsentAll() throws Exception {
+        checkPutIfAbsent(true);
+    }
+
+    /**
+     * @param all Check for set of keys.
+     * @throws Exception If failed.
+     */
+    private void checkPutIfAbsent(final boolean all) throws Exception {
+        Ignite srv = ignite(0);
+
+        final IgniteCache<Integer, Integer> cache1 = srv.getOrCreateCache(replicatedCache());
+        final IgniteCache<Integer, Integer> cache2 = ignite(1).getOrCreateCache(replicatedCache());
+
+        final AtomicInteger fails = new AtomicInteger(0);
+
+        GridTestUtils.runMultiThreaded(new Runnable() {
+            @Override public void run() {
+                for (int i = 0; i < 100; i++) {
+                    if (!cache1.putIfAbsent(i, i)) {
+                        if (all ? !cache2.containsKeys(Collections.singleton(i)) : !cache2.containsKey(i))
+                            fails.incrementAndGet();
+                    }
+                }
+            }
+        }, 100, "put-if-abs");
+
+        assertEquals(0, fails.get());
+    }
+
+    /**
+     * @return replicated cache configuration.
+     */
+    private CacheConfiguration<Integer, Integer> replicatedCache() {
+        return new CacheConfiguration<Integer, Integer>(CACHE_NAME)
+            .setAtomicityMode(ATOMIC)
+            .setWriteSynchronizationMode(FULL_SYNC)
+            .setReadFromBackup(false) // containsKey() must respect this flag
+            .setCacheMode(REPLICATED);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 914aedb40764a..a13a9a4d7245a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicStoreValueTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheConfigurationDefaultTemplateTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheConfigurationTemplateTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheContainsKeyAtomicTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheDynamicStopSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheGetCustomCollectionsSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheInvokeReadThroughSingleNodeTest;
@@ -339,6 +340,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class);
 
+        suite.addTestSuite(IgniteCacheContainsKeyAtomicTest.class);
+
         return suite;
     }
 }
\ No newline at end of file

From 2a818d36395dd1af23acf444adf396b2e2edbede Mon Sep 17 00:00:00 2001
From: Konstantin Dudkov <kdudkov@ya.ru>
Date: Mon, 22 May 2017 16:28:07 +0300
Subject: [PATCH 267/446] Fixed "IGNITE-4205 CassandraCacheStore should start
 IiteThread threads in loadCache() method"

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../store/cassandra/CassandraCacheStore.java  | 13 ++-
 .../tests/IgnitePersistentStoreTest.java      | 62 +++++++++++++
 .../loadall_blob/ignite-config.xml            | 90 +++++++++++++++++++
 .../loadall_blob/persistence-settings.xml     | 29 ++++++
 .../store/jdbc/CacheAbstractJdbcStore.java    |  6 +-
 5 files changed, 198 insertions(+), 2 deletions(-)
 create mode 100644 modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml
 create mode 100644 modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/persistence-settings.xml

diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
index 2e1d3ea70318d..dabf1b09a588e 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
@@ -31,6 +31,7 @@
 import javax.cache.Cache;
 import javax.cache.integration.CacheLoaderException;
 import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.store.CacheStore;
@@ -47,7 +48,9 @@
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.logger.NullLogger;
 import org.apache.ignite.resources.CacheStoreSessionResource;
+import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.thread.IgniteThreadFactory;
 
 /**
  * Implementation of {@link CacheStore} backed by Cassandra database.
@@ -59,6 +62,14 @@ public class CassandraCacheStore<K, V> implements CacheStore<K, V> {
     /** Connection attribute property name. */
     private static final String ATTR_CONN_PROP = "CASSANDRA_STORE_CONNECTION";
 
+    /** Thread name. */
+    private static final String CACHE_LOADER_THREAD_NAME = "cassandra-cache-loader";
+
+    /** Auto-injected ignite instance. */
+    @SuppressWarnings("unused")
+    @IgniteInstanceResource
+    private Ignite ignite;
+
     /** Auto-injected store session. */
     @CacheStoreSessionResource
     private CacheStoreSession storeSes;
@@ -99,7 +110,7 @@ public CassandraCacheStore(DataSource dataSrc, KeyValuePersistenceSettings setti
         Collection<Future<?>> futs = new ArrayList<>(args.length);
 
         try {
-            pool = Executors.newFixedThreadPool(maxPoolSize);
+            pool = Executors.newFixedThreadPool(maxPoolSize, new IgniteThreadFactory(ignite.name(), CACHE_LOADER_THREAD_NAME));
 
             CassandraSession ses = getCassandraSession();
 
diff --git a/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java b/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
index 51d08855388c6..75dff66de8f84 100644
--- a/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
+++ b/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
@@ -23,9 +23,11 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
+import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.tests.pojos.Person;
 import org.apache.ignite.tests.pojos.PersonId;
@@ -237,6 +239,35 @@ public void blobStrategyTest() {
         }
     }
 
+
+    /** */
+    @Test
+    public void blobBinaryLoadCacheTest() {
+        Ignition.stopAll(true);
+
+        try (Ignite ignite = Ignition.start("org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml")) {
+            IgniteCache<Long, PojoPerson> personCache = ignite.getOrCreateCache("cache2");
+
+            assert ignite.configuration().getMarshaller() instanceof BinaryMarshaller;
+
+            personCache.put(1L, new PojoPerson(1, "name"));
+
+            assert personCache.withKeepBinary().get(1L) instanceof BinaryObject;
+        }
+
+        Ignition.stopAll(true);
+
+        try (Ignite ignite = Ignition.start("org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml")) {
+            IgniteCache<Long, PojoPerson> personCache = ignite.getOrCreateCache("cache2");
+
+            personCache.loadCache(null, null);
+
+            PojoPerson person = personCache.get(1L);
+
+            LOGGER.info("loadCache tests passed");
+        }
+    }
+
     /** */
     @Test
     public void pojoStrategyTest() {
@@ -377,4 +408,35 @@ public void loadCacheTest() {
 
         LOGGER.info("loadCache test passed");
     }
+
+
+    /** */
+    public static class PojoPerson {
+        /** */
+        private int id;
+
+        /** */
+        private String name;
+
+        /** */
+        public PojoPerson() {
+            // No-op.
+        }
+
+        /** */
+        public PojoPerson(int id, String name) {
+            this.id = id;
+            this.name = name;
+        }
+
+        /** */
+        public int getId() {
+            return id;
+        }
+
+        /** */
+        public String getName() {
+            return name;
+        }
+    }
 }
diff --git a/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml b/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml
new file mode 100644
index 0000000000000..115e263e017d8
--- /dev/null
+++ b/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~      http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+    <!-- Cassandra connection settings -->
+    <import resource="classpath:org/apache/ignite/tests/cassandra/connection-settings.xml"/>
+
+    <!-- Persistence settings for 'cache2' -->
+    <bean id="cache2_persistence_settings"
+          class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings">
+        <constructor-arg type="org.springframework.core.io.Resource"
+                         value="classpath:org/apache/ignite/tests/persistence/loadall_blob/persistence-settings.xml"/>
+    </bean>
+
+    <!-- Ignite configuration -->
+    <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+
+        <property name="marshaller">
+            <bean class="org.apache.ignite.internal.binary.BinaryMarshaller"/>
+        </property>
+
+        <property name="binaryConfiguration">
+            <bean class="org.apache.ignite.configuration.BinaryConfiguration">
+                <property name="compactFooter" value="false"/>
+            </bean>
+        </property>
+
+        <property name="cacheConfiguration">
+            <list>
+                <!-- Configuring persistence for "cache2" cache -->
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="name" value="cache2"/>
+                    <property name="readThrough" value="true"/>
+                    <property name="writeThrough" value="true"/>
+                    <property name="storeKeepBinary" value="true"/>
+                    <property name="cacheStoreFactory">
+                        <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory">
+                            <property name="dataSourceBean" value="cassandraAdminDataSource"/>
+                            <property name="persistenceSettingsBean" value="cache2_persistence_settings"/>
+                        </bean>
+                    </property>
+                </bean>
+            </list>
+        </property>
+
+        <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. -->
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <!--
+                        Ignite provides several options for automatic discovery that can be used
+                        instead os static IP based discovery. For information on all options refer
+                        to our documentation: http://apacheignite.readme.io/docs/cluster-config
+                    -->
+                    <!-- Uncomment static IP finder to enable static-based discovery of initial nodes. -->
+                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+                        <!--<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">-->
+                        <property name="addresses">
+                            <list>
+                                <!-- In distributed environment, replace with actual host IP address. -->
+                                <value>127.0.0.1:47500..47509</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+    </bean>
+</beans>
diff --git a/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/persistence-settings.xml b/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/persistence-settings.xml
new file mode 100644
index 0000000000000..e872201fcc180
--- /dev/null
+++ b/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/persistence-settings.xml
@@ -0,0 +1,29 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<persistence keyspace="test1" table="blob_test3">
+    <!-- By default Java standard serialization is going to be used -->
+    <keyPersistence class="java.lang.Long"
+                    strategy="BLOB"
+                    column="key"/>
+
+    <!-- Kryo serialization specified to be used -->
+    <valuePersistence class="org.apache.ignite.tests.pojos.Person"
+                      strategy="BLOB"
+                      serializer="org.apache.ignite.cache.store.cassandra.serializer.JavaSerializer"
+                      column="value"/>
+</persistence>
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
index e211fadf2fb74..817b1a5da8531 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
@@ -72,6 +72,7 @@
 import org.apache.ignite.resources.CacheStoreSessionResource;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.thread.IgniteThreadFactory;
 import org.apache.ignite.transactions.Transaction;
 import org.jetbrains.annotations.Nullable;
 
@@ -121,6 +122,9 @@
  * </pre>
  */
 public abstract class CacheAbstractJdbcStore<K, V> implements CacheStore<K, V>, LifecycleAware {
+    /** Thread name. */
+    private static final String CACHE_LOADER_THREAD_NAME = "jdbc-cache-loader";
+
     /** Connection attribute property name. */
     protected static final String ATTR_CONN_PROP = "JDBC_STORE_CONNECTION";
 
@@ -730,7 +734,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
         String cacheName = session().cacheName();
 
         try {
-            pool = Executors.newFixedThreadPool(maxPoolSize);
+            pool = Executors.newFixedThreadPool(maxPoolSize, new IgniteThreadFactory(ignite.name(), CACHE_LOADER_THREAD_NAME));
 
             Collection<Future<?>> futs = new ArrayList<>();
 

From 04fadd4a499239176ba21c390d93e30809abb4c1 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 23 May 2017 15:42:20 +0300
Subject: [PATCH 268/446] IGNITE-5223 Allow use local binary metadata cache if
 it's possible

---
 .../apache/ignite/IgniteSystemProperties.java |   8 +
 .../ignite/internal/binary/BinaryContext.java |   4 +
 .../internal/binary/BinaryReaderExImpl.java   |  23 +-
 .../CacheObjectBinaryProcessorImpl.java       |  98 ++++--
 ...inaryMarshallerLocalMetadataCacheTest.java | 297 ++++++++++++++++++
 .../IgniteBinaryObjectsTestSuite.java         |   3 +
 6 files changed, 406 insertions(+), 27 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCacheTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index d77b2fb2838d4..713defeda552c 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -515,6 +515,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI = "IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI";
 
+    /**
+     * Use local metadata cache instead of distributed one. May be used only when binary objects schema
+     * are not modified and all classes available on each node. Classes that implements Binarylizable are
+     * not supported.
+     * @deprecated Should be removed in Apache Ignite 2.0.
+     */
+    public static final String IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE = "IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 7b21dfbed51f0..9f66b3ed50046 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -612,6 +612,7 @@ else if (cpElement.isFile()) {
 
     /**
      * @param cls Class.
+     * @param deserialize If {@code false}, metadata will be updated.
      * @return Class descriptor.
      * @throws BinaryObjectException In case of error.
      */
@@ -664,6 +665,7 @@ else if (!desc.registered()) {
      * @param userType User type or not.
      * @param typeId Type ID.
      * @param ldr Class loader.
+     * @param deserialize If {@code false}, metadata will be updated.
      * @return Class descriptor.
      */
     public BinaryClassDescriptor descriptorForTypeId(
@@ -719,6 +721,7 @@ public BinaryClassDescriptor descriptorForTypeId(
      * Creates and registers {@link BinaryClassDescriptor} for the given {@code class}.
      *
      * @param cls Class.
+     * @param deserialize If {@code false}, metadata will be updated.
      * @return Class descriptor.
      */
     private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean deserialize) {
@@ -759,6 +762,7 @@ private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean dese
      * Creates and registers {@link BinaryClassDescriptor} for the given user {@code class}.
      *
      * @param cls Class.
+     * @param deserialize If {@code false}, metadata will be updated.
      * @return Class descriptor.
      */
     private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean deserialize) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
index 775f237787cfc..ad2e736fa5f34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
@@ -26,6 +26,7 @@
 import java.util.Date;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryCollectionFactory;
 import org.apache.ignite.binary.BinaryInvalidTypeException;
 import org.apache.ignite.binary.BinaryMapFactory;
@@ -1722,15 +1723,23 @@ public BinarySchema getOrCreateSchema() {
             if (fieldIdLen != BinaryUtils.FIELD_ID_LEN) {
                 BinaryTypeImpl type = (BinaryTypeImpl)ctx.metadata(typeId);
 
-                if (type == null || type.metadata() == null)
-                    throw new BinaryObjectException("Cannot find metadata for object with compact footer: " +
-                        typeId);
+                if (type == null || type.metadata() == null || type.metadata().schemas().isEmpty()) {
+                    if (IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE, false)) {
+                        BinaryClassDescriptor desc = ctx.descriptorForTypeId(true, typeId, getClass().getClassLoader(), false);
 
-                for (BinarySchema typeSchema : type.metadata().schemas()) {
-                    if (schemaId == typeSchema.schemaId()) {
-                        schema = typeSchema;
+                        schema = desc.schema();
+                    }
+                    else
+                        throw new BinaryObjectException("Cannot find metadata for object with compact footer: " +
+                            typeId);
+                }
+                else {
+                    for (BinarySchema typeSchema : type.metadata().schemas()) {
+                        if (schemaId == typeSchema.schemaId()) {
+                            schema = typeSchema;
 
-                        break;
+                            break;
+                        }
                     }
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 1d60c422a6417..6b691c2236e52 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.IgniteBinary;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryBasicNameMapper;
 import org.apache.ignite.binary.BinaryField;
 import org.apache.ignite.binary.BinaryObject;
@@ -126,6 +127,9 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
     /** */
     private volatile IgniteCacheProxy<BinaryMetadataKey, BinaryMetadata> metaDataCache;
 
+    /** */
+    private final ConcurrentHashMap8<BinaryMetadataKey, BinaryMetadata> locMetadataCache;
+
     /** */
     private final ConcurrentHashMap8<Integer, BinaryTypeImpl> clientMetaDataCache;
 
@@ -175,6 +179,10 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
         clientNode = this.ctx.clientNode();
 
         clientMetaDataCache = clientNode ? new ConcurrentHashMap8<Integer, BinaryTypeImpl>() : null;
+
+        boolean useLocCache = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE, false);
+
+        locMetadataCache = useLocCache ? new ConcurrentHashMap8<BinaryMetadataKey, BinaryMetadata>() : null;
     }
 
     /** {@inheritDoc} */
@@ -190,7 +198,7 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
 
                     BinaryMetadata newMeta0 = ((BinaryTypeImpl)newMeta).metadata();
 
-                    if (metaDataCache == null) {
+                    if (metaDataCache == null && locMetadataCache == null) {
                         BinaryMetadata oldMeta = metaBuf.get(typeId);
                         BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0);
 
@@ -213,13 +221,13 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
                             return;
                     }
 
-                    assert metaDataCache != null;
+                    assert metaDataCache != null || locMetadataCache != null;
 
                     CacheObjectBinaryProcessorImpl.this.addMeta(typeId, newMeta0.wrap(binaryCtx));
                 }
 
                 @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
-                    if (metaDataCache == null)
+                    if (metaDataCache == null && locMetadataCache == null)
                         U.awaitQuiet(startLatch);
 
                     return CacheObjectBinaryProcessorImpl.this.metadata(typeId);
@@ -289,6 +297,12 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public void onUtilityCacheStarted() throws IgniteCheckedException {
+        if (locMetadataCache != null) {
+            startLatch.countDown();
+
+            return;
+        }
+
         IgniteCacheProxy<Object, Object> proxy = ctx.cache().jcache(CU.UTILITY_CACHE_NAME);
 
         boolean old = proxy.context().deploy().ignoreOwnership(true);
@@ -573,22 +587,31 @@ public GridBinaryMarshaller marshaller() {
 
         final BinaryMetadataKey key = new BinaryMetadataKey(typeId);
 
-        try {
-            BinaryMetadata oldMeta = metaDataCache.localPeek(key);
-            BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0);
+        if (locMetadataCache != null) {
+            locMetadataCache.merge(key, newMeta0, new ConcurrentHashMap8.BiFun<BinaryMetadata, BinaryMetadata, BinaryMetadata>() {
+                @Override public BinaryMetadata apply(BinaryMetadata curMeta, BinaryMetadata newMeta) {
+                    return BinaryUtils.mergeMetadata(curMeta, newMeta);
+                }
+            });
+        }
+        else {
+            try {
+                BinaryMetadata oldMeta = metaDataCache.localPeek(key);
+                BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0);
 
-            AffinityTopologyVersion topVer = ctx.cache().context().lockedTopologyVersion(null);
+                AffinityTopologyVersion topVer = ctx.cache().context().lockedTopologyVersion(null);
 
-            if (topVer == null)
-                topVer = ctx.cache().context().exchange().readyAffinityVersion();
+                if (topVer == null)
+                    topVer = ctx.cache().context().exchange().readyAffinityVersion();
 
-            BinaryObjectException err = metaDataCache.invoke(topVer, key, new MetadataProcessor(mergedMeta));
+                BinaryObjectException err = metaDataCache.invoke(topVer, key, new MetadataProcessor(mergedMeta));
 
-            if (err != null)
-                throw err;
-        }
-        catch (CacheException e) {
-            throw new BinaryObjectException("Failed to update meta data for type: " + newMeta.typeName(), e);
+                if (err != null)
+                    throw err;
+            }
+            catch (CacheException e) {
+                throw new BinaryObjectException("Failed to update meta data for type: " + newMeta.typeName(), e);
+            }
         }
     }
 
@@ -601,17 +624,28 @@ public GridBinaryMarshaller marshaller() {
                 if (typeMeta != null)
                     return typeMeta;
 
-                BinaryMetadata meta = metaDataCache.getTopologySafe(new BinaryMetadataKey(typeId));
+                BinaryMetadata meta;
+
+                if (locMetadataCache != null)
+                    meta = locMetadataCache.get(new BinaryMetadataKey(typeId));
+                else
+                    meta = metaDataCache.getTopologySafe(new BinaryMetadataKey(typeId));
 
                 return meta != null ? meta.wrap(binaryCtx) : null;
             }
             else {
                 BinaryMetadataKey key = new BinaryMetadataKey(typeId);
 
-                BinaryMetadata meta = metaDataCache.localPeek(key);
+                BinaryMetadata meta;
+
+                if (locMetadataCache != null)
+                    meta = locMetadataCache.get(key);
+                else {
+                    meta = metaDataCache.localPeek(key);
 
-                if (meta == null && !metaDataCache.context().preloader().syncFuture().isDone())
-                    meta = metaDataCache.getTopologySafe(key);
+                    if (meta == null && !metaDataCache.context().preloader().syncFuture().isDone())
+                        meta = metaDataCache.getTopologySafe(key);
+                }
 
                 return meta != null ? meta.wrap(binaryCtx) : null;
             }
@@ -630,7 +664,20 @@ public GridBinaryMarshaller marshaller() {
             for (Integer typeId : typeIds)
                 keys.add(new BinaryMetadataKey(typeId));
 
-            Map<BinaryMetadataKey, BinaryMetadata> meta = metaDataCache.getAll(keys);
+            Map<BinaryMetadataKey, BinaryMetadata> meta;
+
+            if (locMetadataCache != null) {
+                meta = new HashMap<>();
+
+                for (BinaryMetadataKey key : keys) {
+                    BinaryMetadata metadata = locMetadataCache.get(key);
+
+                    if (metadata != null)
+                        meta.put(key, metadata);
+                }
+            }
+            else
+                meta = metaDataCache.getAll(keys);
 
             Map<Integer, BinaryType> res = U.newHashMap(meta.size());
 
@@ -654,6 +701,17 @@ public GridBinaryMarshaller marshaller() {
                 }
             });
         else {
+            if (locMetadataCache != null) {
+                ConcurrentHashMap8.ValuesView<BinaryMetadataKey, BinaryMetadata> vals = locMetadataCache.values();
+
+                ArrayList<BinaryType> res = new ArrayList<>(vals.size());
+
+                for (BinaryMetadata metadata : vals)
+                    res.add(metadata.wrap(binaryCtx));
+
+                return res;
+            }
+
             return F.viewReadOnly(metaDataCache.entrySetx(metaPred),
                 new C1<Cache.Entry<BinaryMetadataKey, BinaryMetadata>, BinaryType>() {
                     private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCacheTest.java
new file mode 100644
index 0000000000000..ad3c5f3e0a83b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCacheTest.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.testframework.junits.IgniteTestResources;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE property.
+ */
+public class BinaryMarshallerLocalMetadataCacheTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName,
+        IgniteTestResources rsrcs) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName, rsrcs);
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        cfg.setClientMode(gridName.startsWith("client"));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE, "true");
+
+        startGrid(0);
+        startGrid(1);
+        startGrid("client");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        try {
+            stopAllGrids();
+        }
+        finally {
+            System.clearProperty(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testComputeLocalMetadata() throws Exception {
+        final BinaryObject obj = grid(0).binary().toBinary(new OptimizedContainer(new Optimized()));
+
+        ClusterGroup remotes = grid(0).cluster().forRemotes();
+
+        OptimizedContainer res = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
+            @Override public OptimizedContainer call() throws Exception {
+
+                return obj.deserialize();
+            }
+        });
+
+        OptimizedContainer res2 = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
+            @Override public OptimizedContainer call() throws Exception {
+
+                return obj.deserialize();
+            }
+        });
+
+        System.out.println(res);
+        System.out.println(res2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCacheLocalMetadata() throws Exception {
+        IgniteCache<Key, Object> cache = grid("client").createCache("cache");
+
+        Map<Key, Object> data = new HashMap<>();
+
+        for (int i = 0; i < 1000; i++)
+            data.put(new Key(i), new OptimizedContainer(new Optimized(String.valueOf(i))));
+
+        for (int i = 1000; i < 2000; i++)
+            data.put(new Key(i), new Simple(i, String.valueOf(i), new BigInteger(String.valueOf(i), 10)));
+
+        cache.putAll(data);
+
+        checkCache(cache, data);
+        checkCache(grid(0).<Key, Object>cache("cache"), data);
+        checkCache(grid(1).<Key, Object>cache("cache"), data);
+    }
+
+    /**
+     * @param cache Cache.
+     * @param data Data.
+     */
+    private void checkCache(IgniteCache<Key, Object> cache,
+        Map<Key, Object> data) {
+        for (Map.Entry<Key, Object> entry : cache.getAll(data.keySet()).entrySet())
+            assertEquals(data.get(entry.getKey()), entry.getValue());
+    }
+
+    /**
+     *
+     */
+    private static class Key {
+        /** */
+        private Integer i;
+
+        /**
+         * @param i I.
+         */
+        public Key(Integer i) {
+            this.i = i;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            Key key = (Key)o;
+
+            return i != null ? i.equals(key.i) : key.i == null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return i != null ? i.hashCode() : 0;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class OptimizedContainer {
+        /** */
+        private Optimized optim;
+
+        /**
+         * @param optim Val.
+         */
+        public OptimizedContainer(Optimized optim) {
+            this.optim = optim;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            OptimizedContainer container = (OptimizedContainer)o;
+
+            return optim != null ? optim.equals(container.optim) : container.optim == null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return optim != null ? optim.hashCode() : 0;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Optimized implements Externalizable {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private String fld;
+
+        /**
+         * @param fld Fld.
+         */
+        public Optimized(String fld) {
+            this.fld = fld;
+        }
+
+        /**
+         * Default constructor (required by Externalizable).
+         */
+        public Optimized() {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(ObjectOutput out) throws IOException {
+            U.writeUTFStringNullable(out, fld);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+            fld = U.readUTFStringNullable(in);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            Optimized optimized = (Optimized)o;
+
+            return fld != null ? fld.equals(optimized.fld) : optimized.fld == null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return fld != null ? fld.hashCode() : 0;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Simple {
+        /** I. */
+        private int i;
+
+        /** String. */
+        private String str;
+
+        /** Big integer. */
+        private BigInteger bigInteger;
+
+        /**
+         * @param i I.
+         * @param str String.
+         * @param bigInteger Big integer.
+         */
+        public Simple(int i, String str, BigInteger bigInteger) {
+            this.i = i;
+            this.str = str;
+            this.bigInteger = bigInteger;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            Simple simple = (Simple)o;
+
+            if (i != simple.i)
+                return false;
+
+            if (str != null ? !str.equals(simple.str) : simple.str != null)
+                return false;
+
+            return bigInteger != null ? bigInteger.equals(simple.bigInteger) : simple.bigInteger == null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            int res = i;
+
+            res = 31 * res + (str != null ? str.hashCode() : 0);
+            res = 31 * res + (bigInteger != null ? bigInteger.hashCode() : 0);
+
+            return res;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
index c1d9974837fb3..29e43e4593415 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.binary.BinaryFieldsOffheapSelfTest;
 import org.apache.ignite.internal.binary.BinaryFooterOffsetsHeapSelfTest;
 import org.apache.ignite.internal.binary.BinaryFooterOffsetsOffheapSelfTest;
+import org.apache.ignite.internal.binary.BinaryMarshallerLocalMetadataCacheTest;
 import org.apache.ignite.internal.binary.BinaryMarshallerSelfTest;
 import org.apache.ignite.internal.binary.BinaryObjectBuilderAdditionalSelfTest;
 import org.apache.ignite.internal.binary.BinaryObjectBuilderDefaultMappersSelfTest;
@@ -147,6 +148,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(GridCacheBinaryObjectUserClassloaderSelfTest.class);
 
+        suite.addTestSuite(BinaryMarshallerLocalMetadataCacheTest.class);
+
         return suite;
     }
 }

From b2040b7a95e421609bcf7ae05b56dc623310b409 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 23 May 2017 16:14:08 +0300
Subject: [PATCH 269/446] IGNITE-5259 Minor serialization fix

---
 modules/core/pom.xml                          |  3 +
 .../apache/ignite/IgniteSystemProperties.java | 12 ++-
 .../ignite/internal/IgniteNodeAttributes.java |  6 ++
 .../discovery/GridDiscoveryManager.java       | 39 ++++++++
 .../CacheContinuousQueryHandler.java          |  1 -
 .../top/GridTopologyCommandHandler.java       |  2 +
 .../processors/security/SecurityUtils.java    | 92 ++++++++++++++++++
 .../security/SecurityBasicPermissionSet.java  | 41 +++++++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 97 +++++++++++++++++--
 ...ridDiscoveryManagerAttributesSelfTest.java | 70 ++++++++++++-
 .../discovery/tcp/TestReconnectProcessor.java | 47 ++++++++-
 11 files changed, 390 insertions(+), 20 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java

diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 4c4343a9c41a1..4a14407ab131f 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -234,6 +234,9 @@
                     <exclude>**/*.java</exclude>
                 </excludes>
             </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+            </testResource>
         </testResources>
 
         <plugins>
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 713defeda552c..6827e0c248290 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -503,7 +503,7 @@ public final class IgniteSystemProperties {
     /**
      * Whether Ignite can access unaligned memory addresses.
      * <p>
-     * Defaults to {@code} false, meaning that unaligned access will be performed only on x86 architecture.
+     * Defaults to {@code false}, meaning that unaligned access will be performed only on x86 architecture.
      */
     public static final String IGNITE_UNALIGNED_MEMORY_ACCESS = "IGNITE_UNALIGNED_MEMORY_ACCESS";
 
@@ -523,6 +523,16 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE = "IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE";
 
+     /**
+     * When set to {@code true}, Ignite switches to compatibility mode with versions that don't
+     * support service security permissions. In this case security permissions will be ignored
+     * (if they set).
+     * <p>
+     *     Default is {@code false}, which means that service security permissions will be respected.
+     * </p>
+     */
+    public static final String IGNITE_SECURITY_COMPATIBILITY_MODE = "IGNITE_SECURITY_COMPATIBILITY_MODE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
index e7c984f22864e..436792459af4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
@@ -136,6 +136,9 @@ public final class IgniteNodeAttributes {
     /** Security subject for authenticated node. */
     public static final String ATTR_SECURITY_SUBJECT = ATTR_PREFIX + ".security.subject";
 
+    /** V2 security subject for authenticated node. */
+    public static final String ATTR_SECURITY_SUBJECT_V2 = ATTR_PREFIX + ".security.subject.v2";
+
     /** Client mode flag. */
     public static final String ATTR_CLIENT_MODE = ATTR_PREFIX + ".cache.client";
 
@@ -160,6 +163,9 @@ public final class IgniteNodeAttributes {
     /** Ignite services compatibility mode (can be {@code null}). */
     public static final String ATTR_SERVICES_COMPATIBILITY_MODE = ATTR_PREFIX + ".services.compatibility.enabled";
 
+    /** Ignite security compatibility mode. */
+    public static final String ATTR_SECURITY_COMPATIBILITY_MODE = ATTR_PREFIX + ".security.compatibility.enabled";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 2ec10705bedbb..b3ba83da68a8d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -121,6 +121,7 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCOVERY_HISTORY_SIZE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SECURITY_COMPATIBILITY_MODE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.IgniteSystemProperties.getInteger;
 import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
@@ -136,9 +137,12 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_COMPATIBILITY_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME;
 import static org.apache.ignite.internal.IgniteVersionUtils.VER;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.SERVICE_PERMISSIONS_SINCE;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.isSecurityCompatibilityMode;
 import static org.apache.ignite.plugin.segmentation.SegmentationPolicy.NOOP;
 
 /**
@@ -442,6 +446,9 @@ private void updateClientNodes(UUID leftNodeId) {
         spi.setMetricsProvider(createMetricsProvider());
 
         if (ctx.security().enabled()) {
+            if (isSecurityCompatibilityMode())
+                ctx.addNodeAttribute(ATTR_SECURITY_COMPATIBILITY_MODE, true);
+
             spi.setAuthenticator(new DiscoverySpiNodeAuthenticator() {
                 @Override public SecurityContext authenticateNode(ClusterNode node, SecurityCredentials cred) {
                     try {
@@ -1044,6 +1051,7 @@ private void checkAttributes(Iterable<ClusterNode> nodes) throws IgniteCheckedEx
         boolean locDelayAssign = locNode.attribute(ATTR_LATE_AFFINITY_ASSIGNMENT);
 
         Boolean locSrvcCompatibilityEnabled = locNode.attribute(ATTR_SERVICES_COMPATIBILITY_MODE);
+        Boolean locSecurityCompatibilityEnabled = locNode.attribute(ATTR_SECURITY_COMPATIBILITY_MODE);
 
         for (ClusterNode n : nodes) {
             int rmtJvmMajVer = nodeJavaMajorVersion(n);
@@ -1157,6 +1165,37 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                     ", rmtNodeAddrs=" + U.addressesAsString(n) +
                     ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
             }
+
+            if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) >= 0
+                && ctx.security().enabled() // Matters only if security enabled.
+               ) {
+                Boolean rmtSecurityCompatibilityEnabled = n.attribute(ATTR_SECURITY_COMPATIBILITY_MODE);
+
+                if (!F.eq(locSecurityCompatibilityEnabled, rmtSecurityCompatibilityEnabled)) {
+                    throw new IgniteCheckedException("Local node's " + IGNITE_SECURITY_COMPATIBILITY_MODE +
+                        " property value differs from remote node's value " +
+                        "(to make sure all nodes in topology have identical Ignite security compatibility mode enabled, " +
+                        "configure system property explicitly) " +
+                        "[locSecurityCompatibilityEnabled=" + locSecurityCompatibilityEnabled +
+                        ", rmtSecurityCompatibilityEnabled=" + rmtSecurityCompatibilityEnabled +
+                        ", locNodeAddrs=" + U.addressesAsString(locNode) +
+                        ", rmtNodeAddrs=" + U.addressesAsString(n) +
+                        ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
+                }
+            }
+
+            if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) < 0
+                && ctx.security().enabled() // Matters only if security enabled.
+                && (locSecurityCompatibilityEnabled == null || !locSecurityCompatibilityEnabled)) {
+                throw new IgniteCheckedException("Remote node does not support service security permissions. " +
+                    "To be able to join to it, local node must be started with " + IGNITE_SECURITY_COMPATIBILITY_MODE +
+                    " system property set to \"true\". " +
+                    "[locSecurityCompatibilityEnabled=" + locSecurityCompatibilityEnabled +
+                    ", locNodeAddrs=" + U.addressesAsString(locNode) +
+                    ", rmtNodeAddrs=" + U.addressesAsString(n) +
+                    ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ", " +
+                    ", rmtNodeVer" + n.version() + ']');
+            }
         }
 
         if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 5f00d580f4231..17f4308c0709b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -998,7 +998,6 @@ private static class PartitionRecovery {
 
             synchronized (pendingEvts) {
                 if (log.isDebugEnabled()) {
-
                     log.debug("Handling event [lastFiredEvt=" + lastFiredEvt +
                         ", curTop=" + curTop +
                         ", entUpdCnt=" + entry.updateCounter() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
index 297785ea760f2..3c68fbfdba92e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
@@ -58,6 +58,7 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_TCP_PORT;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_CONFIG;
 import static org.apache.ignite.internal.processors.rest.GridRestCommand.NODE;
 import static org.apache.ignite.internal.processors.rest.GridRestCommand.TOPOLOGY;
@@ -290,6 +291,7 @@ private GridClientNodeBean createNodeBean(ClusterNode node, boolean mtr, boolean
             attrs.remove(ATTR_CACHE);
             attrs.remove(ATTR_TX_CONFIG);
             attrs.remove(ATTR_SECURITY_SUBJECT);
+            attrs.remove(ATTR_SECURITY_SUBJECT_V2);
             attrs.remove(ATTR_SECURITY_CREDENTIALS);
 
             for (Iterator<Map.Entry<String, Object>> i = attrs.entrySet().iterator(); i.hasNext();) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
new file mode 100644
index 0000000000000..1016335888b3d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.security;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.plugin.security.SecurityPermission;
+
+/**
+ * Security utilities.
+ */
+public class SecurityUtils {
+    /** Version since service security supported. */
+    public static final IgniteProductVersion SERVICE_PERMISSIONS_SINCE = IgniteProductVersion.fromString("1.7.11");
+
+    /** Default serialization version. */
+    private final static int DFLT_SERIALIZE_VERSION = isSecurityCompatibilityMode() ? 1 : 2;
+
+    /** Current serialization version. */
+    private static final ThreadLocal<Integer> SERIALIZE_VERSION = new ThreadLocal<Integer>(){
+        @Override protected Integer initialValue() {
+            return DFLT_SERIALIZE_VERSION;
+        }
+    };
+
+    /**
+     * Private constructor.
+     */
+    private SecurityUtils() {
+    }
+
+    /**
+     * @return Security compatibility mode flag.
+     */
+    public static boolean isSecurityCompatibilityMode() {
+        return IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_SECURITY_COMPATIBILITY_MODE, false);
+    }
+
+    /**
+     * @param ver Serialize version.
+     */
+    public static void serializeVersion(int ver) {
+        SERIALIZE_VERSION.set(ver);
+    }
+
+    /**
+     * @return Serialize version.
+     */
+    public static int serializeVersion() {
+        return SERIALIZE_VERSION.get();
+    }
+
+    /**
+     * Sets default serialize version {@link #DFLT_SERIALIZE_VERSION}.
+     */
+    public static void restoreDefaultSerializeVersion() {
+        serializeVersion(DFLT_SERIALIZE_VERSION);
+    }
+
+    /**
+     * @return Allow all service permissions.
+     */
+    public static Map<String, Collection<SecurityPermission>> compatibleServicePermissions() {
+        Map<String, Collection<SecurityPermission>> srvcPerms = new HashMap<>();
+
+        srvcPerms.put("*", Arrays.asList(
+            SecurityPermission.SERVICE_CANCEL,
+            SecurityPermission.SERVICE_DEPLOY,
+            SecurityPermission.SERVICE_INVOKE));
+
+        return srvcPerms;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
index 44166d936c9aa..370eadd801f83 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
@@ -17,15 +17,24 @@
 
 package org.apache.ignite.plugin.security;
 
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.processors.security.SecurityUtils.compatibleServicePermissions;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.isSecurityCompatibilityMode;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.serializeVersion;
+
 /**
  * Simple implementation of {@link SecurityPermissionSet} interface. Provides
  * convenient way to specify permission set in the XML configuration.
@@ -44,7 +53,9 @@ public class SecurityBasicPermissionSet implements SecurityPermissionSet {
 
     /** Service permissions. */
     @GridToStringInclude
-    private Map<String, Collection<SecurityPermission>> servicePermissions = new HashMap<>();
+    private transient Map<String, Collection<SecurityPermission>> servicePermissions = isSecurityCompatibilityMode()
+            ? compatibleServicePermissions()
+            : new HashMap<String, Collection<SecurityPermission>>();
 
     /** System permissions. */
     @GridToStringInclude
@@ -158,6 +169,34 @@ public void setDefaultAllowAll(boolean dfltAllowAll) {
         return res;
     }
 
+    /**
+     * @param out Out.
+     */
+    private void writeObject(ObjectOutputStream out) throws IOException {
+        out.defaultWriteObject();
+
+        if (serializeVersion() >= 2)
+            U.writeMap(out, servicePermissions);
+    }
+
+    /**
+     * @param in In.
+     */
+    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+        in.defaultReadObject();
+
+        if (serializeVersion() >= 2)
+            servicePermissions = U.readMap(in);
+
+        if (servicePermissions == null) {
+            // Allow all for compatibility mode
+            if (serializeVersion() < 2)
+                servicePermissions = compatibleServicePermissions();
+            else
+                servicePermissions = Collections.emptyMap();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(SecurityBasicPermissionSet.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index afd1c2ba7fc66..58b362ffd0f15 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -72,6 +72,7 @@
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager;
 import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.internal.processors.security.SecurityUtils;
 import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.util.GridBoundedLinkedHashSet;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
@@ -947,7 +948,8 @@ private void localAuthentication(SecurityCredentials locCred){
 
             Map<String, Object> attrs = new HashMap<>(locNode.attributes());
 
-            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT, U.marshal(spi.marshaller(), subj));
+            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2, U.marshal(spi.marshaller(), subj));
+            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT, marshalWithSecurityVersion(subj, 1));
 
             locNode.setAttributes(attrs);
 
@@ -983,7 +985,16 @@ private boolean sendJoinRequestMessage() throws IgniteSpiException {
 
             for (InetSocketAddress addr : addrs) {
                 try {
-                    Integer res = sendMessageDirectly(joinReq, addr);
+                    Integer res;
+
+                    try {
+                        SecurityUtils.serializeVersion(1);
+
+                        res = sendMessageDirectly(joinReq, addr);
+                    }
+                    finally {
+                        SecurityUtils.restoreDefaultSerializeVersion();
+                    }
 
                     assert res != null;
 
@@ -1921,6 +1932,39 @@ private void processMessageFailedNodes(TcpDiscoveryAbstractMessage msg) {
         }
     }
 
+    /**
+     * @param obj Object.
+     * @param ver Security serialize version.
+     * @return Marshaled object.
+     */
+    private byte[] marshalWithSecurityVersion(Object obj, int ver) throws IgniteCheckedException {
+        try {
+            SecurityUtils.serializeVersion(ver);
+
+            return U.marshal(spi.marshaller(), obj);
+        }
+        finally {
+            SecurityUtils.restoreDefaultSerializeVersion();
+        }
+    }
+
+    /**
+     * @param bytes Marshaled object.
+     * @param ver Security serialize version.
+     * @return Unmarshaled object.
+     */
+    private <T> T unmarshalWithSecurityVersion(byte[] bytes, int ver) throws IgniteCheckedException {
+        try {
+            if (ver > 0)
+                SecurityUtils.serializeVersion(ver);
+
+            return spi.marshaller().unmarshal(bytes, U.resolveClassLoader(spi.ignite().configuration()));
+        }
+        finally {
+            SecurityUtils.restoreDefaultSerializeVersion();
+        }
+    }
+
     /**
      * Discovery messages history used for client reconnect.
      */
@@ -2995,6 +3039,8 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof
                                     pendingMsgs.customDiscardId);
 
                             try {
+                                SecurityUtils.serializeVersion(1);
+
                                 long tstamp = U.currentTimeMillis();
 
                                 if (timeoutHelper == null)
@@ -3033,6 +3079,8 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof
                                 }
                             }
                             finally {
+                                SecurityUtils.restoreDefaultSerializeVersion();
+
                                 clearNodeAddedMessage(msg);
                             }
 
@@ -3423,7 +3471,8 @@ else if (log.isDebugEnabled())
                             // Stick in authentication subject to node (use security-safe attributes for copy).
                             Map<String, Object> attrs = new HashMap<>(node.getAttributes());
 
-                            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT, U.marshal(spi.marshaller(), subj));
+                            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2, U.marshal(spi.marshaller(), subj));
+                            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT, marshalWithSecurityVersion(subj, 1));
 
                             node.setAttributes(attrs);
                         }
@@ -4073,9 +4122,22 @@ else if (!locNodeId.equals(node.id()) && ring.node(node.id()) != null) {
                         else {
                             SecurityContext subj = spi.nodeAuth.authenticateNode(node, cred);
 
-                            SecurityContext coordSubj = U.unmarshal(spi.marshaller(),
-                                node.<byte[]>attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT),
-                                U.resolveClassLoader(spi.ignite().configuration()));
+                            byte[] subjBytes = node.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT);
+                            byte[] subjBytesV2 = node.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2);
+
+                            SecurityContext coordSubj;
+
+                            try {
+                                if (subjBytesV2 == null)
+                                    SecurityUtils.serializeVersion(1);
+
+                                coordSubj = U.unmarshal(spi.marshaller(),
+                                    subjBytesV2 != null ? subjBytesV2 : subjBytes,
+                                    U.resolveClassLoader(spi.ignite().configuration()));
+                            }
+                            finally {
+                                SecurityUtils.restoreDefaultSerializeVersion();
+                            }
 
                             if (!permissionsEqual(coordSubj.subject().permissions(), subj.subject().permissions())) {
                                 // Node has not pass authentication.
@@ -4158,13 +4220,23 @@ else if (!locNodeId.equals(node.id()) && ring.node(node.id()) != null) {
                                     new TcpDiscoveryAuthFailedMessage(locNodeId, spi.locHost);
 
                                 try {
-                                    ClassLoader cl = U.resolveClassLoader(spi.ignite().configuration());
-
                                     byte[] rmSubj = node.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT);
                                     byte[] locSubj = locNode.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT);
 
-                                    SecurityContext rmCrd = spi.marshaller().unmarshal(rmSubj, cl);
-                                    SecurityContext locCrd = spi.marshaller().unmarshal(locSubj, cl);
+                                    byte[] rmSubjV2 = node.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2);
+                                    byte[] locSubjV2 = locNode.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2);
+
+                                    int ver = 1; // Compatible version.
+
+                                    if (rmSubjV2 != null && locSubjV2 != null) {
+                                        rmSubj = rmSubjV2;
+                                        locSubj = locSubjV2;
+
+                                        ver = 0; // Default version.
+                                    }
+
+                                    SecurityContext rmCrd = unmarshalWithSecurityVersion(rmSubj, ver);
+                                    SecurityContext locCrd = unmarshalWithSecurityVersion(locSubj, ver);
 
                                     if (!permissionsEqual(locCrd.subject().permissions(),
                                         rmCrd.subject().permissions())) {
@@ -5812,6 +5884,8 @@ else if (e.hasCause(ObjectStreamException.class) ||
 
                 while (!isInterrupted()) {
                     try {
+                        SecurityUtils.serializeVersion(1);
+
                         TcpDiscoveryAbstractMessage msg = U.unmarshal(spi.marshaller(), in,
                             U.resolveClassLoader(spi.ignite().configuration()));
 
@@ -6062,6 +6136,9 @@ else if (msg instanceof TcpDiscoveryLoopbackProblemMessage) {
 
                         return;
                     }
+                    finally {
+                        SecurityUtils.restoreDefaultSerializeVersion();
+                    }
                 }
             }
             finally {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
index ba8fa5b6e15a3..f0096db7bc8a6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
@@ -21,14 +21,18 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DeploymentMode;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TestReconnectPluginProvider;
+import org.apache.ignite.spi.discovery.tcp.TestReconnectProcessor;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SECURITY_COMPATIBILITY_MODE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS;
@@ -258,18 +262,69 @@ public void testServiceCompatibilityEnabled() throws Exception {
      * @throws Exception If failed.
      */
     private void doTestServiceCompatibilityEnabled(Object first, Object second, boolean fail) throws Exception {
+        doTestCompatibilityEnabled(IGNITE_SERVICES_COMPATIBILITY_MODE, first, second, fail);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSecurityCompatibilityEnabled() throws Exception {
+        TestReconnectPluginProvider.enabled = true;
+        TestReconnectProcessor.enabled = true;
+
+        try {
+            doTestSecurityCompatibilityEnabled(true, null, true);
+            doTestSecurityCompatibilityEnabled(true, false, true);
+            doTestSecurityCompatibilityEnabled(false, true, true);
+            doTestSecurityCompatibilityEnabled(null, true, true);
+
+            doTestSecurityCompatibilityEnabled(null, null, false);
+            doTestSecurityCompatibilityEnabled(null, false, false);
+            doTestSecurityCompatibilityEnabled(false, false, false);
+            doTestSecurityCompatibilityEnabled(false, null, false);
+            doTestSecurityCompatibilityEnabled(true, true, false);
+        }
+        finally {
+            TestReconnectPluginProvider.enabled = false;
+            TestReconnectProcessor.enabled = false;
+        }
+    }
+
+    /**
+     * @param first Service compatibility enabled flag for first node.
+     * @param second Service compatibility enabled flag for second node.
+     * @param fail Fail flag.
+     * @throws Exception If failed.
+     */
+    private void doTestSecurityCompatibilityEnabled(Object first, Object second, boolean fail) throws Exception {
+        doTestCompatibilityEnabled(IGNITE_SECURITY_COMPATIBILITY_MODE, first, second, fail);
+    }
+
+    /**
+     * @param prop System property.
+     * @param first Service compatibility enabled flag for first node.
+     * @param second Service compatibility enabled flag for second node.
+     * @param fail Fail flag.
+     * @throws Exception If failed.
+     */
+    private void doTestCompatibilityEnabled(String prop, Object first, Object second, boolean fail) throws Exception {
+        String backup = System.getProperty(prop);
         try {
             if (first != null)
-                System.setProperty(IGNITE_SERVICES_COMPATIBILITY_MODE, String.valueOf(first));
+                System.setProperty(prop, String.valueOf(first));
             else
-                System.clearProperty(IGNITE_SERVICES_COMPATIBILITY_MODE);
+                System.clearProperty(prop);
 
-            startGrid(0);
+            IgniteEx ignite = startGrid(0);
+
+            // Ignore if disabled security plugin used.
+            if (IGNITE_SECURITY_COMPATIBILITY_MODE.equals(prop) && !ignite.context().security().enabled())
+                return;
 
             if (second != null)
-                System.setProperty(IGNITE_SERVICES_COMPATIBILITY_MODE, String.valueOf(second));
+                System.setProperty(prop, String.valueOf(second));
             else
-                System.clearProperty(IGNITE_SERVICES_COMPATIBILITY_MODE);
+                System.clearProperty(prop);
 
             try {
                 startGrid(1);
@@ -284,6 +339,11 @@ private void doTestServiceCompatibilityEnabled(Object first, Object second, bool
         }
         finally {
             stopAllGrids();
+
+            if (backup != null)
+                System.setProperty(prop, backup);
+            else
+                System.clearProperty(prop);
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
index f0ed35c4ccfab..2476bd3d787bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
@@ -17,11 +17,13 @@
 
 package org.apache.ignite.spi.discovery.tcp;
 
+import java.io.Serializable;
 import java.util.Collection;
 import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
 import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
 import org.apache.ignite.internal.processors.security.SecurityContext;
@@ -37,6 +39,9 @@
  * Updates node attributes on disconnect.
  */
 public class TestReconnectProcessor extends GridProcessorAdapter implements GridSecurityProcessor {
+    /** Enabled flag. */
+    public static boolean enabled;
+
     /**
      * @param ctx Kernal context.
      */
@@ -44,10 +49,15 @@ protected TestReconnectProcessor(GridKernalContext ctx) {
         super(ctx);
     }
 
+    /** {@inheritDoc} */
+    @Override public void start() throws IgniteCheckedException {
+        ctx.addNodeAttribute(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS, new SecurityCredentials());
+    }
+
     /** {@inheritDoc} */
     @Override public SecurityContext authenticateNode(ClusterNode node,
         SecurityCredentials cred) throws IgniteCheckedException {
-        return null;
+        return new TestSecurityContext();
     }
 
     /** {@inheritDoc} */
@@ -83,11 +93,44 @@ protected TestReconnectProcessor(GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public boolean enabled() {
-        return false;
+        return enabled;
     }
 
     /** {@inheritDoc} */
     @Override public void onDisconnected(IgniteFuture<?> reconnectFut) throws IgniteCheckedException {
         ctx.addNodeAttribute("test", "2");
     }
+
+    /**
+     *
+     */
+    private static class TestSecurityContext implements SecurityContext, Serializable {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public SecuritySubject subject() {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean taskOperationAllowed(String taskClsName, SecurityPermission perm) {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean cacheOperationAllowed(String cacheName, SecurityPermission perm) {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean serviceOperationAllowed(String srvcName, SecurityPermission perm) {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean systemOperationAllowed(SecurityPermission perm) {
+            return true;
+        }
+    }
 }

From b77428d12658b3ab2cdd43ca61ed71d329e83283 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 10 Jan 2017 16:59:17 +0300
Subject: [PATCH 270/446] Do not evict removed entries, otherwise removes can
 be lost.

(cherry picked from commit 55ac6e7)
---
 .../internal/processors/cache/GridCacheMapEntry.java      | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 58b4ae3130c9d..c8b8cd115fefc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -4300,6 +4300,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                     if (obsoleteVersionExtras() != null)
                         return true;
 
+                    // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                    if (cctx.deferredDelete() && deletedUnlocked())
+                        return false;
+
                     CacheObject prev = saveOldValueUnlocked(false);
 
                     if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {
@@ -4358,6 +4362,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                             // Version has changed since entry passed the filter. Do it again.
                             continue;
 
+                        // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                        if (cctx.deferredDelete() && deletedUnlocked())
+                            return false;
+
                         CacheObject prevVal = saveValueForIndexUnlocked();
 
                         if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {

From 29187ef6b663eafe67eaaaf38e4c09fc244ac7aa Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 24 May 2017 17:31:27 +0300
Subject: [PATCH 271/446] Do not evict removed entries, otherwise removes can
 be lost.

Rollback due to test failings.
---
 .../internal/processors/cache/GridCacheMapEntry.java      | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index c8b8cd115fefc..58b4ae3130c9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -4300,10 +4300,6 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                     if (obsoleteVersionExtras() != null)
                         return true;
 
-                    // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
-                    if (cctx.deferredDelete() && deletedUnlocked())
-                        return false;
-
                     CacheObject prev = saveOldValueUnlocked(false);
 
                     if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {
@@ -4362,10 +4358,6 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                             // Version has changed since entry passed the filter. Do it again.
                             continue;
 
-                        // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
-                        if (cctx.deferredDelete() && deletedUnlocked())
-                            return false;
-
                         CacheObject prevVal = saveValueForIndexUnlocked();
 
                         if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {

From 442aac2507210d39b7f30ab8f8d9a3dbe2610cae Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 24 May 2017 18:32:11 +0300
Subject: [PATCH 272/446] IGNITE-5225: Fix NPE caused by changes in
 IGNITE-4577.

(cherry picked from commit d463840)
---
 .../ignite/internal/util/IgniteUtils.java       |  4 ++--
 .../communication/tcp/TcpCommunicationSpi.java  | 17 +++++++++++------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index ba118cb118cea..ca29adf81e1d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1813,11 +1813,11 @@ public static synchronized boolean isLocalHostChanged() throws IOException {
      * @return List of reachable addresses.
      */
     public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
-        final int reachTimeout = 2000;
-
         if (addrs.isEmpty())
             return Collections.emptyList();
 
+        final int reachTimeout = 2000;
+
         if (addrs.size() == 1) {
             InetAddress addr = F.first(addrs);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 81454f827e58f..6dba6b2b5fd0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2336,22 +2336,27 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
 
         Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
 
-        for (InetSocketAddress addr : addrs)
-            allInetAddrs.add(addr.getAddress());
+        for (InetSocketAddress addr : addrs) {
+            // Skip unresolved as addr.getAddress() can return null.
+            if(!addr.isUnresolved())
+                allInetAddrs.add(addr.getAddress());
+        }
 
         List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
 
         if (reachableInetAddrs.size() < allInetAddrs.size()) {
             LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
 
+            List<InetSocketAddress> unreachableInetAddr = new ArrayList<>(allInetAddrs.size() - reachableInetAddrs.size());
+
             for (InetSocketAddress addr : addrs) {
                 if (reachableInetAddrs.contains(addr.getAddress()))
                     addrs0.add(addr);
+                else
+                    unreachableInetAddr.add(addr);
             }
-            for (InetSocketAddress addr : addrs) {
-                if (!reachableInetAddrs.contains(addr.getAddress()))
-                    addrs0.add(addr);
-            }
+
+            addrs0.addAll(unreachableInetAddr);
 
             addrs = addrs0;
         }

From b1736c0bd87d6cfb65f9ef422241e0f1aba04c8d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 24 May 2017 18:48:52 +0300
Subject: [PATCH 273/446] Fixed thread pools incorrect shutdown.

(cherry picked from commit 66cef22)
---
 .../processors/cache/GridCacheAdapter.java    | 12 ++---
 .../ignite/internal/util/IgniteUtils.java     | 45 ++++++++++---------
 2 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 11bf34b221bb1..189f602328894 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1132,14 +1132,14 @@ public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, bool
         if (!F.isEmpty(jobs)) {
             ExecutorService execSvc = null;
 
-            if (jobs.size() > 1) {
-                execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
+            try {
+                if (jobs.size() > 1) {
+                    execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
 
-                for (int i = 1; i < jobs.size(); i++)
-                    execSvc.submit(jobs.get(i));
-            }
+                    for (int i = 1; i < jobs.size(); i++)
+                        execSvc.submit(jobs.get(i));
+                }
 
-            try {
                 jobs.get(0).run();
             }
             finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index ca29adf81e1d3..c2efb953c9135 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1833,33 +1833,36 @@ public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
 
         ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, addrs.size()));
 
-        for (final InetAddress addr : addrs) {
-            futs.add(executor.submit(new Runnable() {
-                @Override public void run() {
-                    if (reachable(addr, reachTimeout)) {
-                        synchronized (res) {
-                            res.add(addr);
+        try {
+            for (final InetAddress addr : addrs) {
+                futs.add(executor.submit(new Runnable() {
+                    @Override public void run() {
+                        if (reachable(addr, reachTimeout)) {
+                            synchronized (res) {
+                                res.add(addr);
+                            }
                         }
                     }
-                }
-            }));
-        }
-
-        for (Future<?> fut : futs) {
-            try {
-                fut.get();
+                }));
             }
-            catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
 
-                throw new IgniteException("Thread has been interrupted.", e);
-            }
-            catch (ExecutionException e) {
-                throw new IgniteException(e);
+            for (Future<?> fut : futs) {
+                try {
+                    fut.get();
+                }
+                catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+
+                    throw new IgniteException("Thread has been interrupted.", e);
+                }
+                catch (ExecutionException e) {
+                    throw new IgniteException(e);
+                }
             }
         }
-
-        executor.shutdown();
+        finally {
+            executor.shutdown();
+        }
 
         return res;
     }

From 15d94b432fdfe458a826df6ad3c30a0408a93f49 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 25 May 2017 14:27:08 +0300
Subject: [PATCH 274/446] Backport of IGNITE-4336: Manual rebalance can't be
 requested twice. (cherry picked from commit 9a691c4)

---
 .../processors/cache/GridCacheAdapter.java    |  5 +-
 .../GridCachePartitionExchangeManager.java    | 11 ++-
 .../processors/cache/GridCachePreloader.java  |  8 +-
 .../cache/GridCachePreloaderAdapter.java      |  7 +-
 .../processors/cache/IgniteCacheProxy.java    |  4 +-
 .../preloader/GridDhtPartitionDemander.java   | 46 ++++++++++--
 .../GridDhtPartitionsExchangeFuture.java      | 14 +++-
 .../dht/preloader/GridDhtPreloader.java       |  8 +-
 .../cache/CacheRebalancingSelfTest.java       | 75 +++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |  3 +
 10 files changed, 156 insertions(+), 25 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index b49becaacbc74..f10406fce2a80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -86,6 +86,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter;
@@ -4535,9 +4536,7 @@ protected Object readResolve() throws ObjectStreamException {
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<?> rebalance() {
-        ctx.preloader().forcePreload();
-
-        return ctx.preloader().syncFuture();
+        return ctx.preloader().forceRebalance();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 679395727123a..c175c43ae2f2a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -723,9 +723,13 @@ public void forceDummyExchange(boolean reassign,
      *
      * @param exchFut Exchange future.
      */
-    public void forcePreloadExchange(GridDhtPartitionsExchangeFuture exchFut) {
+    public IgniteInternalFuture<Boolean> forceRebalance(GridDhtPartitionsExchangeFuture exchFut) {
+        GridFutureAdapter<Boolean> fut = new GridFutureAdapter<>();
+
         exchWorker.addFuture(
-            new GridDhtPartitionsExchangeFuture(cctx, exchFut.discoveryEvent(), exchFut.exchangeId()));
+            new GridDhtPartitionsExchangeFuture(cctx, exchFut.discoveryEvent(), exchFut.exchangeId(), fut));
+
+        return fut;
     }
 
     /**
@@ -1815,7 +1819,8 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                 Runnable cur = cacheCtx.preloader().addAssignments(assigns,
                                     forcePreload,
                                     cnt,
-                                    r);
+                                    r,
+                                    exchFut.forcedRebalanceFuture());
 
                                 if (cur != null) {
                                     rebList.add(U.maskName(cacheCtx.name()));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index 3c4456d5984d9..0c2869101aa79 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
@@ -90,7 +91,8 @@ public interface GridCachePreloader {
     public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
         boolean forcePreload,
         int cnt,
-        Runnable next);
+        Runnable next,
+        @Nullable GridFutureAdapter<Boolean> forcedRebFut);
 
     /**
      * @param p Preload predicate.
@@ -150,9 +152,9 @@ public IgniteInternalFuture<Object> request(GridNearAtomicAbstractUpdateRequest
         AffinityTopologyVersion topVer);
 
     /**
-     * Force preload process.
+     * Force Rebalance process.
      */
-    public void forcePreload();
+    public IgniteInternalFuture<Boolean> forceRebalance();
 
     /**
      * Unwinds undeploys.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index 656a960b56b9f..8ae67215dabc9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
@@ -88,8 +89,8 @@ public GridCachePreloaderAdapter(GridCacheContext<?, ?> cctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public void forcePreload() {
-        // No-op.
+    @Override public IgniteInternalFuture<Boolean> forceRebalance() {
+        return new GridFinishedFuture<>(true);
     }
 
     /** {@inheritDoc} */
@@ -166,7 +167,7 @@ public GridCachePreloaderAdapter(GridCacheContext<?, ?> cctx) {
 
     /** {@inheritDoc} */
     @Override public Runnable addAssignments(GridDhtPreloaderAssignments assignments, boolean forcePreload,
-        int cnt, Runnable next) {
+        int cnt, Runnable next, @Nullable GridFutureAdapter<Boolean> forcedRebFut) {
         return null;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
index 873c8221ae921..13816709557a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
@@ -2329,9 +2329,7 @@ private void onLeave(GridCacheGateway<K, V> gate) {
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<?> rebalance() {
-        ctx.preloader().forcePreload();
-
-        return new IgniteFutureImpl<>(ctx.preloader().syncFuture());
+        return new IgniteFutureImpl<>(ctx.preloader().forceRebalance());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index a6808c73577e1..9ece00cd01049 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
 import org.apache.ignite.internal.util.GridLeanSet;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -64,6 +65,7 @@
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.spi.IgniteSpiException;
@@ -216,9 +218,9 @@ void preloadPredicate(IgnitePredicate<GridCacheEntryInfo> preloadPred) {
     }
 
     /**
-     * Force preload.
+     * Force Rebalance.
      */
-    void forcePreload() {
+    IgniteInternalFuture<Boolean> forceRebalance() {
         GridTimeoutObject obj = lastTimeoutObj.getAndSet(null);
 
         if (obj != null)
@@ -230,14 +232,31 @@ void forcePreload() {
             if (log.isDebugEnabled())
                 log.debug("Forcing rebalance event for future: " + exchFut);
 
+            final GridFutureAdapter<Boolean> fut = new GridFutureAdapter<>();
+
             exchFut.listen(new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
                 @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> t) {
-                    cctx.shared().exchange().forcePreloadExchange(exchFut);
+                    IgniteInternalFuture<Boolean> fut0 = cctx.shared().exchange().forceRebalance(exchFut);
+
+                    fut0.listen(new IgniteInClosure<IgniteInternalFuture<Boolean>>() {
+                        @Override public void apply(IgniteInternalFuture<Boolean> future) {
+                            try {
+                                fut.onDone(future.get());
+                            }
+                            catch (Exception e) {
+                                fut.onDone(e);
+                            }
+                        }
+                    });
                 }
             });
+
+            return fut;
         }
         else if (log.isDebugEnabled())
             log.debug("Ignoring force rebalance request (no topology event happened yet).");
+
+        return new GridFinishedFuture<>(true);
     }
 
     /**
@@ -274,16 +293,20 @@ void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
      * @param assigns Assignments.
      * @param force {@code True} if dummy reassign.
      * @param cnt Counter.
+     * * @param forcedRebFut External future for forced rebalance.
      * @param next Runnable responsible for cache rebalancing start.
      * @return Rebalancing runnable.
      */
     Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
         boolean force,
         int cnt,
-        final Runnable next) {
+        final Runnable next,
+        @Nullable final GridFutureAdapter<Boolean> forcedRebFut) {
         if (log.isDebugEnabled())
             log.debug("Adding partition assignments: " + assigns);
 
+        assert force == (forcedRebFut != null);
+
         long delay = cctx.config().getRebalanceDelay();
 
         if (delay == 0 || force) {
@@ -301,6 +324,19 @@ Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
                 });
             }
 
+            if (forcedRebFut != null) {
+                fut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
+                    @Override public void apply(IgniteInternalFuture<Boolean> future) {
+                        try {
+                            forcedRebFut.onDone(future.get());
+                        }
+                        catch (Exception e) {
+                            forcedRebFut.onDone(e);
+                        }
+                    }
+                });
+            }
+
             rebalanceFut = fut;
 
             fut.sendRebalanceStartedEvent();
@@ -383,7 +419,7 @@ else if (delay > 0) {
                 @Override public void onTimeout() {
                     exchFut.listen(new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
                         @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> f) {
-                            cctx.shared().exchange().forcePreloadExchange(exchFut);
+                            cctx.shared().exchange().forceRebalance(exchFut);
                         }
                     });
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 9aa0755269b9f..2245d17e83523 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -197,6 +197,9 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** */
     private boolean centralizedAff;
 
+    /** Forced Rebalance future. */
+    private GridFutureAdapter<Boolean> forcedRebFut;
+
     /**
      * Dummy future created to trigger reassignments if partition
      * topology changed while preloading.
@@ -230,15 +233,17 @@ public GridDhtPartitionsExchangeFuture(
      * @param cctx Cache context.
      * @param discoEvt Discovery event.
      * @param exchId Exchange id.
+     * @param forcedRebFut Forced Rebalance future.
      */
     public GridDhtPartitionsExchangeFuture(GridCacheSharedContext cctx, DiscoveryEvent discoEvt,
-        GridDhtPartitionExchangeId exchId) {
+        GridDhtPartitionExchangeId exchId, GridFutureAdapter<Boolean> forcedRebFut) {
         dummy = false;
         forcePreload = true;
 
         this.exchId = exchId;
         this.discoEvt = discoEvt;
         this.cctx = cctx;
+        this.forcedRebFut = forcedRebFut;
 
         reassign = true;
 
@@ -409,6 +414,13 @@ public GridDhtPartitionExchangeId exchangeId() {
         return exchId;
     }
 
+    /**
+     * @return Forced Rebalance future.
+     */
+    @Nullable public GridFutureAdapter<Boolean> forcedRebalanceFuture() {
+        return forcedRebFut;
+    }
+
     /**
      * @return {@code true} if entered to busy state.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 692e7c0de45a6..4aff4d5a71f42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -413,8 +413,8 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
 
     /** {@inheritDoc} */
     @Override public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
-        boolean forcePreload, int cnt, Runnable next) {
-        return demander.addAssignments(assignments, forcePreload, cnt, next);
+        boolean forcePreload, int cnt, Runnable next,  @Nullable GridFutureAdapter<Boolean> forcedRebFut) {
+        return demander.addAssignments(assignments, forcePreload, cnt, next, forcedRebFut);
     }
 
     /**
@@ -728,8 +728,8 @@ private GridDhtFuture<Object> request0(Collection<KeyCacheObject> keys, Affinity
     }
 
     /** {@inheritDoc} */
-    @Override public void forcePreload() {
-        demander.forcePreload();
+    @Override public IgniteInternalFuture<Boolean> forceRebalance() {
+        return demander.forceRebalance();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
new file mode 100644
index 0000000000000..8d1f67af2c33f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for rebalancing.
+ */
+public class CacheRebalancingSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCacheConfiguration(new CacheConfiguration());
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRebalanceFuture() throws Exception {
+        IgniteEx ignite0 = startGrid(0);
+        startGrid(1);
+
+        IgniteCache<Object, Object> cache = ignite0.cache(null);
+
+        IgniteFuture fut1 = cache.rebalance();
+
+        fut1.get();
+
+        startGrid(2);
+
+        IgniteFuture fut2 = cache.rebalance();
+
+        assert internalFuture(fut2) != internalFuture(fut1);
+
+        fut2.get();
+    }
+
+    /**
+     * @param future Future.
+     * @return Internal future.
+     */
+    private static IgniteInternalFuture internalFuture(IgniteFuture future) {
+        assert future instanceof IgniteFutureImpl;
+
+        return ((IgniteFutureImpl)future).internalFuture();
+    }
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 421676654e7d5..58a26e76ddd2f 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.CacheKeepBinaryTransactionTest;
 import org.apache.ignite.internal.processors.cache.CacheNearReaderUpdateTest;
+import org.apache.ignite.internal.processors.cache.CacheRebalancingSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheSerializableTransactionsTest;
 import org.apache.ignite.internal.processors.cache.GridCacheSwapSpaceSpiConsistencySelfTest;
 import org.apache.ignite.internal.processors.cache.EntryVersionConsistencyReadThroughTest;
@@ -64,6 +65,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(GridCacheOffHeapCleanupTest.class);
 
+        suite.addTestSuite(CacheRebalancingSelfTest.class);
+
         return suite;
     }
 }

From 26072dffb8f5b28693731f8367872a8e1e6dfe7e Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Thu, 18 May 2017 19:40:09 +0300
Subject: [PATCH 275/446] ignite-5203 Simple BLOB support added

---
 .../JdbcAbstractDmlStatementSelfTest.java     |  54 +-
 .../JdbcAbstractUpdateStatementSelfTest.java  |  11 +-
 .../ignite/internal/jdbc2/JdbcBlobTest.java   | 485 ++++++++++++++++++
 .../jdbc2/JdbcInsertStatementSelfTest.java    |  12 +-
 .../jdbc2/JdbcMergeStatementSelfTest.java     |  12 +-
 .../jdbc2/JdbcPreparedStatementSelfTest.java  |  46 ++
 .../jdbc/suite/IgniteJdbcDriverTestSuite.java |   6 +-
 .../ignite/internal/jdbc2/JdbcBlob.java       | 188 +++++++
 .../ignite/internal/jdbc2/JdbcConnection.java |   2 +-
 .../internal/jdbc2/JdbcPreparedStatement.java |   4 +-
 .../ignite/internal/jdbc2/JdbcResultSet.java  |   8 +-
 11 files changed, 799 insertions(+), 29 deletions(-)
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcBlobTest.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBlob.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
index 4a97aefcfcbe0..76597f1668bc8 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
@@ -18,9 +18,12 @@
 package org.apache.ignite.internal.jdbc2;
 
 import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.sql.Blob;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Collections;
 import org.apache.ignite.cache.CachePeekMode;
@@ -44,6 +47,9 @@
  * Statement test.
  */
 public abstract class JdbcAbstractDmlStatementSelfTest extends GridCommonAbstractTest {
+    /** UTF 16 character set name. */
+    private static final String UTF_16 = "UTF-16"; // RAWTOHEX function use UTF-16 for conversion strings to byte arrays.
+
     /** IP finder. */
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
@@ -54,7 +60,7 @@ public abstract class JdbcAbstractDmlStatementSelfTest extends GridCommonAbstrac
     static final String BASE_URL_BIN = CFG_URL_PREFIX + "modules/clients/src/test/config/jdbc-bin-config.xml";
 
     /** SQL SELECT query for verification. */
-    private static final String SQL_SELECT = "select _key, id, firstName, lastName, age from Person";
+    private static final String SQL_SELECT = "select _key, id, firstName, lastName, age, data from Person";
 
     /** Connection. */
     protected Connection conn;
@@ -117,6 +123,7 @@ IgniteConfiguration getBinaryConfiguration(String gridName) throws Exception {
         e.addQueryField("age", Integer.class.getName(), null);
         e.addQueryField("firstName", String.class.getName(), null);
         e.addQueryField("lastName", String.class.getName(), null);
+        e.addQueryField("data", byte[].class.getName(), null);
 
         ccfg.setQueryEntities(Collections.singletonList(e));
 
@@ -165,6 +172,7 @@ protected String getCfgUrl() {
                         assertEquals("John", rs.getString("firstName"));
                         assertEquals("White", rs.getString("lastName"));
                         assertEquals(25, rs.getInt("age"));
+                        assertEquals("White", str(getBytes(rs.getBlob("data"))));
                         break;
 
                     case 2:
@@ -172,6 +180,7 @@ protected String getCfgUrl() {
                         assertEquals("Joe", rs.getString("firstName"));
                         assertEquals("Black", rs.getString("lastName"));
                         assertEquals(35, rs.getInt("age"));
+                        assertEquals("Black", str(getBytes(rs.getBlob("data"))));
                         break;
 
                     case 3:
@@ -179,6 +188,7 @@ protected String getCfgUrl() {
                         assertEquals("Mike", rs.getString("firstName"));
                         assertEquals("Green", rs.getString("lastName"));
                         assertEquals(40, rs.getInt("age"));
+                        assertEquals("Green", str(getBytes(rs.getBlob("data"))));
                         break;
 
                     case 4:
@@ -186,6 +196,7 @@ protected String getCfgUrl() {
                         assertEquals("Leah", rs.getString("firstName"));
                         assertEquals("Grey", rs.getString("lastName"));
                         assertEquals(22, rs.getInt("age"));
+                        assertEquals("Grey", str(getBytes(rs.getBlob("data"))));
                         break;
 
                     default:
@@ -199,6 +210,42 @@ protected String getCfgUrl() {
         assertEquals(0, grid(0).cache(null).size(CachePeekMode.ALL));
     }
 
+    /**
+     * @param str String.
+     */
+    static byte[] getBytes(String str) {
+        try {
+            return str.getBytes(UTF_16);
+        }
+        catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * @param blob Blob.
+     */
+    static byte[] getBytes(Blob blob) {
+        try {
+            return blob.getBytes(1, (int)blob.length());
+        }
+        catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * @param arr Array.
+     */
+    static String str(byte[] arr) {
+        try {
+            return new String(arr, UTF_16);
+        }
+        catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      * Person.
      */
@@ -220,6 +267,10 @@ static class Person implements Serializable {
         @QuerySqlField
         private final int age;
 
+        /** Binary data. */
+        @QuerySqlField
+        private final byte[] data;
+
         /**
          * @param id ID.
          * @param firstName First name.
@@ -235,6 +286,7 @@ static class Person implements Serializable {
             this.firstName = firstName;
             this.lastName = lastName;
             this.age = age;
+            this.data = getBytes(lastName);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractUpdateStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractUpdateStatementSelfTest.java
index a20b8157ef1e8..ace1be665614c 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractUpdateStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractUpdateStatementSelfTest.java
@@ -19,12 +19,15 @@
 
 import java.sql.Statement;
 
+/**
+ *
+ */
 public abstract class JdbcAbstractUpdateStatementSelfTest extends JdbcAbstractDmlStatementSelfTest {
     /** SQL query to populate cache. */
-    private static final String ITEMS_SQL = "insert into Person(_key, id, firstName, lastName, age) values " +
-        "('p1', 1, 'John', 'White', 25), " +
-        "('p2', 2, 'Joe', 'Black', 35), " +
-        "('p3', 3, 'Mike', 'Green', 40)";
+    private static final String ITEMS_SQL = "insert into Person(_key, id, firstName, lastName, age, data) values " +
+        "('p1', 1, 'John', 'White', 25, RAWTOHEX('White')), " +
+        "('p2', 2, 'Joe', 'Black', 35, RAWTOHEX('Black')), " +
+        "('p3', 3, 'Mike', 'Green', 40, RAWTOHEX('Green'))";
 
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcBlobTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcBlobTest.java
new file mode 100644
index 0000000000000..9e0e0d2f6aab1
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcBlobTest.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.SQLException;
+import java.util.Arrays;
+import junit.framework.TestCase;
+
+/**
+ *
+ */
+public class JdbcBlobTest extends TestCase {
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLength() throws Exception {
+        JdbcBlob blob = new JdbcBlob(new byte[16]);
+
+        assertEquals(16, (int)blob.length());
+
+        blob.free();
+
+        try {
+            blob.length();
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testGetBytes() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        try {
+            blob.getBytes(0, 16);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.getBytes(17, 16);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.getBytes(1, -1);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        byte[] res = blob.getBytes(1, 0);
+        assertEquals(0, res.length);
+
+        assertTrue(Arrays.equals(arr, blob.getBytes(1, 16)));
+
+        res = blob.getBytes(1, 20);
+        assertEquals(16, res.length);
+        assertTrue(Arrays.equals(arr, res));
+
+        res = blob.getBytes(1, 10);
+        assertEquals(10, res.length);
+        assertEquals(0, res[0]);
+        assertEquals(9, res[9]);
+
+        res = blob.getBytes(7, 10);
+        assertEquals(10, res.length);
+        assertEquals(6, res[0]);
+        assertEquals(15, res[9]);
+
+        res = blob.getBytes(7, 20);
+        assertEquals(10, res.length);
+        assertEquals(6, res[0]);
+        assertEquals(15, res[9]);
+
+        res = blob.getBytes(1, 0);
+        assertEquals(0, res.length);
+
+        blob.free();
+
+        try {
+            blob.getBytes(1, 16);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testGetBinaryStream() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        InputStream is = blob.getBinaryStream();
+
+        byte[] res = readBytes(is);
+
+        assertTrue(Arrays.equals(arr, res));
+
+        blob.free();
+
+        try {
+            blob.getBinaryStream();
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testGetBinaryStreamWithParams() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        try {
+            blob.getBinaryStream(0, arr.length);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.getBinaryStream(1, 0);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.getBinaryStream(17, arr.length);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.getBinaryStream(1, arr.length + 1);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        InputStream is = blob.getBinaryStream(1, arr.length);
+        byte[] res = readBytes(is);
+        assertTrue(Arrays.equals(arr, res));
+
+        is = blob.getBinaryStream(1, 10);
+        res = readBytes(is);
+        assertEquals(10, res.length);
+        assertEquals(0, res[0]);
+        assertEquals(9, res[9]);
+
+        is = blob.getBinaryStream(6, 10);
+        res = readBytes(is);
+        assertEquals(10, res.length);
+        assertEquals(5, res[0]);
+        assertEquals(14, res[9]);
+
+        blob.free();
+
+        try {
+            blob.getBinaryStream(1, arr.length);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPositionBytePattern() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        assertEquals(-1, blob.position(new byte[] {1, 2, 3}, 0));
+        assertEquals(-1, blob.position(new byte[] {1, 2, 3}, arr.length + 1));
+        assertEquals(-1, blob.position(new byte[0], 1));
+        assertEquals(-1, blob.position(new byte[17], 1));
+        assertEquals(-1, blob.position(new byte[] {3, 2, 1}, 1));
+        assertEquals(1, blob.position(new byte[] {0, 1, 2}, 1));
+        assertEquals(2, blob.position(new byte[] {1, 2, 3}, 1));
+        assertEquals(2, blob.position(new byte[] {1, 2, 3}, 2));
+        assertEquals(-1, blob.position(new byte[] {1, 2, 3}, 3));
+        assertEquals(14, blob.position(new byte[] {13, 14, 15}, 3));
+        assertEquals(-1, blob.position(new byte[] {0, 1, 3}, 1));
+        assertEquals(-1, blob.position(new byte[] {0, 2, 3}, 1));
+        assertEquals(-1, blob.position(new byte[] {1, 2, 4}, 1));
+
+        blob.free();
+
+        try {
+            blob.position(new byte[] {0, 1, 2}, 1);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPositionBlobPattern() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {1, 2, 3}), 0));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {1, 2, 3}), arr.length + 1));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[0]), 1));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[17]), 1));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {3, 2, 1}), 1));
+        assertEquals(1, blob.position(new JdbcBlob(new byte[] {0, 1, 2}), 1));
+        assertEquals(2, blob.position(new JdbcBlob(new byte[] {1, 2, 3}), 1));
+        assertEquals(2, blob.position(new JdbcBlob(new byte[] {1, 2, 3}), 2));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {1, 2, 3}), 3));
+        assertEquals(14, blob.position(new JdbcBlob(new byte[] {13, 14, 15}), 3));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {0, 1, 3}), 1));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {0, 2, 3}), 1));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {1, 2, 4}), 1));
+
+        blob.free();
+
+        try {
+            blob.position(new JdbcBlob(new byte[] {0, 1, 2}), 1);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSetBytes() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        try {
+            blob.setBytes(0, new byte[4]);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.setBytes(17, new byte[4]);
+
+            fail();
+        }
+        catch (ArrayIndexOutOfBoundsException e) {
+            // No-op.
+        }
+
+        assertEquals(4, blob.setBytes(1, new byte[] {3, 2, 1, 0}));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 4, 5, 6, 7}, blob.getBytes(1, arr.length)));
+
+        assertEquals(4, blob.setBytes(5, new byte[] {7, 6, 5, 4}));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 7, 6, 5, 4}, blob.getBytes(1, arr.length)));
+
+        assertEquals(4, blob.setBytes(7, new byte[] {8, 9, 10, 11}));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 7, 6, 8, 9, 10, 11}, blob.getBytes(1, (int)blob.length())));
+
+        blob = new JdbcBlob(new byte[] {15, 16});
+        assertEquals(8, blob.setBytes(1, new byte[] {0, 1, 2, 3, 4, 5, 6, 7}));
+        assertTrue(Arrays.equals(new byte[] {0, 1, 2, 3, 4, 5, 6, 7}, blob.getBytes(1, (int)blob.length())));
+
+        blob.free();
+
+        try {
+            blob.setBytes(1, new byte[] {0, 1, 2});
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSetBytesWithOffsetAndLength() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        try {
+            blob.setBytes(0, new byte[4], 0, 2);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.setBytes(17, new byte[4], 0, 2);
+
+            fail();
+        }
+        catch (ArrayIndexOutOfBoundsException e) {
+            // No-op.
+        }
+
+        try {
+            blob.setBytes(1, new byte[4], -1, 2);
+
+            fail();
+        }
+        catch (ArrayIndexOutOfBoundsException e) {
+            // No-op.
+        }
+
+        try {
+            blob.setBytes(1, new byte[4], 0, 5);
+
+            fail();
+        }
+        catch (ArrayIndexOutOfBoundsException e) {
+            // No-op.
+        }
+
+        assertEquals(4, blob.setBytes(1, new byte[] {3, 2, 1, 0}, 0, 4));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 4, 5, 6, 7}, blob.getBytes(1, arr.length)));
+
+        assertEquals(4, blob.setBytes(5, new byte[] {7, 6, 5, 4}, 0, 4));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 7, 6, 5, 4}, blob.getBytes(1, arr.length)));
+
+        assertEquals(4, blob.setBytes(7, new byte[] {8, 9, 10, 11}, 0, 4));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 7, 6, 8, 9, 10, 11}, blob.getBytes(1, (int)blob.length())));
+
+        assertEquals(2, blob.setBytes(1, new byte[] {3, 2, 1, 0}, 2, 2));
+        assertTrue(Arrays.equals(new byte[] {1, 0, 1, 0, 7, 6, 8, 9, 10, 11}, blob.getBytes(1, (int)blob.length())));
+
+        assertEquals(2, blob.setBytes(9, new byte[] {3, 2, 1, 0}, 1, 2));
+        assertTrue(Arrays.equals(new byte[] {1, 0, 1, 0, 7, 6, 8, 9, 2, 1}, blob.getBytes(1, (int)blob.length())));
+
+        assertEquals(3, blob.setBytes(9, new byte[] {3, 2, 1, 0}, 0, 3));
+        assertTrue(Arrays.equals(new byte[] {1, 0, 1, 0, 7, 6, 8, 9, 3, 2, 1}, blob.getBytes(1, (int)blob.length())));
+
+        blob = new JdbcBlob(new byte[] {15, 16});
+        assertEquals(8, blob.setBytes(1, new byte[] {0, 1, 2, 3, 4, 5, 6, 7}, 0, 8));
+        assertTrue(Arrays.equals(new byte[] {0, 1, 2, 3, 4, 5, 6, 7}, blob.getBytes(1, (int)blob.length())));
+
+        blob.free();
+
+        try {
+            blob.setBytes(1, new byte[] {0, 1, 2}, 0, 2);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTruncate() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        try {
+            blob.truncate(-1);
+
+            fail();
+        }
+        catch(SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.truncate(arr.length + 1);
+
+            fail();
+        }
+        catch(SQLException e) {
+            // No-op.
+        }
+
+        blob.truncate(4);
+        assertTrue(Arrays.equals(new byte[] {0, 1, 2, 3}, blob.getBytes(1, (int)blob.length())));
+
+        blob.truncate(0);
+        assertEquals(0, (int)blob.length());
+
+        blob.free();
+
+        try {
+            blob.truncate(0);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+            System.out.println();
+        }
+    }
+
+    /**
+     * @param is Input stream.
+     */
+    private static byte[] readBytes(InputStream is) throws IOException {
+        byte[] tmp = new byte[16];
+
+        int i = 0;
+        int read;
+        int cnt = 0;
+
+        while ((read = is.read()) != -1) {
+            tmp[i++] = (byte)read;
+            cnt++;
+        }
+
+        byte[] res = new byte[cnt];
+
+        System.arraycopy(tmp, 0, res, 0, cnt);
+
+        return res;
+    }
+}
\ No newline at end of file
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java
index 7fc92de20595c..9a44006dd6055 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java
@@ -31,14 +31,14 @@
  */
 public class JdbcInsertStatementSelfTest extends JdbcAbstractDmlStatementSelfTest {
     /** SQL query. */
-    private static final String SQL = "insert into Person(_key, id, firstName, lastName, age) values " +
-        "('p1', 1, 'John', 'White', 25), " +
-        "('p2', 2, 'Joe', 'Black', 35), " +
-        "('p3', 3, 'Mike', 'Green', 40)";
+    private static final String SQL = "insert into Person(_key, id, firstName, lastName, age, data) values " +
+        "('p1', 1, 'John', 'White', 25, RAWTOHEX('White')), " +
+        "('p2', 2, 'Joe', 'Black', 35, RAWTOHEX('Black')), " +
+        "('p3', 3, 'Mike', 'Green', 40, RAWTOHEX('Green'))";
 
     /** SQL query. */
-    private static final String SQL_PREPARED = "insert into Person(_key, id, firstName, lastName, age) values " +
-        "(?, ?, ?, ?, ?), (?, ?, ?, ?, ?)";
+    private static final String SQL_PREPARED = "insert into Person(_key, id, firstName, lastName, age, data) values " +
+        "(?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?)";
 
     /** Statement. */
     private Statement stmt;
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java
index ecf6032db745f..1c93239ddf518 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java
@@ -26,14 +26,14 @@
  */
 public class JdbcMergeStatementSelfTest extends JdbcAbstractDmlStatementSelfTest {
     /** SQL query. */
-    private static final String SQL = "merge into Person(_key, id, firstName, lastName, age) values " +
-        "('p1', 1, 'John', 'White', 25), " +
-        "('p2', 2, 'Joe', 'Black', 35), " +
-        "('p3', 3, 'Mike', 'Green', 40)";
+    private static final String SQL = "merge into Person(_key, id, firstName, lastName, age, data) values " +
+        "('p1', 1, 'John', 'White', 25, RAWTOHEX('White')), " +
+        "('p2', 2, 'Joe', 'Black', 35, RAWTOHEX('Black')), " +
+        "('p3', 3, 'Mike', 'Green', 40, RAWTOHEX('Green'))";
 
     /** SQL query. */
-    protected static final String SQL_PREPARED = "merge into Person(_key, id, firstName, lastName, age) values " +
-        "(?, ?, ?, ?, ?), (?, ?, ?, ?, ?)";
+    protected static final String SQL_PREPARED = "merge into Person(_key, id, firstName, lastName, age, data) values " +
+        "(?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?)";
 
     /** Statement. */
     protected Statement stmt;
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
index ea586b297a579..edc6031e7d3ec 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
@@ -98,6 +98,7 @@ public class JdbcPreparedStatementSelfTest extends GridCommonAbstractTest {
         o.bigVal = new BigDecimal(1);
         o.strVal = "str";
         o.arrVal = new byte[] {1};
+        o.blobVal = new byte[] {1};
         o.dateVal = new Date(1);
         o.timeVal = new Time(1);
         o.tsVal = new Timestamp(1);
@@ -507,6 +508,47 @@ public void testArray() throws Exception {
         assert cnt == 1;
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBlob() throws Exception {
+        stmt = conn.prepareStatement("select * from TestObject where blobVal is not distinct from ?");
+
+        Blob blob = conn.createBlob();
+
+        blob.setBytes(1, new byte[] {1});
+
+        stmt.setBlob(1, blob);
+
+        ResultSet rs = stmt.executeQuery();
+
+        int cnt = 0;
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assert rs.getInt("id") == 1;
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+
+        stmt.setNull(1, BINARY);
+
+        rs = stmt.executeQuery();
+
+        cnt = 0;
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assert rs.getInt("id") == 2;
+
+            cnt++;
+        }
+
+        assert cnt == 1;
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -704,6 +746,10 @@ private static class TestObject implements Serializable {
         @QuerySqlField
         private byte[] arrVal;
 
+        /** */
+        @QuerySqlField
+        private byte[] blobVal;
+
         /** */
         @QuerySqlField
         private Date dateVal;
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index a28d29e00c52b..2718013f9d014 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.jdbc.suite;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.jdbc2.JdbcBlobTest;
 import org.apache.ignite.internal.jdbc2.JdbcDistributedJoinsQueryTest;
 import org.apache.ignite.jdbc.JdbcComplexQuerySelfTest;
 import org.apache.ignite.jdbc.JdbcConnectionSelfTest;
@@ -42,7 +43,7 @@ public class IgniteJdbcDriverTestSuite extends TestSuite {
     public static TestSuite suite() throws Exception {
         TestSuite suite = new TestSuite("Ignite JDBC Driver Test Suite");
 
-        // Thin client based driver tests
+        // Thin client based driver tests.
         suite.addTest(new TestSuite(JdbcConnectionSelfTest.class));
         suite.addTest(new TestSuite(JdbcStatementSelfTest.class));
         suite.addTest(new TestSuite(JdbcPreparedStatementSelfTest.class));
@@ -61,7 +62,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcPreparedStatementSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcResultSetSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcComplexQuerySelfTest.class));
-        suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcDistributedJoinsQueryTest.class));
+        suite.addTest(new TestSuite(JdbcDistributedJoinsQueryTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcMetadataSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcEmptyCacheSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcLocalCachesSelfTest.class));
@@ -71,6 +72,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcInsertStatementSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcBinaryMarshallerInsertStatementSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcDeleteStatementSelfTest.class));
+        suite.addTest(new TestSuite(JdbcBlobTest.class));
 
         return suite;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBlob.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBlob.java
new file mode 100644
index 0000000000000..daa0d1fbd357d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBlob.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.sql.Blob;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.util.Arrays;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Simple BLOB implementation. Actually there is no such entity as BLOB in Ignite. So using arrays is preferable way
+ * to work with binary objects.
+ *
+ * This implementation can be useful for reading binary fields of objects through JDBC.
+ */
+public class JdbcBlob implements Blob {
+    /** Byte array. */
+    private byte[] arr;
+
+    /**
+     * @param arr Byte array.
+     */
+    public JdbcBlob(byte[] arr) {
+        this.arr = arr;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long length() throws SQLException {
+        ensureNotClosed();
+
+        return arr.length;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte[] getBytes(long pos, int len) throws SQLException {
+        ensureNotClosed();
+
+        if (pos < 1 || arr.length - pos < 0 || len < 0)
+            throw new SQLException("Invalid argument. Position can't be less than 1 or " +
+                "greater than size of underlying byte array. Requested length also can't be negative " + "" +
+                "[pos=" + pos + ", len=" + len +']');
+
+        int idx = (int)(pos - 1);
+
+        int size = len > arr.length - idx ? arr.length - idx : len;
+
+        byte[] res = new byte[size];
+
+        U.arrayCopy(arr, idx, res, 0, size);
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public InputStream getBinaryStream() throws SQLException {
+        ensureNotClosed();
+
+        return new ByteArrayInputStream(arr);
+    }
+
+    /** {@inheritDoc} */
+    @Override public InputStream getBinaryStream(long pos, long len) throws SQLException {
+        ensureNotClosed();
+
+        if (pos < 1 || len < 1 || pos > arr.length || len > arr.length - pos + 1)
+            throw new SQLException("Invalid argument. Position can't be less than 1 or " +
+                "greater than size of underlying byte array. Requested length can't be negative and can't be " +
+                "greater than available bytes from given position [pos=" + pos + ", len=" + len +']');
+
+
+        return new ByteArrayInputStream(arr, (int)(pos - 1), (int)len);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long position(byte[] ptrn, long start) throws SQLException {
+        ensureNotClosed();
+
+        if (start < 1 || start > arr.length || ptrn.length == 0 || ptrn.length > arr.length)
+            return -1;
+
+        for(int i = 0, pos = (int)(start - 1); pos < arr.length;) {
+            if (arr[pos] == ptrn[i]) {
+                pos++;
+
+                i++;
+
+                if (i == ptrn.length)
+                    return pos - ptrn.length + 1;
+            }
+            else {
+                pos = pos - i + 1;
+
+                i = 0;
+            }
+        }
+
+        return -1;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long position(Blob ptrn, long start) throws SQLException {
+        ensureNotClosed();
+
+        if (start < 1 || start > arr.length || ptrn.length() == 0 || ptrn.length() > arr.length)
+            return -1;
+
+        return position(ptrn.getBytes(1, (int)ptrn.length()), start);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int setBytes(long pos, byte[] bytes) throws SQLException {
+        return setBytes(pos, bytes, 0, bytes.length);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int setBytes(long pos, byte[] bytes, int off, int len) throws SQLException {
+        ensureNotClosed();
+
+        if (pos < 1)
+            throw new SQLException("Invalid argument. Position can't be less than 1 [pos=" + pos + ']');
+
+        int idx = (int)(pos - 1);
+
+        byte[] dst = arr;
+
+        if (idx + len > arr.length) {
+            dst = new byte[arr.length + (len - (arr.length - idx))];
+
+            U.arrayCopy(arr, 0, dst, 0, idx);
+
+            arr = dst;
+        }
+
+        U.arrayCopy(bytes, off, dst, idx, len);
+
+        return len;
+    }
+
+    /** {@inheritDoc} */
+    @Override public OutputStream setBinaryStream(long pos) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void truncate(long len) throws SQLException {
+        ensureNotClosed();
+
+        if (len < 0 || len > arr.length)
+            throw new SQLException("Invalid argument. Length can't be " +
+                "less than zero or greater than Blob length [len=" + len + ']');
+
+        arr = Arrays.copyOf(arr, (int)len);
+
+    }
+
+    /** {@inheritDoc} */
+    @Override public void free() throws SQLException {
+        if (arr != null)
+            arr = null;
+    }
+
+    /**
+     *
+     */
+    private void ensureNotClosed() throws SQLException {
+        if (arr == null)
+            throw new SQLException("Blob instance can't be used after free() has been called.");
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
index dc3fe7f454624..3810117a785ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
@@ -550,7 +550,7 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public Blob createBlob() throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("SQL-specific types are not supported.");
+        return new JdbcBlob(new byte[0]);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
index 57badd2df728e..371c009df0b8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
@@ -227,9 +227,7 @@ public class JdbcPreparedStatement extends JdbcStatement implements PreparedStat
 
     /** {@inheritDoc} */
     @Override public void setBlob(int paramIdx, Blob x) throws SQLException {
-        ensureNotClosed();
-
-        throw new SQLFeatureNotSupportedException("SQL-specific types are not supported.");
+        setBytes(paramIdx, x.getBytes(1, (int)x.length()));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
index b53521e4cfa4d..187930ec4cc66 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
@@ -1012,9 +1012,7 @@ void closeInternal() throws SQLException  {
 
     /** {@inheritDoc} */
     @Override public Blob getBlob(int colIdx) throws SQLException {
-        ensureNotClosed();
-
-        throw new SQLFeatureNotSupportedException("SQL-specific types are not supported.");
+        return new JdbcBlob(getBytes(colIdx));
     }
 
     /** {@inheritDoc} */
@@ -1045,9 +1043,7 @@ void closeInternal() throws SQLException  {
 
     /** {@inheritDoc} */
     @Override public Blob getBlob(String colLb) throws SQLException {
-        ensureNotClosed();
-
-        throw new SQLFeatureNotSupportedException("SQL-specific types are not supported.");
+        return new JdbcBlob(getBytes(colLb));
     }
 
     /** {@inheritDoc} */

From d77a134fffee431cd7fa0bae2349419bc97ec1cf Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 30 May 2017 19:00:47 +0300
Subject: [PATCH 276/446] IGNITE-5342 - Skip permission check for TASK_EXECUTE
 for service jobs

---
 .../processors/affinity/GridAffinityProcessor.java    |  2 +-
 .../internal/processors/cache/GridCacheAdapter.java   |  6 +++---
 .../datastructures/CacheDataStructuresManager.java    |  4 ++--
 .../processors/closure/GridClosureProcessor.java      | 11 ++++++++---
 .../processors/service/GridServiceProcessor.java      |  4 ++--
 .../internal/processors/service/GridServiceProxy.java |  4 ++--
 .../internal/processors/task/GridTaskProcessor.java   |  6 ++++--
 .../processors/task/GridTaskThreadContextKey.java     |  5 ++++-
 .../internal/IgniteComputeTopologyExceptionTest.java  |  2 +-
 9 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
index b6efafbab394c..1be59784db6ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
@@ -497,7 +497,7 @@ private <K> Map<ClusterNode, Collection<K>> keysToNodes(@Nullable final String c
     private AffinityInfo affinityInfoFromNode(@Nullable String cacheName, AffinityTopologyVersion topVer, ClusterNode n)
         throws IgniteCheckedException {
         GridTuple3<GridAffinityMessage, GridAffinityMessage, GridAffinityAssignment> t = ctx.closure()
-            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0).get();
+            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0, false).get();
 
         AffinityFunction f = (AffinityFunction)unmarshall(ctx, n.id(), t.get1());
         AffinityKeyMapper m = (AffinityKeyMapper)unmarshall(ctx, n.id(), t.get2());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 189f602328894..ac1d2682d5f2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -84,8 +84,8 @@
 import org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityImpl;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteExternalizableExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter;
@@ -3665,14 +3665,14 @@ private IgniteInternalFuture<?> runLoadKeysCallable(final Set<? extends K> keys,
                 new LoadKeysCallableV2<>(ctx.name(), keys, update, plc, keepBinary),
                 nodes,
                 true,
-                0);
+                0, false);
         }
         else {
             return ctx.closures().callAsyncNoFailover(BROADCAST,
                 new LoadKeysCallable<>(ctx.name(), keys, update, plc),
                 nodes,
                 true,
-                0);
+                0, false);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index c1983df4f4d86..366a4a920b9ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -492,7 +492,7 @@ public void removeSetData(IgniteUuid id) throws IgniteCheckedException {
                         new BlockSetCallable(cctx.name(), id),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
@@ -516,7 +516,7 @@ else if (!pingNodes(nodes)) {
                         new RemoveSetDataCallable(cctx.name(), id, topVer),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 20fb6a0d0f485..aea7fe0a1e89b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -80,6 +80,7 @@
 import static org.apache.ignite.compute.ComputeJobResultPolicy.FAILOVER;
 import static org.apache.ignite.compute.ComputeJobResultPolicy.REDUCE;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
 
@@ -514,11 +515,12 @@ public ComputeTaskInternalFuture<?> affinityRun(@NotNull Collection<String> cach
     }
 
     /**
+     * @param <R> Type.
      * @param mode Distribution mode.
      * @param job Closure to execute.
      * @param nodes Grid nodes.
      * @param sys If {@code true}, then system pool will be used.
-     * @param <R> Type.
+     * @param skipAuth Skip authorization check.
      * @return Grid future for collection of closure results.
      */
     public <R> IgniteInternalFuture<R> callAsyncNoFailover(
@@ -526,8 +528,8 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
         @Nullable Callable<R> job,
         @Nullable Collection<ClusterNode> nodes,
         boolean sys,
-        long timeout
-    ) {
+        long timeout,
+        boolean skipAuth) {
         assert mode != null;
         assert timeout >= 0 : timeout;
 
@@ -543,6 +545,9 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
             ctx.task().setThreadContext(TC_NO_FAILOVER, true);
             ctx.task().setThreadContext(TC_SUBGRID, nodes);
 
+            if (skipAuth)
+                ctx.task().setThreadContext(TC_SKIP_AUTH, true);
+
             if (timeout > 0)
                 ctx.task().setThreadContext(TC_TIMEOUT, timeout);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index d7b9abc7fc1bd..25a8edbfc7c17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -721,8 +721,8 @@ public Map<UUID, Integer> serviceTopology(String name, long timeout) throws Igni
                 call,
                 Collections.singletonList(node),
                 false,
-                timeout
-            ).get();
+                timeout,
+                true).get();
         }
         else
             return serviceTopology(cache, name);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index aa609340b3bfb..2286cff9717aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -182,8 +182,8 @@ else if (U.isToStringMethod(mtd))
                             new ServiceProxyCallable(mtd.getName(), name, mtd.getParameterTypes(), args),
                             Collections.singleton(node),
                             false,
-                            waitTimeout
-                        ).get();
+                            waitTimeout,
+                            true).get();
                     }
                 }
                 catch (GridServiceNotFoundException | ClusterTopologyCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 935686456e593..12213581b3286 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -82,6 +82,7 @@
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK;
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK_CANCEL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBJ_ID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TASK_NAME;
@@ -477,8 +478,6 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
         else
             taskClsName = taskCls != null ? taskCls.getName() : taskName;
 
-        ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
-
         // Get values from thread-local context.
         Map<GridTaskThreadContextKey, Object> map = thCtx.get();
 
@@ -488,6 +487,9 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
             // Reset thread-local context.
             thCtx.set(null);
 
+        if (map.get(TC_SKIP_AUTH) == null)
+            ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
+
         Long timeout = (Long)map.get(TC_TIMEOUT);
 
         long timeout0 = timeout == null || timeout == 0 ? Long.MAX_VALUE : timeout;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
index 3bb19241a4ebf..a45f851d902a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
@@ -34,5 +34,8 @@ public enum GridTaskThreadContextKey {
     TC_TIMEOUT,
 
     /** Security subject ID. */
-    TC_SUBJ_ID
+    TC_SUBJ_ID,
+
+    /** Skip authorization for the task. */
+    TC_SKIP_AUTH
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
index 3ed91e805d656..a82373b618507 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
@@ -89,7 +89,7 @@ public void testCorrectCheckedException() throws Exception {
             },
             nodes,
             false,
-            0);
+            0, false);
 
         try {
             fut.get();

From d0186c368b12ab6893d0985e3fb92600708d5b65 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 30 May 2017 19:00:47 +0300
Subject: [PATCH 277/446] IGNITE-5342 - Skip permission check for TASK_EXECUTE
 for service jobs

(cherry picked from commit d77a134)
---
 .../processors/affinity/GridAffinityProcessor.java    |  2 +-
 .../internal/processors/cache/GridCacheAdapter.java   |  5 ++---
 .../datastructures/CacheDataStructuresManager.java    |  4 ++--
 .../processors/closure/GridClosureProcessor.java      | 11 ++++++++---
 .../processors/service/GridServiceProcessor.java      |  4 ++--
 .../internal/processors/service/GridServiceProxy.java |  4 ++--
 .../internal/processors/task/GridTaskProcessor.java   |  6 ++++--
 .../processors/task/GridTaskThreadContextKey.java     |  5 ++++-
 .../internal/IgniteComputeTopologyExceptionTest.java  |  2 +-
 9 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
index b6efafbab394c..1be59784db6ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
@@ -497,7 +497,7 @@ private <K> Map<ClusterNode, Collection<K>> keysToNodes(@Nullable final String c
     private AffinityInfo affinityInfoFromNode(@Nullable String cacheName, AffinityTopologyVersion topVer, ClusterNode n)
         throws IgniteCheckedException {
         GridTuple3<GridAffinityMessage, GridAffinityMessage, GridAffinityAssignment> t = ctx.closure()
-            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0).get();
+            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0, false).get();
 
         AffinityFunction f = (AffinityFunction)unmarshall(ctx, n.id(), t.get1());
         AffinityKeyMapper m = (AffinityKeyMapper)unmarshall(ctx, n.id(), t.get2());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index f10406fce2a80..d84060db4ce58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -84,7 +84,6 @@
 import org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityImpl;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteExternalizableExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo;
@@ -3512,14 +3511,14 @@ private IgniteInternalFuture<?> runLoadKeysCallable(final Set<? extends K> keys,
                 new LoadKeysCallableV2<>(ctx.name(), keys, update, plc, keepBinary),
                 nodes,
                 true,
-                0);
+                0, false);
         }
         else {
             return ctx.closures().callAsyncNoFailover(BROADCAST,
                 new LoadKeysCallable<>(ctx.name(), keys, update, plc),
                 nodes,
                 true,
-                0);
+                0, false);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index c1983df4f4d86..366a4a920b9ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -492,7 +492,7 @@ public void removeSetData(IgniteUuid id) throws IgniteCheckedException {
                         new BlockSetCallable(cctx.name(), id),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
@@ -516,7 +516,7 @@ else if (!pingNodes(nodes)) {
                         new RemoveSetDataCallable(cctx.name(), id, topVer),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 20fb6a0d0f485..aea7fe0a1e89b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -80,6 +80,7 @@
 import static org.apache.ignite.compute.ComputeJobResultPolicy.FAILOVER;
 import static org.apache.ignite.compute.ComputeJobResultPolicy.REDUCE;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
 
@@ -514,11 +515,12 @@ public ComputeTaskInternalFuture<?> affinityRun(@NotNull Collection<String> cach
     }
 
     /**
+     * @param <R> Type.
      * @param mode Distribution mode.
      * @param job Closure to execute.
      * @param nodes Grid nodes.
      * @param sys If {@code true}, then system pool will be used.
-     * @param <R> Type.
+     * @param skipAuth Skip authorization check.
      * @return Grid future for collection of closure results.
      */
     public <R> IgniteInternalFuture<R> callAsyncNoFailover(
@@ -526,8 +528,8 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
         @Nullable Callable<R> job,
         @Nullable Collection<ClusterNode> nodes,
         boolean sys,
-        long timeout
-    ) {
+        long timeout,
+        boolean skipAuth) {
         assert mode != null;
         assert timeout >= 0 : timeout;
 
@@ -543,6 +545,9 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
             ctx.task().setThreadContext(TC_NO_FAILOVER, true);
             ctx.task().setThreadContext(TC_SUBGRID, nodes);
 
+            if (skipAuth)
+                ctx.task().setThreadContext(TC_SKIP_AUTH, true);
+
             if (timeout > 0)
                 ctx.task().setThreadContext(TC_TIMEOUT, timeout);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6bda8446b9b04..98439d4c4ec4b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -721,8 +721,8 @@ public Map<UUID, Integer> serviceTopology(String name, long timeout) throws Igni
                 call,
                 Collections.singletonList(node),
                 false,
-                timeout
-            ).get();
+                timeout,
+                true).get();
         }
         else
             return serviceTopology(cache, name);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index d2e96bac52c0c..d16a4c48dcf86 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -192,8 +192,8 @@ else if (U.isToStringMethod(mtd))
                             new ServiceProxyCallable(mtd.getName(), name, mtd.getParameterTypes(), args),
                             Collections.singleton(node),
                             false,
-                            waitTimeout
-                        ).get();
+                            waitTimeout,
+                            true).get();
                     }
                 }
                 catch (GridServiceNotFoundException | ClusterTopologyCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 935686456e593..12213581b3286 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -82,6 +82,7 @@
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK;
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK_CANCEL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBJ_ID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TASK_NAME;
@@ -477,8 +478,6 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
         else
             taskClsName = taskCls != null ? taskCls.getName() : taskName;
 
-        ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
-
         // Get values from thread-local context.
         Map<GridTaskThreadContextKey, Object> map = thCtx.get();
 
@@ -488,6 +487,9 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
             // Reset thread-local context.
             thCtx.set(null);
 
+        if (map.get(TC_SKIP_AUTH) == null)
+            ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
+
         Long timeout = (Long)map.get(TC_TIMEOUT);
 
         long timeout0 = timeout == null || timeout == 0 ? Long.MAX_VALUE : timeout;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
index 2ec63df4ed83c..f0e56c731fd2c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
@@ -37,5 +37,8 @@ public enum GridTaskThreadContextKey {
     TC_SUBJ_ID,
 
     /** IO manager policy. */
-    TC_IO_POLICY
+    TC_IO_POLICY,
+
+    /** Skip authorization for the task. */
+    TC_SKIP_AUTH
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
index 3ed91e805d656..a82373b618507 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
@@ -89,7 +89,7 @@ public void testCorrectCheckedException() throws Exception {
             },
             nodes,
             false,
-            0);
+            0, false);
 
         try {
             fut.get();

From 0a0c5be1eb4793bc838da3184591d91537bd9cad Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 31 May 2017 15:24:14 +0300
Subject: [PATCH 278/446] Merge compilation fix.

---
 .../apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
index 0b3f2e318e3d5..9c72da5a76077 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.binary.BinaryFooterOffsetsHeapSelfTest;
 import org.apache.ignite.internal.binary.BinaryFooterOffsetsOffheapSelfTest;
 import org.apache.ignite.internal.binary.BinaryIdentityResolverConfigurationSelfTest;
+import org.apache.ignite.internal.binary.BinaryMarshallerLocalMetadataCacheTest;
 import org.apache.ignite.internal.binary.BinaryMarshallerSelfTest;
 import org.apache.ignite.internal.binary.BinaryObjectBuilderAdditionalSelfTest;
 import org.apache.ignite.internal.binary.BinaryObjectBuilderDefaultMappersSelfTest;

From 705d9cfca519183fa9e79e0686c1db48c0f5afe2 Mon Sep 17 00:00:00 2001
From: rfqu <rfq@list.ru>
Date: Thu, 1 Jun 2017 19:31:11 +0300
Subject: [PATCH 279/446] ticket fixed: IGN-7062 (TcpDiscoverySpi ignores
 maxMissedClientHeartbeats property)

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 19 +++++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 58 ++++++++++++++++++-
 2 files changed, 72 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 02ba56a884b33..2412624e900fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -366,7 +366,7 @@ else if (state == DISCONNECTED) {
                 else {
                     final GridFutureAdapter<Boolean> finalFut = fut;
 
-                    timer.schedule(new TimerTask() {
+                    TimerTask task = new TimerTask() {
                         @Override public void run() {
                             if (pingFuts.remove(nodeId, finalFut)) {
                                 if (ClientImpl.this.state == DISCONNECTED)
@@ -376,7 +376,13 @@ else if (state == DISCONNECTED) {
                                     finalFut.onDone(false);
                             }
                         }
-                    }, spi.netTimeout);
+                    };
+
+                    try {
+                        timer.schedule(task, spi.netTimeout);
+                    } catch (IllegalStateException e) {
+                        return false;  // timer is cancelled because this client node is dying
+                    }
 
                     sockWriter.sendMessage(new TcpDiscoveryClientPingRequest(getLocalNodeId(), nodeId));
                 }
@@ -803,7 +809,12 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
         U.warn(log, "Simulating client node failure: " + getLocalNodeId());
 
         U.interrupt(sockWriter);
+        U.interrupt(sockReader);
         U.interrupt(msgWorker);
+        try {
+            timer.cancel();
+        } catch (Throwable e) {
+        }
 
         U.join(sockWriter, log);
         U.join(
@@ -870,6 +881,10 @@ private class HeartbeatSender extends TimerTask {
                 msg.client(true);
 
                 sockWriter.sendMessage(msg);
+
+                if (log.isDebugEnabled())
+                    log.debug("*** Send heartbeat message from node:    "+msg.creatorNodeId());
+
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index afd1c2ba7fc66..2769fae3cc5ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6088,9 +6088,11 @@ private void processClientHeartbeatMessage(TcpDiscoveryClientHeartbeatMessage ms
             assert msg.client();
 
             ClientMessageWorker wrk = clientMsgWorkers.get(msg.creatorNodeId());
+            if (wrk != null) {
+                msg.verify(getLocalNodeId());
 
-            if (wrk != null)
-                wrk.metrics(msg.metrics());
+                wrk.addMessage(msg);
+            }
             else if (log.isDebugEnabled())
                 log.debug("Received heartbeat message from unknown client node: " + msg);
         }
@@ -6223,6 +6225,9 @@ private class StatisticsPrinter extends IgniteSpiThread {
     /**
      */
     private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAbstractMessage, byte[]>> {
+        /** minimal period of time which can be used as heartbeat timeout */
+        public static final int minHeartbeaTimeout = 10; // ms
+
         /** Node ID. */
         private final UUID clientNodeId;
 
@@ -6238,15 +6243,25 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
         /** */
         private IgniteProductVersion clientVer;
 
+        /** period of time after which client node which does not send heartbeat messages, is considered dead.
+         * Measured in milliseconds
+         */
+        private long heartBeatTimeOut;
+
+        /** timestamp of the last received heartbeat message, in milliseconds */
+        private long lastHeartBeatTime;
+
         /**
          * @param sock Socket.
          * @param clientNodeId Node ID.
          */
         protected ClientMessageWorker(Socket sock, UUID clientNodeId) throws IOException {
-            super("tcp-disco-client-message-worker", 2000);
+            super("tcp-disco-client-message-worker", Math.max(spi.getHeartbeatFrequency(), minHeartbeaTimeout));
 
             this.sock = sock;
             this.clientNodeId = clientNodeId;
+            this.heartBeatTimeOut = spi.getHeartbeatFrequency() * spi.getMaxMissedClientHeartbeats();
+            this.lastHeartBeatTime=U.currentTimeMillis();
         }
 
         /**
@@ -6295,6 +6310,33 @@ void addMessage(TcpDiscoveryAbstractMessage msg, @Nullable byte[] msgBytes) {
                 log.debug("Message has been added to client queue: " + msg);
         }
 
+
+        /**
+         * Check the last time a heartbeat message received.
+         * In case of timeout, expel client node from the topology
+         */
+        @Override
+        protected void noMessageLoop() {
+            long period = U.currentTimeMillis() - lastHeartBeatTime;
+
+            if (period >= heartBeatTimeOut) {
+                if (log.isInfoEnabled())
+                    log.info("### No heartbeat message from node:" + clientNodeId + "; timeOut=" + heartBeatTimeOut + "; period=" + period);
+
+                TcpDiscoveryAbstractMessage msg = new TcpDiscoveryNodeLeftMessage(clientNodeId);
+
+                msg.senderNodeId(getLocalNodeId());
+
+                msgWorker.addMessage(msg);
+
+                clientMsgWorkers.remove(clientNodeId, this);
+
+                U.interrupt(this);
+
+                U.closeQuiet(sock);
+            }
+        }
+
         /** {@inheritDoc} */
         @Override protected void processMessage(T2<TcpDiscoveryAbstractMessage, byte[]> msgT) {
             boolean success = false;
@@ -6333,6 +6375,16 @@ else if (msgLog.isDebugEnabled())
                             spi.failureDetectionTimeout() : spi.getSocketTimeout());
                     }
                 }
+                else if (msg instanceof TcpDiscoveryClientHeartbeatMessage) {
+                    TcpDiscoveryClientHeartbeatMessage hbmsg = (TcpDiscoveryClientHeartbeatMessage)msg;
+
+                    if (log.isDebugEnabled()) // TODO turn to debug
+                        log.debug("###  Received heartbeat message from node:" + hbmsg.creatorNodeId());
+
+                    this.metrics = hbmsg.metrics();
+
+                    this.lastHeartBeatTime=U.currentTimeMillis();
+                }
                 else {
                     if (msgLog.isDebugEnabled())
                         msgLog.debug("Redirecting message to client [sock=" + sock + ", locNodeId="

From 5f9dc362f82bc6351d01a77418e42c01db9391cf Mon Sep 17 00:00:00 2001
From: rfqu <rfq@list.ru>
Date: Fri, 2 Jun 2017 12:11:40 +0300
Subject: [PATCH 280/446] code style fixed

---
 .../java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java | 1 -
 .../java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java | 5 +++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 2412624e900fe..b83a3809aa26d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -884,7 +884,6 @@ private class HeartbeatSender extends TimerTask {
 
                 if (log.isDebugEnabled())
                     log.debug("*** Send heartbeat message from node:    "+msg.creatorNodeId());
-
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 2769fae3cc5ef..178d22b577fe3 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6243,7 +6243,8 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
         /** */
         private IgniteProductVersion clientVer;
 
-        /** period of time after which client node which does not send heartbeat messages, is considered dead.
+        /**
+         * Period of time after which client node which does not send heartbeat messages, is considered dead.
          * Measured in milliseconds
          */
         private long heartBeatTimeOut;
@@ -6321,7 +6322,7 @@ protected void noMessageLoop() {
 
             if (period >= heartBeatTimeOut) {
                 if (log.isInfoEnabled())
-                    log.info("### No heartbeat message from node:" + clientNodeId + "; timeOut=" + heartBeatTimeOut + "; period=" + period);
+                    log.info("Heartbeat timeout for node:" + clientNodeId + "; timeOut=" + heartBeatTimeOut + "; period=" + period);
 
                 TcpDiscoveryAbstractMessage msg = new TcpDiscoveryNodeLeftMessage(clientNodeId);
 

From 05c639f041ffbbc53678addaf46fc806fb7c168c Mon Sep 17 00:00:00 2001
From: rfqu <rfq@list.ru>
Date: Fri, 2 Jun 2017 12:25:53 +0300
Subject: [PATCH 281/446] coding style fixed

---
 .../java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java     | 1 +
 .../java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java     | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index b83a3809aa26d..9225a922b5dd3 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -814,6 +814,7 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
         try {
             timer.cancel();
         } catch (Throwable e) {
+            // No-op.
         }
 
         U.join(sockWriter, log);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 178d22b577fe3..c7e6d9619ed54 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6383,7 +6383,6 @@ else if (msg instanceof TcpDiscoveryClientHeartbeatMessage) {
                         log.debug("###  Received heartbeat message from node:" + hbmsg.creatorNodeId());
 
                     this.metrics = hbmsg.metrics();
-
                     this.lastHeartBeatTime=U.currentTimeMillis();
                 }
                 else {

From 4d2c9ef1cd50be0a73cd8ac4a0edc809631b23a2 Mon Sep 17 00:00:00 2001
From: rfqu <rfq@list.ru>
Date: Fri, 2 Jun 2017 13:49:31 +0300
Subject: [PATCH 282/446] test added, taken from tiket IGNITE-5103

---
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   3 +-
 .../TcpDiscoveryClientSuspensionSelfTest.java | 116 ++++++++++++++++++
 2 files changed, 117 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index c7e6d9619ed54..318cc23f16101 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6316,8 +6316,7 @@ void addMessage(TcpDiscoveryAbstractMessage msg, @Nullable byte[] msgBytes) {
          * Check the last time a heartbeat message received.
          * In case of timeout, expel client node from the topology
          */
-        @Override
-        protected void noMessageLoop() {
+        @Override protected void noMessageLoop() {
             long period = U.currentTimeMillis() - lastHeartBeatTime;
 
             if (period >= heartBeatTimeOut) {
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
new file mode 100644
index 0000000000000..9dc6e506cc74a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.Timer;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for missed client heartbeats.
+ */
+public class TcpDiscoveryClientSuspensionSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+        disco.setHeartbeatFrequency(200);
+        disco.setMaxMissedClientHeartbeats(10);
+
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneServer() throws Exception {
+        doTestClientSuspension(1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTwoServers() throws Exception {
+        doTestClientSuspension(2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testThreeServers() throws Exception {
+        doTestClientSuspension(3);
+    }
+
+    /**
+     * @param serverCnt Servers count.
+     * @throws Exception If failed.
+     */
+    private void doTestClientSuspension(int serverCnt) throws Exception {
+        startGrids(serverCnt);
+
+        Ignition.setClientMode(true);
+
+        Ignite client = startGrid("client");
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(1, grid(i).cluster().forClients().nodes().size());
+
+        Thread.sleep(3000);
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(1, grid(i).cluster().forClients().nodes().size());
+
+        suspendClientHeartbeats(client);
+
+        Thread.sleep(3000);
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(0, grid(i).cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @param client Client.
+     */
+    private void suspendClientHeartbeats(Ignite client) {
+        assert client.cluster().localNode().isClient();
+
+        ClientImpl impl = U.field(client.configuration().getDiscoverySpi(), "impl");
+
+        impl.simulateNodeFailure();
+
+        //Timer timer = U.field(impl, "timer"); timer.cancel(); -- client node successfully reconnects
+    }
+}

From 744a81ba937ba83ecdefa7c71f198d92d21527bb Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Wed, 31 May 2017 15:27:33 +0300
Subject: [PATCH 283/446] IGNITE-5232 [BACKPORT]
 GridDhtPartitionDemander.requestPartitions invokes sendMessages consequently,
 which lead to significant increase of node start time on large clusters with
 ssl

---
 .../preloader/GridDhtPartitionDemander.java   | 131 ++++++++++--------
 1 file changed, 71 insertions(+), 60 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index a6808c73577e1..daae1e2aa144a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -331,41 +331,21 @@ Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
 
             return new Runnable() {
                 @Override public void run() {
-                    try {
-                        if (next != null)
-                            fut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
-                                @Override public void apply(IgniteInternalFuture<Boolean> f) {
-                                    try {
-                                        if (f.get()) // Not cancelled.
-                                            next.run(); // Starts next cache rebalancing (according to the order).
-                                    }
-                                    catch (IgniteCheckedException ignored) {
-                                        if (log.isDebugEnabled())
-                                            log.debug(ignored.getMessage());
-                                    }
+                    if (next != null)
+                        fut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
+                            @Override public void apply(IgniteInternalFuture<Boolean> f) {
+                                try {
+                                    if (f.get()) // Not cancelled.
+                                        next.run(); // Starts next cache rebalancing (according to the order).
                                 }
-                            });
-
-                        requestPartitions(fut, assigns);
-                    }
-                    catch (IgniteCheckedException e) {
-                        ClusterTopologyCheckedException cause = e.getCause(ClusterTopologyCheckedException.class);
-
-                        if (cause != null)
-                            log.warning("Failed to send initial demand request to node. " + e.getMessage());
-                        else
-                            log.error("Failed to send initial demand request to node.", e);
-
-                        fut.cancel();
-                    }
-                    catch (Throwable th) {
-                        log.error("Runtime error caught during initial demand request sending.", th);
-
-                        fut.cancel();
+                                catch (IgniteCheckedException ignored) {
+                                    if (log.isDebugEnabled())
+                                        log.debug(ignored.getMessage());
+                                }
+                            }
+                        });
 
-                        if (th instanceof Error)
-                            throw th;
-                    }
+                    requestPartitions(fut, assigns);
                 }
             };
         }
@@ -404,9 +384,8 @@ else if (delay > 0) {
      * @return Partitions were requested.
      */
     private void requestPartitions(
-        RebalanceFuture fut,
-        GridDhtPreloaderAssignments assigns
-    ) throws IgniteCheckedException {
+        final RebalanceFuture fut,
+        GridDhtPreloaderAssignments assigns){
         if (topologyChanged(fut)) {
             fut.cancel();
 
@@ -438,7 +417,7 @@ private void requestPartitions(
 
                 int lsnrCnt = cctx.gridConfig().getRebalanceThreadPoolSize();
 
-                List<Set<Integer>> sParts = new ArrayList<>(lsnrCnt);
+            final List<Set<Integer>> sParts = new ArrayList<>(lsnrCnt);
 
                 for (int cnt = 0; cnt < lsnrCnt; cnt++)
                     sParts.add(new HashSet<Integer>());
@@ -453,42 +432,74 @@ private void requestPartitions(
                 for (cnt = 0; cnt < lsnrCnt; cnt++) {
                     if (!sParts.get(cnt).isEmpty()) {
                         // Create copy.
-                        GridDhtPartitionDemandMessage initD = new GridDhtPartitionDemandMessage(d, sParts.get(cnt));
+                        final GridDhtPartitionDemandMessage initD = new GridDhtPartitionDemandMessage(d, sParts.get(cnt));
 
                         initD.topic(rebalanceTopics.get(cnt));
                         initD.updateSequence(fut.updateSeq);
                         initD.timeout(cctx.config().getRebalanceTimeout());
 
-                        synchronized (fut) {
-                            if (!fut.isDone()) {
-                                // Future can be already cancelled at this moment and all failovers happened.
-                                // New requests will not be covered by failovers.
-                                cctx.io().sendOrderedMessage(node,
-                                    rebalanceTopics.get(cnt), initD, cctx.ioPolicy(), initD.timeout());
-                            }
-                        }
+                        final int finalCnt = cnt;
 
-                        if (log.isDebugEnabled())
-                            log.debug("Requested rebalancing [from node=" + node.id() + ", listener index=" +
-                                cnt + ", partitions count=" + sParts.get(cnt).size() +
-                                " (" + partitionsList(sParts.get(cnt)) + ")]");
+                        cctx.kernalContext().closure().runLocalSafe(new Runnable() {
+                            @Override public void run() {
+                                try {
+                                    if (!fut.isDone()) {
+                                        cctx.io().sendOrderedMessage(node,
+                                            rebalanceTopics.get(finalCnt), initD, cctx.ioPolicy(), initD.timeout());
+
+                                        // Cleanup required in case partitions demanded in parallel with cancellation.
+                                        synchronized (fut) {
+                                            if (fut.isDone())
+                                                fut.cleanupRemoteContexts(node.id());
+                                        }
+
+                                        if (log.isDebugEnabled())
+                                            log.debug("Requested rebalancing [from node=" + node.id() + ", listener index=" +
+                                                finalCnt + ", partitions count=" + sParts.get(finalCnt).size() +
+                                                " (" + partitionsList(sParts.get(finalCnt)) + ")]");
+                                    }
+                                }
+                                catch (IgniteCheckedException e) {
+                                    ClusterTopologyCheckedException cause = e.getCause(ClusterTopologyCheckedException.class);
+
+                                    if (cause != null)
+                                        log.warning("Failed to send initial demand request to node. " + e.getMessage());
+                                    else
+                                        log.error("Failed to send initial demand request to node.", e);
+
+                                    fut.cancel();
+                                }
+                                catch (Throwable th) {
+                                    log.error("Runtime error caught during initial demand request sending.", th);
+
+                                    fut.cancel();
+                                }
+                            }
+                        }, /*system pool*/true);
                     }
                 }
             }
             else {
-                U.log(log, "Starting rebalancing (old api) [cache=" + cctx.name() +
-                    ", mode=" + cfg.getRebalanceMode() +
-                    ", fromNode=" + node.id() +
-                    ", partitionsCount=" + parts.size() +
-                    ", topology=" + fut.topologyVersion() +
-                    ", updateSeq=" + fut.updateSeq + "]");
+                try {
+                    U.log(log, "Starting rebalancing (old api) [cache=" + cctx.name() +
+                        ", mode=" + cfg.getRebalanceMode() +
+                        ", fromNode=" + node.id() +
+                        ", partitionsCount=" + parts.size() +
+                        ", topology=" + fut.topologyVersion() +
+                        ", updateSeq=" + fut.updateSeq + "]");
 
-                d.timeout(cctx.config().getRebalanceTimeout());
-                d.workerId(0);//old api support.
+                    d.timeout(cctx.config().getRebalanceTimeout());
+                    d.workerId(0);//old api support.
+
+                    worker = new DemandWorker(dmIdx.incrementAndGet(), fut);
 
-                worker = new DemandWorker(dmIdx.incrementAndGet(), fut);
+                    worker.run(node, d);
+                }
+                catch (Throwable th) {
+                    log.error("Runtime error caught during initial demand request sending.", th);
 
-                worker.run(node, d);
+                    fut.cancel();
+                }
             }
         }
     }

From 8220fb121eec86c18a711816f3db478b1d31a4e6 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 24 May 2017 19:55:09 +0300
Subject: [PATCH 284/446] ignite-5283 Fix of transaction recovery on backup
 when primary node failed

---
 .../GridDistributedLockRequest.java           |  20 ++
 .../distributed/dht/GridDhtLockFuture.java    |   1 +
 .../distributed/dht/GridDhtLockRequest.java   |   4 +
 .../dht/GridDhtTransactionalCacheAdapter.java |   3 +-
 .../dht/GridDhtTxPrepareFuture.java           |   2 +
 .../dht/GridDhtTxPrepareRequest.java          |  25 +-
 .../distributed/dht/GridDhtTxRemote.java      |  24 +-
 .../cache/transactions/IgniteTxHandler.java   |   3 +-
 .../IgniteTxCachePrimarySyncTest.java         |  35 ++-
 .../IgniteCachePutRetryAbstractSelfTest.java  |  48 ++++
 .../dht/TxRecoveryStoreEnabledTest.java       | 227 ++++++++++++++++++
 .../IgniteCacheTxRecoverySelfTestSuite.java   |   3 +
 12 files changed, 375 insertions(+), 20 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
index 9639a9a7288ee..3142c8ba836ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
@@ -48,6 +48,9 @@ public class GridDistributedLockRequest extends GridDistributedBaseMessage {
     /** */
     private static final long serialVersionUID = 0L;
 
+    /** */
+    private static final int STORE_USED_FLAG_MASK = 0x04;
+
     /** Sender node ID. */
     private UUID nodeId;
 
@@ -262,6 +265,23 @@ public boolean keepBinary() {
         return (flags & KEEP_BINARY_FLAG_MASK) != 0;
     }
 
+    /**
+     * @return Flag indicating whether transaction use cache store.
+     */
+    public boolean storeUsed() {
+        return (flags & STORE_USED_FLAG_MASK) != 0;
+    }
+
+    /**
+     * @param storeUsed Store used value.
+     */
+    public void storeUsed(boolean storeUsed) {
+        if (storeUsed)
+            flags = (byte)(flags | STORE_USED_FLAG_MASK);
+        else
+            flags &= ~STORE_USED_FLAG_MASK;
+    }
+
     /**
      * @return Transaction isolation or <tt>null</tt> if not in transaction.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
index 686a4c6bed872..ddeb1ad1da507 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
@@ -873,6 +873,7 @@ private void map(Iterable<GridDhtCacheEntry> entries) {
                         inTx() ? tx.taskNameHash() : 0,
                         read ? accessTtl : -1L,
                         skipStore,
+                        cctx.store().configured(),
                         keepBinary,
                         cctx.deploymentEnabled());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
index 95c6dfc06ce44..289ad93d884fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
@@ -120,6 +120,7 @@ public GridDhtLockRequest() {
      * @param taskNameHash Task name hash code.
      * @param accessTtl TTL for read operation.
      * @param skipStore Skip store flag.
+     * @param storeUsed Cache store configured flag.
      * @param keepBinary Keep binary flag.
      * @param addDepInfo Deployment info flag.
      */
@@ -144,6 +145,7 @@ public GridDhtLockRequest(
         int taskNameHash,
         long accessTtl,
         boolean skipStore,
+        boolean storeUsed,
         boolean keepBinary,
         boolean addDepInfo
     ) {
@@ -166,6 +168,8 @@ public GridDhtLockRequest(
 
         this.topVer = topVer;
 
+        storeUsed(storeUsed);
+
         nearKeys = nearCnt == 0 ? Collections.<KeyCacheObject>emptyList() : new ArrayList<KeyCacheObject>(nearCnt);
         invalidateEntries = new BitSet(dhtCnt == 0 ? nearCnt : dhtCnt);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index a9e3bc496721c..13a88e9720ed7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -245,7 +245,8 @@ protected GridDhtTransactionalCacheAdapter(GridCacheContext<K, V> ctx, GridCache
                                     req.timeout(),
                                     req.txSize(),
                                     req.subjectId(),
-                                    req.taskNameHash());
+                                    req.taskNameHash(),
+                                    !req.skipStore() && req.storeUsed());
 
                                 tx = ctx.tm().onCreated(null, tx);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index a759194453a78..4283ccb878a47 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -1247,6 +1247,7 @@ private void prepare0() {
                         tx.subjectId(),
                         tx.taskNameHash(),
                         tx.activeCachesDeploymentEnabled(),
+                        tx.storeUsed(),
                         retVal);
 
                     int idx = 0;
@@ -1359,6 +1360,7 @@ private void prepare0() {
                             tx.subjectId(),
                             tx.taskNameHash(),
                             tx.activeCachesDeploymentEnabled(),
+                            tx.storeUsed(),
                             retVal);
 
                         for (IgniteTxEntry entry : nearMapping.entries()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
index a8f20876abd29..9fa20d8b75b65 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
@@ -55,6 +55,9 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
     /** */
     public static final int NEED_RETURN_VALUE_FLAG_MASK = 0x01;
 
+    /** */
+    public static final int STORE_USED_FLAG_MASK = 0x02;
+
     /** Max order. */
     private UUID nearNodeId;
 
@@ -124,8 +127,9 @@ public GridDhtTxPrepareRequest() {
      * @param txNodes Transaction nodes mapping.
      * @param nearXidVer Near transaction ID.
      * @param last {@code True} if this is last prepare request for node.
-     * @param retVal Need return value flag.
      * @param addDepInfo Deployment info flag.
+     * @param storeUsed Cache store used flag.
+     * @param retVal Need return value flag.
      */
     public GridDhtTxPrepareRequest(
         IgniteUuid futId,
@@ -142,6 +146,7 @@ public GridDhtTxPrepareRequest(
         UUID subjId,
         int taskNameHash,
         boolean addDepInfo,
+        boolean storeUsed,
         boolean retVal) {
         super(tx, timeout, null, dhtWrites, txNodes, onePhaseCommit, addDepInfo);
 
@@ -157,6 +162,7 @@ public GridDhtTxPrepareRequest(
         this.subjId = subjId;
         this.taskNameHash = taskNameHash;
 
+        storeUsed(storeUsed);
         needReturnValue(retVal);
 
         invalidateNearEntries = new BitSet(dhtWrites == null ? 0 : dhtWrites.size());
@@ -181,6 +187,23 @@ public void needReturnValue(boolean retVal) {
             flags &= ~NEED_RETURN_VALUE_FLAG_MASK;
     }
 
+    /**
+     * @return Flag indicating whether transaction use cache store.
+     */
+    public boolean storeUsed() {
+        return (flags & STORE_USED_FLAG_MASK) != 0;
+    }
+
+    /**
+     * @param storeUsed Store used value.
+     */
+    public void storeUsed(boolean storeUsed) {
+        if (storeUsed)
+            flags = (byte)(flags | STORE_USED_FLAG_MASK);
+        else
+            flags &= ~STORE_USED_FLAG_MASK;
+    }
+
     /**
      * @return {@code True} if this is last prepare request for node.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
index 6ad20c7566553..8942ef9d9178c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
@@ -64,6 +64,9 @@ public class GridDhtTxRemote extends GridDistributedTxRemoteAdapter {
     /** Near transaction ID. */
     private GridCacheVersion nearXidVer;
 
+    /** Store used. */
+    private boolean storeUsed;
+
     /**
      * Empty constructor required for {@link Externalizable}.
      */
@@ -74,6 +77,7 @@ public GridDhtTxRemote() {
     /**
      * This constructor is meant for optimistic transactions.
      *
+     * @param ctx Cache context.
      * @param nearNodeId Near node ID.
      * @param rmtFutId Remote future ID.
      * @param nodeId Node ID.
@@ -85,10 +89,10 @@ public GridDhtTxRemote() {
      * @param isolation Transaction isolation.
      * @param invalidate Invalidate flag.
      * @param timeout Timeout.
-     * @param ctx Cache context.
      * @param txSize Expected transaction size.
      * @param nearXidVer Near transaction ID.
      * @param txNodes Transaction nodes mapping.
+     * @param storeUsed Cache store used flag.
      */
     public GridDhtTxRemote(
         GridCacheSharedContext ctx,
@@ -109,8 +113,8 @@ public GridDhtTxRemote(
         Map<UUID, Collection<UUID>> txNodes,
         @Nullable UUID subjId,
         int taskNameHash,
-        boolean single
-    ) {
+        boolean single,
+        boolean storeUsed) {
         super(
             ctx,
             nodeId,
@@ -134,6 +138,7 @@ public GridDhtTxRemote(
         this.rmtFutId = rmtFutId;
         this.nearXidVer = nearXidVer;
         this.txNodes = txNodes;
+        this.storeUsed = storeUsed;
 
         txState = single ? new IgniteTxRemoteSingleStateImpl() :
             new IgniteTxRemoteStateImpl(
@@ -148,6 +153,7 @@ public GridDhtTxRemote(
     /**
      * This constructor is meant for pessimistic transactions.
      *
+     * @param ctx Cache context.
      * @param nearNodeId Near node ID.
      * @param rmtFutId Remote future ID.
      * @param nodeId Node ID.
@@ -160,8 +166,8 @@ public GridDhtTxRemote(
      * @param isolation Transaction isolation.
      * @param invalidate Invalidate flag.
      * @param timeout Timeout.
-     * @param ctx Cache context.
      * @param txSize Expected transaction size.
+     * @param storeUsed Cache store used flag.
      */
     public GridDhtTxRemote(
         GridCacheSharedContext ctx,
@@ -180,8 +186,8 @@ public GridDhtTxRemote(
         long timeout,
         int txSize,
         @Nullable UUID subjId,
-        int taskNameHash
-    ) {
+        int taskNameHash,
+        boolean storeUsed) {
         super(
             ctx,
             nodeId,
@@ -204,6 +210,7 @@ public GridDhtTxRemote(
         this.nearXidVer = nearXidVer;
         this.nearNodeId = nearNodeId;
         this.rmtFutId = rmtFutId;
+        this.storeUsed = storeUsed;
 
         txState = new IgniteTxRemoteStateImpl(
             Collections.<IgniteTxKey, IgniteTxEntry>emptyMap(),
@@ -226,6 +233,11 @@ public void transactionNodes(Map<UUID, Collection<UUID>> txNodes) {
         return true;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean storeUsed() {
+        return storeUsed;
+    }
+
     /** {@inheritDoc} */
     @Override public UUID eventNodeId() {
         return nearNodeId();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index d56415652bc73..71bf08cfdfdbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -1417,7 +1417,8 @@ protected void sendReply(UUID nodeId, GridDhtTxFinishRequest req, boolean commit
                     req.transactionNodes(),
                     req.subjectId(),
                     req.taskNameHash(),
-                    single);
+                    single,
+                    req.storeUsed());
 
                 tx.writeVersion(req.writeVersion());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java
index 3bc22ef8c05c7..020ec5c8646d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
 import javax.cache.Cache;
 import javax.cache.configuration.Factory;
 import javax.cache.integration.CacheLoaderException;
@@ -62,6 +63,7 @@
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
+import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
@@ -1096,19 +1098,30 @@ private static class TestStoreFactory implements Factory<CacheStore<Object, Obje
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
         @Override public CacheStore<Object, Object> create() {
-            return new CacheStoreAdapter() {
-                @Override public Object load(Object key) throws CacheLoaderException {
-                    return null;
-                }
+            return new TestCacheStore();
+        }
+    }
 
-                @Override public void write(Cache.Entry entry) throws CacheWriterException {
-                    // No-op.
-                }
+    /**
+     *
+     */
+    private static class TestCacheStore extends CacheStoreAdapter<Object, Object> {
+        /** Store map. */
+        private static final Map STORE_MAP = new ConcurrentHashMap();
 
-                @Override public void delete(Object key) throws CacheWriterException {
-                    // No-op.
-                }
-            };
+        /** {@inheritDoc} */
+        @Override public Object load(Object key) throws CacheLoaderException {
+            return STORE_MAP.get(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<?, ?> entry) throws CacheWriterException {
+            STORE_MAP.put(entry.getKey(), entry.getValue());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            STORE_MAP.remove(key);
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
index c3d194b0c4f27..fc9017927d3bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
@@ -24,6 +24,7 @@
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.cache.Cache;
 import javax.cache.configuration.Factory;
@@ -56,6 +57,7 @@
 import org.apache.ignite.spi.swapspace.inmemory.GridTestSwapSpaceSpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -650,4 +652,50 @@ private static class TestStoreFactory implements Factory<CacheStore> {
             };
         }
     }
+
+
+/*
+    private static class TestStoreFactory implements Factory<CacheStore> {
+        */
+/** {@inheritDoc} *//*
+
+        @Override public CacheStore create() {
+            return new TestCacheStore();
+        }
+    }
+
+    */
+/**
+     *
+     *//*
+
+    private static class TestCacheStore extends CacheStoreAdapter {
+        */
+/** Store map. *//*
+
+        private static Map STORE_MAP = new ConcurrentHashMap();
+
+        */
+/** {@inheritDoc} *//*
+
+        @Override public Object load(Object key) throws CacheLoaderException {
+            return STORE_MAP.get(key);
+        }
+
+        */
+/** {@inheritDoc} *//*
+
+        @Override public void write(Cache.Entry entry) throws CacheWriterException {
+            STORE_MAP.put(entry.getKey(), entry.getValue());
+        }
+
+        */
+/** {@inheritDoc} *//*
+
+        @Override public void delete(Object key) throws CacheWriterException {
+            STORE_MAP.remove(key);
+        }
+    }
+*/
+
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
new file mode 100644
index 0000000000000..73f0268c42ef3
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.concurrent.CountDownLatch;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+
+/**
+ *
+ */
+public class TxRecoveryStoreEnabledTest extends GridCommonAbstractTest {
+    /** Nodes count. */
+    private static final int NODES_CNT = 2;
+
+    /** Latch. */
+    private static CountDownLatch latch;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCommunicationSpi(new TestCommunicationSpi());
+        cfg.setDiscoverySpi(new TestDiscoverySpi());
+
+        CacheConfiguration ccfg = defaultCacheConfiguration();
+
+        ccfg.setNearConfiguration(null);
+        ccfg.setCacheMode(CacheMode.PARTITIONED);
+        ccfg.setBackups(1);
+        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+        ccfg.setCacheStoreFactory(new TestCacheStoreFactory());
+        ccfg.setReadThrough(true);
+        ccfg.setWriteThrough(true);
+        ccfg.setWriteBehindEnabled(false);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        latch = new CountDownLatch(1);
+
+        startGrids(NODES_CNT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOptimistic() throws Exception {
+        checkTxRecovery(OPTIMISTIC);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPessimistic() throws Exception {
+        checkTxRecovery(PESSIMISTIC);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkTxRecovery(TransactionConcurrency concurrency) throws Exception {
+        final Ignite node0 = ignite(0);
+        Ignite node1 = ignite(1);
+
+        IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    latch.await();
+
+                    IgniteConfiguration cfg = node0.configuration();
+
+                    ((TestCommunicationSpi)cfg.getCommunicationSpi()).block();
+                    ((TestDiscoverySpi)cfg.getDiscoverySpi()).simulateNodeFailure();
+                }
+                catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                }
+            }
+        }, 1);
+
+        IgniteCache<Object, Object> cache0 = node0.cache(null);
+
+        Integer key = primaryKey(cache0);
+
+        try (Transaction tx = node0.transactions().txStart(concurrency, READ_COMMITTED)) {
+            cache0.put(key, key);
+
+            tx.commit();
+        }
+        catch (Exception e) {
+            // No-op.
+        }
+
+        fut.get();
+
+        IgniteCache<Object, Object> cache1 = node1.cache(null);
+
+        assertNull(cache1.get(key));
+    }
+
+    /**
+     *
+     */
+    private static class TestCacheStoreFactory implements Factory<CacheStore> {
+        /** {@inheritDoc} */
+        @Override public CacheStore create() {
+            return new TestCacheStore();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestCacheStore extends CacheStoreAdapter<Integer, Integer> {
+        /** {@inheritDoc} */
+        @Override public void sessionEnd(boolean commit) {
+            if (latch.getCount() > 0) { // Need wait only on primary node.
+                latch.countDown();
+
+                try {
+                    U.sleep(3000);
+                }
+                catch (IgniteInterruptedCheckedException e) {
+                    Thread.currentThread().interrupt();
+                }
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer load(Integer key) throws CacheLoaderException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Integer, ? extends Integer> entry) throws CacheWriterException {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            // no-op.
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void simulateNodeFailure() {
+            super.simulateNodeFailure();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** Block. */
+        private volatile boolean block;
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+            if (!block)
+                super.sendMessage(node, msg);
+        }
+
+        /**
+         *
+         */
+        private void block() {
+            block = true;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
index 7363c7cc1f2a8..c7c8db67fe31e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedNearDisabledPrimaryNodeFailureRecoveryTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedPrimaryNodeFailureRecoveryTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedTwoBackupsPrimaryNodeFailureRecoveryTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.TxRecoveryStoreEnabledTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearTxPessimisticOriginatingNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxOriginatingNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxPessimisticOriginatingNodeFailureSelfTest;
@@ -54,6 +55,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheNearTxPessimisticOriginatingNodeFailureSelfTest.class);
         suite.addTestSuite(GridCacheReplicatedTxPessimisticOriginatingNodeFailureSelfTest.class);
 
+        suite.addTestSuite(TxRecoveryStoreEnabledTest.class);
+
         return suite;
     }
 }
\ No newline at end of file

From 5cb580ad7043f27e4a0396aea1f877c21d49078e Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 6 Jun 2017 16:17:01 +0300
Subject: [PATCH 285/446] IGNITE-5259 Minor serialization fix

(cherry picked from commit 374cba8)
---
 .../discovery/GridDiscoveryManager.java       |  6 ++---
 .../processors/security/SecurityUtils.java    | 22 +++++++++++++++++--
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 17 +++++++++++++-
 3 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index fb72989eb81ba..71b7217bd6afc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -138,8 +138,8 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME;
 import static org.apache.ignite.internal.IgniteVersionUtils.VER;
-import static org.apache.ignite.internal.processors.security.SecurityUtils.SERVICE_PERMISSIONS_SINCE;
 import static org.apache.ignite.internal.processors.security.SecurityUtils.isSecurityCompatibilityMode;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.isServiceSecuritySupported;
 import static org.apache.ignite.plugin.segmentation.SegmentationPolicy.NOOP;
 
 /**
@@ -1163,7 +1163,7 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                     ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
             }
 
-            if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) >= 0
+            if (isServiceSecuritySupported(n.version())
                 && ctx.security().enabled() // Matters only if security enabled.
                ) {
                 Boolean rmtSecurityCompatibilityEnabled = n.attribute(ATTR_SECURITY_COMPATIBILITY_MODE);
@@ -1181,7 +1181,7 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                 }
             }
 
-            if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) < 0
+            if (!isServiceSecuritySupported(n.version())
                 && ctx.security().enabled() // Matters only if security enabled.
                 && (locSecurityCompatibilityEnabled == null || !locSecurityCompatibilityEnabled)) {
                 throw new IgniteCheckedException("Remote node does not support service security permissions. " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
index 1016335888b3d..306d8de58d87f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
@@ -29,8 +29,11 @@
  * Security utilities.
  */
 public class SecurityUtils {
-    /** Version since service security supported. */
-    public static final IgniteProductVersion SERVICE_PERMISSIONS_SINCE = IgniteProductVersion.fromString("1.7.11");
+    /** Versions since service security supported. */
+    private static final IgniteProductVersion[] SERVICE_PERMISSIONS_SINCE = {
+        IgniteProductVersion.fromString("1.7.11"),
+        IgniteProductVersion.fromString("1.8.7")
+    };
 
     /** Default serialization version. */
     private final static int DFLT_SERIALIZE_VERSION = isSecurityCompatibilityMode() ? 1 : 2;
@@ -89,4 +92,19 @@ public static Map<String, Collection<SecurityPermission>> compatibleServicePermi
 
         return srvcPerms;
     }
+
+    /**
+     * Checks whether provided release supports service security permissions.
+     *
+     * @param ver Version to ckeck.
+     * @return {@code True} if passed release supports service security permissions.
+     */
+    public static boolean isServiceSecuritySupported(IgniteProductVersion ver) {
+        for (IgniteProductVersion v : SERVICE_PERMISSIONS_SINCE) {
+            if (v.major() == ver.major() && v.minor() == ver.minor())
+                return ver.compareToIgnoreTimestamp(v) >= 0;
+        }
+
+        return ver.compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE[SERVICE_PERMISSIONS_SINCE.length]) >= 0;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 9225a922b5dd3..e7bc81e5bc55f 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.processors.security.SecurityUtils;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
@@ -522,7 +523,16 @@ else if (state == DISCONNECTED) {
 
                 InetSocketAddress addr = it.next();
 
-                T3<SocketStream, Integer, Boolean> sockAndRes = sendJoinRequest(recon, addr);
+                T3<SocketStream, Integer, Boolean> sockAndRes;
+
+                SecurityUtils.serializeVersion(1);
+
+                try {
+                    sockAndRes = sendJoinRequest(recon, addr);
+                }
+                finally {
+                    SecurityUtils.restoreDefaultSerializeVersion();
+                }
 
                 if (sockAndRes == null) {
                     it.remove();
@@ -984,6 +994,8 @@ private void forceStopRead() throws InterruptedException {
                     while (!isInterrupted()) {
                         TcpDiscoveryAbstractMessage msg;
 
+                        SecurityUtils.serializeVersion(1);
+
                         try {
                             msg = U.unmarshal(spi.marshaller(), in, U.resolveClassLoader(spi.ignite().configuration()));
                         }
@@ -1009,6 +1021,9 @@ private void forceStopRead() throws InterruptedException {
 
                             continue;
                         }
+                        finally {
+                            SecurityUtils.restoreDefaultSerializeVersion();
+                        }
 
                         msg.senderNodeId(rmtNodeId);
 

From f03252f9b2c6f0e777f307fd85cc8bd20ab21423 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 6 Jun 2017 16:17:01 +0300
Subject: [PATCH 286/446] IGNITE-5259 Minor serialization fix

(cherry picked from commit 374cba8)
---
 .../ignite/spi/discovery/tcp/ClientImpl.java    | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 02ba56a884b33..b075dc1fb5289 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.processors.security.SecurityUtils;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
@@ -516,7 +517,16 @@ else if (state == DISCONNECTED) {
 
                 InetSocketAddress addr = it.next();
 
-                T3<SocketStream, Integer, Boolean> sockAndRes = sendJoinRequest(recon, addr);
+                T3<SocketStream, Integer, Boolean> sockAndRes;
+
+                SecurityUtils.serializeVersion(1);
+
+                try {
+                    sockAndRes = sendJoinRequest(recon, addr);
+                }
+                finally {
+                    SecurityUtils.restoreDefaultSerializeVersion();
+                }
 
                 if (sockAndRes == null) {
                     it.remove();
@@ -969,6 +979,8 @@ private void forceStopRead() throws InterruptedException {
                     while (!isInterrupted()) {
                         TcpDiscoveryAbstractMessage msg;
 
+                        SecurityUtils.serializeVersion(1);
+
                         try {
                             msg = U.unmarshal(spi.marshaller(), in, U.resolveClassLoader(spi.ignite().configuration()));
                         }
@@ -994,6 +1006,9 @@ private void forceStopRead() throws InterruptedException {
 
                             continue;
                         }
+                        finally {
+                            SecurityUtils.restoreDefaultSerializeVersion();
+                        }
 
                         msg.senderNodeId(rmtNodeId);
 

From 3a1d560cd8741de9e7a6dd1110b42814d0ccff6b Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 6 Jun 2017 17:13:52 +0300
Subject: [PATCH 287/446] IGNITE-5259 Minor serialization fix

---
 .../ignite/internal/processors/security/SecurityUtils.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
index 306d8de58d87f..ca256c922f545 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
@@ -105,6 +105,6 @@ public static boolean isServiceSecuritySupported(IgniteProductVersion ver) {
                 return ver.compareToIgnoreTimestamp(v) >= 0;
         }
 
-        return ver.compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE[SERVICE_PERMISSIONS_SINCE.length]) >= 0;
+        return ver.compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE[SERVICE_PERMISSIONS_SINCE.length - 1]) >= 0;
     }
 }

From cfbe8da934741e76c8964af87671a38ec7b6c9a3 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 6 Jun 2017 19:15:59 +0300
Subject: [PATCH 288/446] IGNITE-5103 Rolled back due to test failings.

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  19 +--
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  57 +--------
 .../TcpDiscoveryClientSuspensionSelfTest.java | 116 ------------------
 3 files changed, 5 insertions(+), 187 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index e7bc81e5bc55f..b075dc1fb5289 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -367,7 +367,7 @@ else if (state == DISCONNECTED) {
                 else {
                     final GridFutureAdapter<Boolean> finalFut = fut;
 
-                    TimerTask task = new TimerTask() {
+                    timer.schedule(new TimerTask() {
                         @Override public void run() {
                             if (pingFuts.remove(nodeId, finalFut)) {
                                 if (ClientImpl.this.state == DISCONNECTED)
@@ -377,13 +377,7 @@ else if (state == DISCONNECTED) {
                                     finalFut.onDone(false);
                             }
                         }
-                    };
-
-                    try {
-                        timer.schedule(task, spi.netTimeout);
-                    } catch (IllegalStateException e) {
-                        return false;  // timer is cancelled because this client node is dying
-                    }
+                    }, spi.netTimeout);
 
                     sockWriter.sendMessage(new TcpDiscoveryClientPingRequest(getLocalNodeId(), nodeId));
                 }
@@ -819,13 +813,7 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
         U.warn(log, "Simulating client node failure: " + getLocalNodeId());
 
         U.interrupt(sockWriter);
-        U.interrupt(sockReader);
         U.interrupt(msgWorker);
-        try {
-            timer.cancel();
-        } catch (Throwable e) {
-            // No-op.
-        }
 
         U.join(sockWriter, log);
         U.join(
@@ -892,9 +880,6 @@ private class HeartbeatSender extends TimerTask {
                 msg.client(true);
 
                 sockWriter.sendMessage(msg);
-
-                if (log.isDebugEnabled())
-                    log.debug("*** Send heartbeat message from node:    "+msg.creatorNodeId());
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index a8feab41abd9a..c91954cb18c92 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6165,11 +6165,9 @@ private void processClientHeartbeatMessage(TcpDiscoveryClientHeartbeatMessage ms
             assert msg.client();
 
             ClientMessageWorker wrk = clientMsgWorkers.get(msg.creatorNodeId());
-            if (wrk != null) {
-                msg.verify(getLocalNodeId());
 
-                wrk.addMessage(msg);
-            }
+            if (wrk != null)
+                wrk.metrics(msg.metrics());
             else if (log.isDebugEnabled())
                 log.debug("Received heartbeat message from unknown client node: " + msg);
         }
@@ -6302,9 +6300,6 @@ private class StatisticsPrinter extends IgniteSpiThread {
     /**
      */
     private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAbstractMessage, byte[]>> {
-        /** minimal period of time which can be used as heartbeat timeout */
-        public static final int minHeartbeaTimeout = 10; // ms
-
         /** Node ID. */
         private final UUID clientNodeId;
 
@@ -6320,26 +6315,15 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
         /** */
         private IgniteProductVersion clientVer;
 
-        /**
-         * Period of time after which client node which does not send heartbeat messages, is considered dead.
-         * Measured in milliseconds
-         */
-        private long heartBeatTimeOut;
-
-        /** timestamp of the last received heartbeat message, in milliseconds */
-        private long lastHeartBeatTime;
-
         /**
          * @param sock Socket.
          * @param clientNodeId Node ID.
          */
         protected ClientMessageWorker(Socket sock, UUID clientNodeId) throws IOException {
-            super("tcp-disco-client-message-worker", Math.max(spi.getHeartbeatFrequency(), minHeartbeaTimeout));
+            super("tcp-disco-client-message-worker", 2000);
 
             this.sock = sock;
             this.clientNodeId = clientNodeId;
-            this.heartBeatTimeOut = spi.getHeartbeatFrequency() * spi.getMaxMissedClientHeartbeats();
-            this.lastHeartBeatTime=U.currentTimeMillis();
         }
 
         /**
@@ -6388,32 +6372,6 @@ void addMessage(TcpDiscoveryAbstractMessage msg, @Nullable byte[] msgBytes) {
                 log.debug("Message has been added to client queue: " + msg);
         }
 
-
-        /**
-         * Check the last time a heartbeat message received.
-         * In case of timeout, expel client node from the topology
-         */
-        @Override protected void noMessageLoop() {
-            long period = U.currentTimeMillis() - lastHeartBeatTime;
-
-            if (period >= heartBeatTimeOut) {
-                if (log.isInfoEnabled())
-                    log.info("Heartbeat timeout for node:" + clientNodeId + "; timeOut=" + heartBeatTimeOut + "; period=" + period);
-
-                TcpDiscoveryAbstractMessage msg = new TcpDiscoveryNodeLeftMessage(clientNodeId);
-
-                msg.senderNodeId(getLocalNodeId());
-
-                msgWorker.addMessage(msg);
-
-                clientMsgWorkers.remove(clientNodeId, this);
-
-                U.interrupt(this);
-
-                U.closeQuiet(sock);
-            }
-        }
-
         /** {@inheritDoc} */
         @Override protected void processMessage(T2<TcpDiscoveryAbstractMessage, byte[]> msgT) {
             boolean success = false;
@@ -6452,15 +6410,6 @@ else if (msgLog.isDebugEnabled())
                             spi.failureDetectionTimeout() : spi.getSocketTimeout());
                     }
                 }
-                else if (msg instanceof TcpDiscoveryClientHeartbeatMessage) {
-                    TcpDiscoveryClientHeartbeatMessage hbmsg = (TcpDiscoveryClientHeartbeatMessage)msg;
-
-                    if (log.isDebugEnabled()) // TODO turn to debug
-                        log.debug("###  Received heartbeat message from node:" + hbmsg.creatorNodeId());
-
-                    this.metrics = hbmsg.metrics();
-                    this.lastHeartBeatTime=U.currentTimeMillis();
-                }
                 else {
                     if (msgLog.isDebugEnabled())
                         msgLog.debug("Redirecting message to client [sock=" + sock + ", locNodeId="
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
deleted file mode 100644
index 9dc6e506cc74a..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.tcp;
-
-import java.util.Timer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Test for missed client heartbeats.
- */
-public class TcpDiscoveryClientSuspensionSelfTest extends GridCommonAbstractTest {
-    /** */
-    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        TcpDiscoverySpi disco = new TcpDiscoverySpi();
-
-        disco.setIpFinder(IP_FINDER);
-        disco.setHeartbeatFrequency(200);
-        disco.setMaxMissedClientHeartbeats(10);
-
-        cfg.setDiscoverySpi(disco);
-
-        return cfg;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        stopAllGrids();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testOneServer() throws Exception {
-        doTestClientSuspension(1);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testTwoServers() throws Exception {
-        doTestClientSuspension(2);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testThreeServers() throws Exception {
-        doTestClientSuspension(3);
-    }
-
-    /**
-     * @param serverCnt Servers count.
-     * @throws Exception If failed.
-     */
-    private void doTestClientSuspension(int serverCnt) throws Exception {
-        startGrids(serverCnt);
-
-        Ignition.setClientMode(true);
-
-        Ignite client = startGrid("client");
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(1, grid(i).cluster().forClients().nodes().size());
-
-        Thread.sleep(3000);
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(1, grid(i).cluster().forClients().nodes().size());
-
-        suspendClientHeartbeats(client);
-
-        Thread.sleep(3000);
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(0, grid(i).cluster().forClients().nodes().size());
-    }
-
-    /**
-     * @param client Client.
-     */
-    private void suspendClientHeartbeats(Ignite client) {
-        assert client.cluster().localNode().isClient();
-
-        ClientImpl impl = U.field(client.configuration().getDiscoverySpi(), "impl");
-
-        impl.simulateNodeFailure();
-
-        //Timer timer = U.field(impl, "timer"); timer.cancel(); -- client node successfully reconnects
-    }
-}

From 83307da08289c873c5c2eb02d5eb314018bc5c13 Mon Sep 17 00:00:00 2001
From: Ivan Veselovskiy <iveselovskiy@gridgain.com>
Date: Tue, 6 Jun 2017 16:56:09 +0300
Subject: [PATCH 289/446] IGNITE-5410: Fixed assertion in HadoopDataOutStream.
 This closes #2084.

(cherry picked from commit d2bf961)
---
 .../processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
index acc9be62cf677..655346eaae93a 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
@@ -92,7 +92,7 @@ public void pointer(long ptr) {
      * @return Old position pointer or {@code 0} if move goes beyond the end of the buffer.
      */
     public long move(long size) {
-        assert size > 0 : size;
+        assert size >= 0 : size;
 
         long oldPos = posPtr;
         long newPos = oldPos + size;

From e95626d609ee225918b49653b7981b180e5d4e49 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 1 Jun 2017 19:56:34 +0300
Subject: [PATCH 290/446] SSL fix

(cherry picked from commit 95d5595)
---
 .../ignite/spi/communication/tcp/TcpCommunicationSpi.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 60d97348d42cf..dbad401e9d037 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3198,10 +3198,10 @@ else if (log.isDebugEnabled())
                             assert sslHnd != null;
 
                             buf = ByteBuffer.allocate(1000);
+                            buf.order(ByteOrder.nativeOrder());
 
                             ByteBuffer decode = ByteBuffer.allocate(2 * buf.capacity());
-
-                            buf.order(ByteOrder.nativeOrder());
+                            decode.order(ByteOrder.nativeOrder());
 
                             for (int i = 0; i < 9; ) {
                                 int read = ch.read(buf);

From f3f726e9059e492573dc5125fd5edb5d2f71e9d3 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 13 Jun 2017 14:11:17 +0300
Subject: [PATCH 291/446] IGNITE-4196: Added means to specify port for H2 debug
 console. This closes #1486.

(cherry picked from commit b246260)
---
 .../java/org/apache/ignite/IgniteSystemProperties.java    | 8 ++++++++
 .../internal/processors/query/h2/IgniteH2Indexing.java    | 7 ++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index e933d83dda840..88aeae6bfccd5 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -326,6 +326,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_H2_DEBUG_CONSOLE = "IGNITE_H2_DEBUG_CONSOLE";
 
+    /**
+     * This property allows to specify user defined port which H2 indexing SPI will use
+     * to start H2 debug console on. If this property is not set or set to 0, H2 debug
+     * console will use system-provided dynamic port.
+     * This property is only relevant when {@link #IGNITE_H2_DEBUG_CONSOLE} property is set.
+     */
+    public static final String IGNITE_H2_DEBUG_CONSOLE_PORT = "IGNITE_H2_DEBUG_CONSOLE_PORT";
+
     /**
      * If this property is set to {@code true} then shared memory space native debug will be enabled.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 8c5c2a364cabf..66fb7ae94f5be 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -176,6 +176,7 @@
 import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE_PORT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_CLEANUP_PERIOD;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_THREAD_USAGE_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.getInteger;
@@ -1797,11 +1798,15 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
             if (getString(IGNITE_H2_DEBUG_CONSOLE) != null) {
                 Connection c = DriverManager.getConnection(dbUrl);
 
+                int port = getInteger(IGNITE_H2_DEBUG_CONSOLE_PORT, 0);
+
                 WebServer webSrv = new WebServer();
-                Server web = new Server(webSrv, "-webPort", "0");
+                Server web = new Server(webSrv, "-webPort", Integer.toString(port));
                 web.start();
                 String url = webSrv.addSession(c);
 
+                U.quietAndInfo(log, "H2 debug console URL: " + url);
+
                 try {
                     Server.openBrowser(url);
                 }

From c2c237d1222557d3e6b35d9a51a61a4c78e56782 Mon Sep 17 00:00:00 2001
From: Sergey Kalashnikov <skalashnikov@gridgain.com>
Date: Fri, 3 Feb 2017 11:41:14 +0300
Subject: [PATCH 292/446] IGNITE-4196: Added means to specify port for H2 debug
 console. This closes #1486.

(cherry picked from commit b246260)
---
 .../java/org/apache/ignite/IgniteSystemProperties.java    | 8 ++++++++
 .../internal/processors/query/h2/IgniteH2Indexing.java    | 8 +++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 6827e0c248290..f35e9e620070a 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -320,6 +320,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_H2_DEBUG_CONSOLE = "IGNITE_H2_DEBUG_CONSOLE";
 
+    /**
+     * This property allows to specify user defined port which H2 indexing SPI will use
+     * to start H2 debug console on. If this property is not set or set to 0, H2 debug
+     * console will use system-provided dynamic port.
+     * This property is only relevant when {@link #IGNITE_H2_DEBUG_CONSOLE} property is set.
+     */
+    public static final String IGNITE_H2_DEBUG_CONSOLE_PORT = "IGNITE_H2_DEBUG_CONSOLE_PORT";
+
     /**
      * If this property is set to {@code true} then shared memory space native debug will be enabled.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 62b47b85c6507..8b2993f263cf7 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -171,8 +171,10 @@
 import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE_PORT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_CLEANUP_PERIOD;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_THREAD_USAGE_TIMEOUT;
+import static org.apache.ignite.IgniteSystemProperties.getInteger;
 import static org.apache.ignite.IgniteSystemProperties.getString;
 import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL;
 import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
@@ -1751,11 +1753,15 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
             if (getString(IGNITE_H2_DEBUG_CONSOLE) != null) {
                 Connection c = DriverManager.getConnection(dbUrl);
 
+                int port = getInteger(IGNITE_H2_DEBUG_CONSOLE_PORT, 0);
+
                 WebServer webSrv = new WebServer();
-                Server web = new Server(webSrv, "-webPort", "0");
+                Server web = new Server(webSrv, "-webPort", Integer.toString(port));
                 web.start();
                 String url = webSrv.addSession(c);
 
+                U.quietAndInfo(log, "H2 debug console URL: " + url);
+
                 try {
                     Server.openBrowser(url);
                 }

From 4a8f295f2f2f34e8472b1d1320f03744135b2504 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 13 Jun 2017 19:47:00 +0300
Subject: [PATCH 293/446] IGNITE-5478: ODBC: SQLNumParams now returns number of
 required parameters.

(cherry picked from commit b1c56a1)
---
 .../cpp/odbc-test/src/queries_test.cpp        | 30 +++++++++++++++++
 .../cpp/odbc/include/ignite/odbc/statement.h  | 13 ++++++--
 modules/platforms/cpp/odbc/src/odbc.cpp       |  7 +++-
 modules/platforms/cpp/odbc/src/statement.cpp  | 33 +++++++++++++++++--
 4 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 52b885d1d2298..6dc5d4f63cf3c 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -273,6 +273,25 @@ struct QueriesTestSuiteFixture
         BOOST_CHECK(ret == SQL_NO_DATA);
     }
 
+    void CheckParamsNum(const std::string& req, SQLSMALLINT expectedParamsNum)
+    {
+        std::vector<SQLCHAR> req0(req.begin(), req.end());
+
+        SQLRETURN ret = SQLPrepare(stmt, &req0[0], static_cast<SQLINTEGER>(req0.size()));
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        SQLSMALLINT paramsNum = -1;
+
+        ret = SQLNumParams(stmt, &paramsNum);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECK_EQUAL(paramsNum, expectedParamsNum);
+    }
+
     int CountRows(SQLHSTMT stmt)
     {
         int res = 0;
@@ -1342,4 +1361,15 @@ BOOST_AUTO_TEST_CASE(TestInsertMergeSelect)
     BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
 }
 
+BOOST_AUTO_TEST_CASE(TestParamsNum)
+{
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+
+    CheckParamsNum("SELECT * FROM TestType", 0);
+    CheckParamsNum("SELECT * FROM TestType WHERE _key=?", 1);
+    CheckParamsNum("SELECT * FROM TestType WHERE _key=? AND _val=?", 2);
+    CheckParamsNum("INSERT INTO TestType(_key, strField) VALUES(1, 'some')", 0);
+    CheckParamsNum("INSERT INTO TestType(_key, strField) VALUES(?, ?)", 2);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
index db566609d1b86..1ee56197ce81b 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
@@ -137,11 +137,11 @@ namespace ignite
             void GetAttribute(int attr, void* buf, SQLINTEGER bufLen, SQLINTEGER *valueLen);
 
             /**
-             * Get number of binded parameters.
+             * Get number parameters required by the prepared statement.
              *
-             * @return Number of binded parameters.
+             * @param paramNum Number of parameters.
              */
-            uint16_t GetParametersNumber();
+            void GetParametersNumber(uint16_t& paramNum);
 
             /**
              * Set parameter binding offset pointer.
@@ -394,6 +394,13 @@ namespace ignite
              */
             SqlResult InternalGetAttribute(int attr, void* buf, SQLINTEGER bufLen, SQLINTEGER* valueLen);
 
+            /**
+             * Get number parameters required by the prepared statement.
+             *
+             * @param paramNum Number of parameters.
+             */
+            SqlResult InternalGetParametersNumber(uint16_t& paramNum);
+
             /**
              * Get value of the column in the result set.
              *
diff --git a/modules/platforms/cpp/odbc/src/odbc.cpp b/modules/platforms/cpp/odbc/src/odbc.cpp
index 0b18a116f4993..caf31d91860b1 100644
--- a/modules/platforms/cpp/odbc/src/odbc.cpp
+++ b/modules/platforms/cpp/odbc/src/odbc.cpp
@@ -929,7 +929,12 @@ namespace ignite
             return SQL_INVALID_HANDLE;
 
         if (paramCnt)
-            *paramCnt = static_cast<SQLSMALLINT>(statement->GetParametersNumber());
+        {
+            uint16_t paramNum = 0;
+            statement->GetParametersNumber(paramNum);
+
+            *paramCnt = static_cast<SQLSMALLINT>(paramNum);
+        }
 
         return statement->GetDiagnosticRecords().GetReturnCode();
     }
diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp
index 02c6dd972154b..9aca8c9164209 100644
--- a/modules/platforms/cpp/odbc/src/statement.cpp
+++ b/modules/platforms/cpp/odbc/src/statement.cpp
@@ -296,11 +296,38 @@ namespace ignite
             return SQL_RESULT_SUCCESS;
         }
 
-        uint16_t Statement::GetParametersNumber()
+        void Statement::GetParametersNumber(uint16_t& paramNum)
         {
-            IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
+            IGNITE_ODBC_API_CALL(InternalGetParametersNumber(paramNum));
+        }
+
+        SqlResult Statement::InternalGetParametersNumber(uint16_t& paramNum)
+        {
+            if (!currentQuery.get())
+            {
+                AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query is not prepared.");
+
+                return SQL_RESULT_ERROR;
+            }
+
+            if (currentQuery->GetType() != query::Query::DATA)
+            {
+                paramNum = 0;
 
-            return static_cast<uint16_t>(paramBindings.size());
+                return SQL_RESULT_SUCCESS;
+            }
+
+            if (paramTypes.empty())
+            {
+                SqlResult res = UpdateParamsMeta();
+
+                if (res != SQL_RESULT_SUCCESS)
+                    return res;
+            }
+
+            paramNum = static_cast<uint16_t>(paramTypes.size());
+
+            return SQL_RESULT_SUCCESS;
         }
 
         void Statement::SetParamBindOffsetPtr(int* ptr)

From a2a4ec1ee9794cb542f146a07c6c67002cad444e Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Wed, 14 Jun 2017 12:16:43 +0300
Subject: [PATCH 294/446] IGNITE-5478: Fix for cherry pick

---
 modules/platforms/cpp/odbc-test/src/queries_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 6dc5d4f63cf3c..3a3858802459e 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -1363,7 +1363,7 @@ BOOST_AUTO_TEST_CASE(TestInsertMergeSelect)
 
 BOOST_AUTO_TEST_CASE(TestParamsNum)
 {
-    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
 
     CheckParamsNum("SELECT * FROM TestType", 0);
     CheckParamsNum("SELECT * FROM TestType WHERE _key=?", 1);

From d268b32cb252a5f06887d2b803d27ddc20ded95f Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Fri, 16 Jun 2017 12:27:35 +0300
Subject: [PATCH 295/446] IGNITE-4370: Implemented writing of batch of
 parameters for ODBC.

(cherry picked from commit c10be5780589cc84e7929e234e4411d515166e0b)
---
 .../processors/odbc/OdbcMessageParser.java    |  57 ++-
 .../processors/odbc/OdbcNioListener.java      |   3 +-
 .../odbc/OdbcQueryExecuteBatchRequest.java    |  95 ++++
 .../odbc/OdbcQueryExecuteBatchResult.java     |  75 ++++
 .../odbc/OdbcQueryExecuteRequest.java         |   6 +-
 .../odbc/OdbcQueryExecuteResult.java          |   2 +-
 .../internal/processors/odbc/OdbcRequest.java |   3 +
 .../processors/odbc/OdbcRequestHandler.java   | 106 ++++-
 .../cpp/core-test/src/test_utils.cpp          |   0
 .../src/application_data_buffer_test.cpp      | 146 ++-----
 .../cpp/odbc-test/src/column_test.cpp         |  12 +-
 .../cpp/odbc-test/src/queries_test.cpp        | 386 ++++++++++++++++-
 .../platforms/cpp/odbc-test/src/row_test.cpp  |   9 +-
 modules/platforms/cpp/odbc/Makefile.am        |   3 +
 .../platforms/cpp/odbc/include/Makefile.am    |   2 +
 .../ignite/odbc/app/application_data_buffer.h |  38 +-
 .../odbc/include/ignite/odbc/app/parameter.h  |  20 +-
 .../include/ignite/odbc/app/parameter_set.h   | 268 ++++++++++++
 .../odbc/include/ignite/odbc/common_types.h   |   8 +-
 .../ignite/odbc/diagnostic/diagnosable.h      |   4 +-
 .../cpp/odbc/include/ignite/odbc/message.h    | 408 +++++++-----------
 .../include/ignite/odbc/query/batch_query.h   | 160 +++++++
 .../include/ignite/odbc/query/data_query.h    |  20 +-
 .../odbc/include/ignite/odbc/query/query.h    |   3 +
 .../ignite/odbc/query/type_info_query.h       |   2 +-
 .../cpp/odbc/include/ignite/odbc/statement.h  |  59 ++-
 .../cpp/odbc/project/vs/odbc.vcxproj          |   5 +
 .../cpp/odbc/project/vs/odbc.vcxproj.filters  |  15 +
 .../odbc/src/app/application_data_buffer.cpp  |  94 +++-
 .../platforms/cpp/odbc/src/app/parameter.cpp  |  13 +-
 .../cpp/odbc/src/app/parameter_set.cpp        | 242 +++++++++++
 .../cpp/odbc/src/config/connection_info.cpp   |  68 ++-
 .../odbc/src/diagnostic/diagnostic_record.cpp |  12 +
 modules/platforms/cpp/odbc/src/message.cpp    | 366 ++++++++++++++++
 modules/platforms/cpp/odbc/src/odbc.cpp       |  23 +-
 .../cpp/odbc/src/query/batch_query.cpp        | 197 +++++++++
 .../cpp/odbc/src/query/data_query.cpp         |  15 +-
 modules/platforms/cpp/odbc/src/statement.cpp  | 260 +++++++----
 38 files changed, 2621 insertions(+), 584 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java
 create mode 100644 modules/platforms/cpp/core-test/src/test_utils.cpp
 create mode 100644 modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
 create mode 100644 modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
 create mode 100644 modules/platforms/cpp/odbc/src/app/parameter_set.cpp
 create mode 100644 modules/platforms/cpp/odbc/src/message.cpp
 create mode 100644 modules/platforms/cpp/odbc/src/query/batch_query.cpp

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java
index 7b863d6b2af21..0cb89d71f626b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
 
 import java.util.Collection;
+import org.jetbrains.annotations.NotNull;
 
 /**
  * ODBC message parser.
@@ -99,23 +100,37 @@ public OdbcRequest decode(byte[] msg) {
         }
 
         OdbcRequest res;
-
+        
         switch (cmd) {
             case OdbcRequest.EXECUTE_SQL_QUERY: {
                 String cache = reader.readString();
                 String sql = reader.readString();
-                int argsNum = reader.readInt();
-
-                Object[] params = new Object[argsNum];
+                int paramNum = reader.readInt();
 
-                for (int i = 0; i < argsNum; ++i)
-                    params[i] = reader.readObjectDetached();
+                Object[] params = readParameterRow(reader, paramNum);
 
                 res = new OdbcQueryExecuteRequest(cache, sql, params);
 
                 break;
             }
 
+            case OdbcRequest.QRY_EXEC_BATCH: {
+                String schema = reader.readString();
+                String sql = reader.readString();
+                int paramRowLen = reader.readInt();
+                int rowNum = reader.readInt();
+                boolean last = reader.readBoolean();
+
+                Object[][] params = new Object[rowNum][];
+
+                for (int i = 0; i < rowNum; ++i)
+                    params[i] = readParameterRow(reader, paramRowLen);
+
+                res = new OdbcQueryExecuteBatchRequest(schema, sql, last, params);
+
+                break;
+            }
+
             case OdbcRequest.FETCH_SQL_QUERY: {
                 long queryId = reader.readLong();
                 int pageSize = reader.readInt();
@@ -170,6 +185,21 @@ public OdbcRequest decode(byte[] msg) {
         return res;
     }
 
+    /**
+     * Read row of parameters using reader.
+     * @param reader reader
+     * @param paramNum Number of parameters in a row
+     * @return Parameters array.
+     */
+    @NotNull private static Object[] readParameterRow(BinaryReaderExImpl reader, int paramNum) {
+        Object[] params = new Object[paramNum];
+
+        for (int i = 0; i < paramNum; ++i)
+            params[i] = reader.readObjectDetached();
+
+        return params;
+    }
+
     /**
      * Encode OdbcResponse to byte array.
      *
@@ -218,9 +248,9 @@ else if (res0 instanceof OdbcQueryExecuteResult) {
             OdbcQueryExecuteResult res = (OdbcQueryExecuteResult) res0;
 
             if (log.isDebugEnabled())
-                log.debug("Resulting query ID: " + res.getQueryId());
+                log.debug("Resulting query ID: " + res.queryId());
 
-            writer.writeLong(res.getQueryId());
+            writer.writeLong(res.queryId());
 
             Collection<OdbcColumnMeta> metas = res.getColumnsMetadata();
 
@@ -231,6 +261,17 @@ else if (res0 instanceof OdbcQueryExecuteResult) {
             for (OdbcColumnMeta meta : metas)
                 meta.write(writer);
         }
+        else if (res0 instanceof OdbcQueryExecuteBatchResult) {
+            OdbcQueryExecuteBatchResult res = (OdbcQueryExecuteBatchResult) res0;
+
+            writer.writeBoolean(res.errorMessage() == null);
+            writer.writeLong(res.rowsAffected());
+
+            if (res.errorMessage() != null) {
+                writer.writeLong(res.errorSetIdx());
+                writer.writeString(res.errorMessage());
+            }
+        }
         else if (res0 instanceof OdbcQueryFetchResult) {
             OdbcQueryFetchResult res = (OdbcQueryFetchResult) res0;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java
index 378e5f25ebebc..a68cf888af48b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.internal.util.nio.GridNioServerListenerAdapter;
 import org.apache.ignite.internal.util.nio.GridNioSession;
 import org.apache.ignite.internal.util.nio.GridNioSessionMetaKey;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.concurrent.atomic.AtomicLong;
@@ -100,7 +101,7 @@ public OdbcNioListener(GridKernalContext ctx, GridSpinBusyLock busyLock, int max
             req = parser.decode(msg);
         }
         catch (Exception e) {
-            log.error("Failed to parse message [id=" + reqId + ", err=" + e + ']');
+            U.error(log, "Failed to parse message [id=" + reqId + ']', e);
 
             ses.close();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java
new file mode 100644
index 0000000000000..0ace947126349
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java
@@ -0,0 +1,95 @@
+package org.apache.ignite.internal.processors.odbc;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * ODBC query execute request with the batch of parameters.
+ */
+public class OdbcQueryExecuteBatchRequest extends OdbcRequest {
+    /** Schema. */
+    @GridToStringInclude(sensitive = true)
+    private final String schema;
+
+    /** Sql query. */
+    @GridToStringInclude(sensitive = true)
+    private final String sqlQry;
+
+    /** Last param page flag. */
+    private final boolean last;
+
+    /** Sql query arguments. */
+    @GridToStringExclude
+    private final Object[][] args;
+
+    /**
+     * @param schema Schema.
+     * @param sqlQry SQL query.
+     * @param last Last page flag.
+     * @param args Arguments list.
+     */
+    public OdbcQueryExecuteBatchRequest(@Nullable String schema, String sqlQry, boolean last, Object[][] args) {
+        super(QRY_EXEC_BATCH);
+
+        assert sqlQry != null : "SQL query should not be null";
+        assert args != null : "Parameters should not be null";
+
+        this.schema = schema;
+        this.sqlQry = sqlQry;
+        this.last = last;
+        this.args = args;
+    }
+
+    /**
+     * @return Sql query.
+     */
+    public String sqlQuery() {
+        return sqlQry;
+    }
+
+    /**
+     * @return Sql query arguments.
+     */
+    public Object[][] arguments() {
+        return args;
+    }
+
+    /**
+     * @return Schema.
+     */
+    @Nullable
+    public String schema() {
+        return schema;
+    }
+
+    /**
+     * @return Last page flag.
+     */
+    public boolean last() {
+        return last;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(OdbcQueryExecuteBatchRequest.class, this, "args", args, true);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java
new file mode 100644
index 0000000000000..6fc3873dea267
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.odbc;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * ODBC query execute with batch of parameters result.
+ */
+public class OdbcQueryExecuteBatchResult {
+    /** Rows affected. */
+    private final long rowsAffected;
+
+    /** Index of the set which caused an error. */
+    private final long errorSetIdx;
+
+    /** Error message. */
+    private final String errorMessage;
+
+    /**
+     * @param rowsAffected Number of rows affected by the query.
+     */
+    public OdbcQueryExecuteBatchResult(long rowsAffected) {
+        this.rowsAffected = rowsAffected;
+        this.errorSetIdx = -1;
+        this.errorMessage = null;
+    }
+
+    /**
+     * @param rowsAffected Number of rows affected by the query.
+     * @param errorSetIdx Sets processed.
+     * @param errorMessage Error message.
+     */
+    public OdbcQueryExecuteBatchResult(long rowsAffected, long errorSetIdx, String errorMessage) {
+        this.rowsAffected = rowsAffected;
+        this.errorSetIdx = errorSetIdx;
+        this.errorMessage = errorMessage;
+    }
+
+    /**
+     * @return Number of rows affected by the query.
+     */
+    public long rowsAffected() {
+        return rowsAffected;
+    }
+
+    /**
+     * @return Index of the set which caused an error or -1 if no error occurred.
+     */
+    public long errorSetIdx() {
+        return errorSetIdx;
+    }
+
+    /**
+     * @return Error message.
+     */
+    @Nullable public String errorMessage() {
+        return errorMessage;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
index c0d1c601c92b7..029135cd52e80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
@@ -42,10 +42,12 @@ public class OdbcQueryExecuteRequest extends OdbcRequest {
      * @param sqlQry SQL query.
      * @param args Arguments list.
      */
-    public OdbcQueryExecuteRequest(String cacheName, String sqlQry, Object[] args) {
+    public OdbcQueryExecuteRequest(@Nullable String cacheName, String sqlQry, Object[] args) {
         super(EXECUTE_SQL_QUERY);
 
-        this.cacheName = cacheName.isEmpty() ? null : cacheName;
+        assert sqlQry != null : "SQL query should not be null";
+
+        this.cacheName = cacheName;
         this.sqlQry = sqlQry;
         this.args = args;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java
index a4038123d2d4d..efa432b615614 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java
@@ -41,7 +41,7 @@ public OdbcQueryExecuteResult(long queryId, Collection<OdbcColumnMeta> columnsMe
     /**
      * @return Query ID.
      */
-    public long getQueryId() {
+    public long queryId() {
         return queryId;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java
index f056fedf079bd..3c0c12e09cd28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java
@@ -42,6 +42,9 @@ public class OdbcRequest {
     /** Get parameters meta. */
     public static final int GET_PARAMS_META = 7;
 
+    /** Execute sql query with the batch of parameters. */
+    public static final int QRY_EXEC_BATCH = 8;
+
     /** Command. */
     private final int cmd;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
index c6b41d2d75d36..06430834b2d8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.internal.processors.odbc;
 
 import org.apache.ignite.IgniteCache;
+
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
@@ -36,7 +38,11 @@
 import java.sql.ParameterMetaData;
 import java.sql.PreparedStatement;
 import java.sql.Types;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -104,6 +110,9 @@ public OdbcResponse handle(long reqId, OdbcRequest req) {
                 case HANDSHAKE:
                     return performHandshake(reqId, (OdbcHandshakeRequest)req);
 
+                case QRY_EXEC_BATCH:
+                    return executeBatchQuery((OdbcQueryExecuteBatchRequest)req);
+
                 case EXECUTE_SQL_QUERY:
                     return executeQuery(reqId, (OdbcQueryExecuteRequest)req);
 
@@ -229,6 +238,101 @@ private OdbcResponse executeQuery(long reqId, OdbcQueryExecuteRequest req) {
         }
     }
 
+    /**
+     * {@link OdbcQueryExecuteBatchRequest} command handler.
+     *
+     * @param req Execute query request.
+     * @return Response.
+     */
+    private OdbcResponse executeBatchQuery(OdbcQueryExecuteBatchRequest req) {
+        long rowsAffected = 0;
+        int currentSet = 0;
+
+        try {
+            String sql = OdbcEscapeUtils.parse(req.sqlQuery());
+
+            if (log.isDebugEnabled())
+                log.debug("ODBC query parsed [original=" + req.sqlQuery() + ", parsed=" + sql + ']');
+
+            SqlFieldsQuery qry = new SqlFieldsQuery(sql);
+
+            qry.setDistributedJoins(distributedJoins);
+            qry.setEnforceJoinOrder(enforceJoinOrder);
+
+            IgniteCache<Object, Object> cache0 = ctx.grid().cache(req.schema());
+
+            if (cache0 == null)
+                return new OdbcResponse(OdbcResponse.STATUS_FAILED,
+                        "Cache doesn't exist (did you configure it?): " + req.schema());
+
+            IgniteCache<Object, Object> cache = cache0.withKeepBinary();
+
+            if (cache == null)
+                return new OdbcResponse(OdbcResponse.STATUS_FAILED,
+                        "Can not get cache with keep binary: " + req.schema());
+
+            Object[][] paramSet = req.arguments();
+
+            if (paramSet.length <= 0)
+                throw new IgniteException("Batch execute request with non-positive batch length. [len="
+                        + paramSet.length + ']');
+
+            // Getting meta and do the checks for the first execution.
+            qry.setArgs(paramSet[0]);
+
+            QueryCursorImpl<List<?>> qryCur = (QueryCursorImpl<List<?>>)cache.query(qry);
+
+            if (qryCur.isQuery())
+                throw new IgniteException("Batching of parameters only supported for DML statements. [query=" +
+                        req.sqlQuery() + ']');
+
+            rowsAffected += getRowsAffected(qryCur);
+
+            for (currentSet = 1; currentSet < paramSet.length; ++currentSet)
+            {
+                qry.setArgs(paramSet[currentSet]);
+
+                QueryCursor<List<?>> cur = cache.query(qry);
+
+                rowsAffected += getRowsAffected(cur);
+            }
+
+            OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected);
+
+            return new OdbcResponse(res);
+        }
+        catch (Exception e) {
+            U.error(log, "Failed to execute SQL query [req=" + req + ']', e);
+
+            OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected, currentSet,
+                    e.getMessage());
+
+            return new OdbcResponse(res);
+        }
+    }
+
+    /**
+     * Get affected rows for DML statement.
+     * @param qryCur Cursor.
+     * @return Number of table rows affected.
+     */
+    private static long getRowsAffected(QueryCursor<List<?>> qryCur) {
+        Iterator<List<?>> iter = qryCur.iterator();
+
+        if (iter.hasNext()) {
+            List<?> res = iter.next();
+
+            if (res.size() > 0) {
+                Long affected = (Long) res.get(0);
+
+                if (affected != null)
+                    return affected;
+            }
+        }
+
+        return 0;
+    }
+
     /**
      * {@link OdbcQueryCloseRequest} command handler.
      *
diff --git a/modules/platforms/cpp/core-test/src/test_utils.cpp b/modules/platforms/cpp/core-test/src/test_utils.cpp
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp b/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp
index fe502950a2898..fcc744ac89107 100644
--- a/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp
@@ -44,9 +44,8 @@ BOOST_AUTO_TEST_CASE(TestPutIntToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     appBuf.PutInt8(12);
     BOOST_CHECK(!strcmp(buffer, "12"));
@@ -77,9 +76,8 @@ BOOST_AUTO_TEST_CASE(TestPutFloatToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     appBuf.PutFloat(12.42f);
     BOOST_CHECK(!strcmp(buffer, "12.42"));
@@ -102,9 +100,8 @@ BOOST_AUTO_TEST_CASE(TestPutGuidToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     ignite::Guid guid(0x1da1ef8f39ff4d62ULL, 0x8b72e8e9f3371801ULL);
 
@@ -119,7 +116,7 @@ BOOST_AUTO_TEST_CASE(TestGetGuidFromString)
     char buffer[] = "1da1ef8f-39ff-4d62-8b72-e8e9f3371801";
     SqlLen reslen = sizeof(buffer) - 1;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer) - 1, &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer) - 1, &reslen);
 
     ignite::Guid guid = appBuf.GetGuid();
 
@@ -130,9 +127,8 @@ BOOST_AUTO_TEST_CASE(TestPutBinaryToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     uint8_t binary[] = { 0x21, 0x84, 0xF4, 0xDC, 0x01, 0x00, 0xFF, 0xF0 };
 
@@ -146,9 +142,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     std::string testString("Test string");
 
@@ -162,9 +157,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToWstring)
 {
     wchar_t buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_WCHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_WCHAR, buffer, sizeof(buffer), &reslen);
 
     std::string testString("Test string");
 
@@ -176,9 +170,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToLong)
 {
     long numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutString("424242424");
     BOOST_CHECK(numBuf == 424242424L);
@@ -191,9 +184,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToTiny)
 {
     int8_t numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_TINYINT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_TINYINT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutString("12");
     BOOST_CHECK(numBuf == 12);
@@ -206,9 +198,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToFloat)
 {
     float numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutString("12.21");
     BOOST_CHECK_CLOSE_FRACTION(numBuf, 12.21, FLOAT_PRECISION);
@@ -221,9 +212,8 @@ BOOST_AUTO_TEST_CASE(TestPutIntToFloat)
 {
     float numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutInt8(5);
     BOOST_CHECK_CLOSE_FRACTION(numBuf, 5.0, FLOAT_PRECISION);
@@ -248,9 +238,8 @@ BOOST_AUTO_TEST_CASE(TestPutFloatToShort)
 {
     short numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutDouble(5.42);
     BOOST_CHECK(numBuf == 5);
@@ -270,7 +259,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToDouble)
     double numBuf;
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, sizeof(numBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, sizeof(numBuf), &reslen);
 
     common::Decimal decimal;
 
@@ -299,7 +288,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToLong)
     long numBuf;
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen);
 
     common::Decimal decimal;
 
@@ -326,7 +315,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToString)
     char strBuf[64];
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     common::Decimal decimal;
 
@@ -353,7 +342,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToNumeric)
     SQL_NUMERIC_STRUCT buf;
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_NUMERIC, &buf, sizeof(buf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_NUMERIC, &buf, sizeof(buf), &reslen);
 
     common::Decimal decimal;
 
@@ -404,7 +393,7 @@ BOOST_AUTO_TEST_CASE(TestPutDateToString)
     char strBuf[64] = { 0 };
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     Date date = BinaryUtils::MakeDateGmt(1999, 2, 22);
 
@@ -418,7 +407,7 @@ BOOST_AUTO_TEST_CASE(TestPutTimestampToString)
     char strBuf[64] = { 0 };
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     Timestamp date = BinaryUtils::MakeTimestampGmt(2018, 11, 1, 17, 45, 59);
 
@@ -432,10 +421,7 @@ BOOST_AUTO_TEST_CASE(TestPutDateToDate)
     SQL_DATE_STRUCT buf = { 0 };
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Date date = BinaryUtils::MakeDateGmt(1984, 5, 27);
 
@@ -451,10 +437,7 @@ BOOST_AUTO_TEST_CASE(TestPutTimestampToDate)
     SQL_DATE_STRUCT buf = { 0 };
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = BinaryUtils::MakeTimestampGmt(2004, 8, 14, 6, 34, 51, 573948623);
 
@@ -470,10 +453,7 @@ BOOST_AUTO_TEST_CASE(TestPutTimestampToTimestamp)
     SQL_TIMESTAMP_STRUCT buf = { 0 };
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = BinaryUtils::MakeTimestampGmt(2004, 8, 14, 6, 34, 51, 573948623);
 
@@ -494,10 +474,7 @@ BOOST_AUTO_TEST_CASE(TestPutDateToTimestamp)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Date date = BinaryUtils::MakeDateGmt(1984, 5, 27);
 
@@ -516,9 +493,8 @@ BOOST_AUTO_TEST_CASE(TestGetStringFromLong)
 {
     long numBuf = 42;
     SqlLen reslen = sizeof(numBuf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, reslen, &reslen);
 
     std::string res = appBuf.GetString(32);
 
@@ -535,9 +511,8 @@ BOOST_AUTO_TEST_CASE(TestGetStringFromDouble)
 {
     double numBuf = 43.36;
     SqlLen reslen = sizeof(numBuf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, reslen, &reslen);
 
     std::string res = appBuf.GetString(32);
 
@@ -554,9 +529,8 @@ BOOST_AUTO_TEST_CASE(TestGetStringFromString)
 {
     char buf[] = "Some data 32d2d5hs";
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen);
 
     std::string res = appBuf.GetString(reslen);
 
@@ -567,9 +541,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromUshort)
 {
     unsigned short numBuf = 7162;
     SqlLen reslen = sizeof(numBuf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT, &numBuf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT, &numBuf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -584,9 +557,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromString)
 {
     char buf[] = "28.562";
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -601,9 +573,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromFloat)
 {
     float buf = 207.49f;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -618,9 +589,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromDouble)
 {
     double buf = 893.162;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -635,9 +605,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromString)
 {
     char buf[] = "39";
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -660,9 +629,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromFloat)
 {
     float buf = -107.49f;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -685,9 +653,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromDouble)
 {
     double buf = 42.97f;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -710,9 +677,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromBigint)
 {
     uint64_t buf = 19;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -744,22 +710,19 @@ BOOST_AUTO_TEST_CASE(TestGetIntWithOffset)
         { 42, sizeof(uint64_t) }
     };
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen);
 
     int64_t val = appBuf.GetInt64();
 
     BOOST_CHECK(val == 12);
 
-    offset += sizeof(TestStruct);
+    appBuf.SetByteOffset(sizeof(TestStruct));
 
     val = appBuf.GetInt64();
 
     BOOST_CHECK(val == 42);
 
-    offsetPtr = 0;
+    appBuf.SetByteOffset(0);
 
     val = appBuf.GetInt64();
 
@@ -779,10 +742,7 @@ BOOST_AUTO_TEST_CASE(TestSetStringWithOffset)
         { "", 0 }
     };
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen);
 
     appBuf.PutString("Hello Ignite!");
 
@@ -792,7 +752,7 @@ BOOST_AUTO_TEST_CASE(TestSetStringWithOffset)
     BOOST_CHECK(res == "Hello Ignite!");
     BOOST_CHECK(res.size() == strlen("Hello Ignite!"));
 
-    offset += sizeof(TestStruct);
+    appBuf.SetByteOffset(sizeof(TestStruct));
 
     appBuf.PutString("Hello with offset!");
 
@@ -814,10 +774,7 @@ BOOST_AUTO_TEST_CASE(TestGetDateFromString)
     char buf[] = "1999-02-22";
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen);
 
     Date date = appBuf.GetDate();
 
@@ -840,10 +797,7 @@ BOOST_AUTO_TEST_CASE(TestGetTimestampFromString)
     char buf[] = "2018-11-01 17:45:59";
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen);
 
     Timestamp date = appBuf.GetTimestamp();
 
@@ -871,10 +825,7 @@ BOOST_AUTO_TEST_CASE(TestGetDateFromDate)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Date date = appBuf.GetDate();
 
@@ -902,10 +853,7 @@ BOOST_AUTO_TEST_CASE(TestGetTimestampFromDate)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = appBuf.GetTimestamp();
 
@@ -937,10 +885,7 @@ BOOST_AUTO_TEST_CASE(TestGetTimestampFromTimestamp)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = appBuf.GetTimestamp();
 
@@ -973,10 +918,7 @@ BOOST_AUTO_TEST_CASE(TestGetDateFromTimestamp)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Date date = appBuf.GetDate();
 
diff --git a/modules/platforms/cpp/odbc-test/src/column_test.cpp b/modules/platforms/cpp/odbc-test/src/column_test.cpp
index 6cbea8b648f4b..a5b0d0854103a 100644
--- a/modules/platforms/cpp/odbc-test/src/column_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/column_test.cpp
@@ -66,9 +66,8 @@ BOOST_AUTO_TEST_CASE(TestColumnShort)
 
     short shortBuf = 0;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &shortBuf, sizeof(shortBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &shortBuf, sizeof(shortBuf), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -114,9 +113,8 @@ BOOST_AUTO_TEST_CASE(TestColumnString)
 
     char strBuf[1024] = {};
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -164,9 +162,8 @@ BOOST_AUTO_TEST_CASE(TestColumnStringSeveral)
 
     std::string strBuf(data.size() / 3 + 2, 0);
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf[0], strBuf.size(), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf[0], strBuf.size(), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -246,9 +243,8 @@ BOOST_AUTO_TEST_CASE(TestColumnMultiString)
 
     char strBuf[1024] = {};
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     BOOST_REQUIRE(column1.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 3a3858802459e..9aca77de784ed 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -33,6 +33,7 @@
 #include <boost/test/unit_test.hpp>
 
 #include "ignite/ignite.h"
+#include "ignite/common/fixed_size_array.h"
 #include "ignite/ignition.h"
 #include "ignite/impl/binary/binary_utils.h"
 
@@ -324,10 +325,10 @@ struct QueriesTestSuiteFixture
     }
 
     /**
-     * Insert requested number of TestType vlaues with all defaults except
+     * Insert requested number of TestType values with all defaults except
      * for the strFields, which are generated using getTestString().
      *
-     * @param num Number of records to insert.
+     * @param recordsNum Number of records to insert.
      * @param merge Set to true to use merge instead.
      */
     void InsertTestStrings(int recordsNum, bool merge = false)
@@ -385,6 +386,317 @@ struct QueriesTestSuiteFixture
             BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
     }
 
+    /**
+     * Insert requested number of TestType values in a batch.
+     *
+     * @param from Index to start from.
+     * @param to Index to stop.
+     * @param expectedToAffect Expected number of affected records.
+     * @param merge Set to true to use merge instead of insert.
+     * @return Records inserted.
+     */
+    int InsertTestBatch(int from, int to, int expectedToAffect, bool merge = false)
+    {
+        SQLCHAR insertReq[] = "INSERT "
+            "INTO TestType(_key, i8Field, i16Field, i32Field, strField, floatField, doubleField, boolField, dateField, "
+            "timestampField) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+        SQLCHAR mergeReq[] = "MERGE "
+            "INTO TestType(_key, i8Field, i16Field, i32Field, strField, floatField, doubleField, boolField, dateField, "
+            "timestampField) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+        SQLRETURN ret;
+
+        int recordsNum = to - from;
+
+        ret = SQLPrepare(stmt, merge ? mergeReq : insertReq, SQL_NTS);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        FixedSizeArray<int64_t> keys(recordsNum);
+        FixedSizeArray<int8_t> i8Fields(recordsNum);
+        FixedSizeArray<int16_t> i16Fields(recordsNum);
+        FixedSizeArray<int32_t> i32Fields(recordsNum);
+        FixedSizeArray<char> strFields(recordsNum * 1024);
+        FixedSizeArray<float> floatFields(recordsNum);
+        FixedSizeArray<double> doubleFields(recordsNum);
+        FixedSizeArray<bool> boolFields(recordsNum);
+        FixedSizeArray<SQL_DATE_STRUCT> dateFields(recordsNum);
+        FixedSizeArray<SQL_TIME_STRUCT> timeFields(recordsNum);
+        FixedSizeArray<SQL_TIMESTAMP_STRUCT> timestampFields(recordsNum);
+
+        FixedSizeArray<SQLLEN> strFieldsLen(recordsNum);
+
+        BOOST_CHECKPOINT("Filling param data");
+
+        for (int i = 0; i < recordsNum; ++i)
+        {
+            int seed = from + i;
+
+            keys[i] = seed;
+            i8Fields[i] = seed * 8;
+            i16Fields[i] = seed * 16;
+            i32Fields[i] = seed * 32;
+
+            std::string val = getTestString(seed);
+            strncpy(strFields.GetData() + 1024 * i, val.c_str(), 1023);
+            strFieldsLen[i] = val.size();
+
+            floatFields[i] = seed * 0.5f;
+            doubleFields[i] = seed * 0.25f;
+            boolFields[i] = seed % 2 == 0;
+
+            dateFields[i].year = 2017 + seed / 365;
+            dateFields[i].month = ((seed / 28) % 12) + 1;
+            dateFields[i].day = (seed % 28) + 1;
+
+            timeFields[i].hour = (seed / 3600) % 24;
+            timeFields[i].minute = (seed / 60) % 60;
+            timeFields[i].second = seed % 60;
+
+            timestampFields[i].year = dateFields[i].year;
+            timestampFields[i].month = dateFields[i].month;
+            timestampFields[i].day = dateFields[i].day;
+            timestampFields[i].hour = timeFields[i].hour;
+            timestampFields[i].minute = timeFields[i].minute;
+            timestampFields[i].second = timeFields[i].second;
+            timestampFields[i].fraction = std::abs(seed * 914873) % 1000000000;
+        }
+
+        SQLULEN setsProcessed = 0;
+
+        BOOST_CHECKPOINT("Setting processed pointer");
+        ret = SQLSetStmtAttr(stmt, SQL_ATTR_PARAMS_PROCESSED_PTR, &setsProcessed, SQL_IS_POINTER);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding keys");
+        ret = SQLBindParameter(stmt, 1, SQL_PARAM_INPUT, SQL_C_SBIGINT, SQL_BIGINT, 0, 0, keys.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding i8Fields");
+        ret = SQLBindParameter(stmt, 2, SQL_PARAM_INPUT, SQL_C_STINYINT, SQL_TINYINT, 0, 0, i8Fields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding i16Fields");
+        ret = SQLBindParameter(stmt, 3, SQL_PARAM_INPUT, SQL_C_SSHORT, SQL_SMALLINT, 0, 0, i16Fields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding i32Fields");
+        ret = SQLBindParameter(stmt, 4, SQL_PARAM_INPUT, SQL_C_SLONG, SQL_INTEGER, 0, 0, i32Fields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding strFields");
+        ret = SQLBindParameter(stmt, 5, SQL_PARAM_INPUT, SQL_C_CHAR, SQL_VARCHAR, 1024, 0, strFields.GetData(), 1024, strFieldsLen.GetData());
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding floatFields");
+        ret = SQLBindParameter(stmt, 6, SQL_PARAM_INPUT, SQL_C_FLOAT, SQL_FLOAT, 0, 0, floatFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding doubleFields");
+        ret = SQLBindParameter(stmt, 7, SQL_PARAM_INPUT, SQL_C_DOUBLE, SQL_DOUBLE, 0, 0, doubleFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding boolFields");
+        ret = SQLBindParameter(stmt, 8, SQL_PARAM_INPUT, SQL_C_BIT, SQL_BIT, 0, 0, boolFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding dateFields");
+        ret = SQLBindParameter(stmt, 9, SQL_PARAM_INPUT, SQL_C_DATE, SQL_DATE, 0, 0, dateFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding timestampFields");
+        ret = SQLBindParameter(stmt, 10, SQL_PARAM_INPUT, SQL_C_TIMESTAMP, SQL_TIMESTAMP, 0, 0, timestampFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Setting paramset size");
+        ret = SQLSetStmtAttr(stmt, SQL_ATTR_PARAMSET_SIZE, reinterpret_cast<SQLPOINTER>(recordsNum), 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Executing query");
+        ret = SQLExecute(stmt);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        SQLLEN affected = 0;
+        ret = SQLRowCount(stmt, &affected);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECK_EQUAL(affected, expectedToAffect);
+
+        BOOST_CHECKPOINT("Getting next result set");
+        ret = SQLMoreResults(stmt);
+
+        if (ret != SQL_NO_DATA)
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Resetting parameters.");
+        ret = SQLFreeStmt(stmt, SQL_RESET_PARAMS);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Setting paramset size");
+        ret = SQLSetStmtAttr(stmt, SQL_ATTR_PARAMSET_SIZE, reinterpret_cast<SQLPOINTER>(1), 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        return static_cast<int>(setsProcessed);
+    }
+
+    void InsertBatchSelect(int recordsNum)
+    {
+        Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
+
+        // Inserting values.
+        int inserted = InsertTestBatch(0, recordsNum, recordsNum);
+
+        BOOST_REQUIRE_EQUAL(inserted, recordsNum);
+
+        int64_t key = 0;
+        char strField[1024] = { 0 };
+        SQLLEN strFieldLen = 0;
+
+        // Binding columns.
+        SQLRETURN ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Binding columns.
+        ret = SQLBindCol(stmt, 2, SQL_C_CHAR, &strField, sizeof(strField), &strFieldLen);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Just selecting everything to make sure everything is OK
+        SQLCHAR selectReq[] = "SELECT _key, strField FROM TestType ORDER BY _key";
+
+        ret = SQLExecDirect(stmt, selectReq, sizeof(selectReq));
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        int selectedRecordsNum = 0;
+
+        ret = SQL_SUCCESS;
+
+        while (ret == SQL_SUCCESS)
+        {
+            ret = SQLFetch(stmt);
+
+            if (ret == SQL_NO_DATA)
+                break;
+
+            if (!SQL_SUCCEEDED(ret))
+                BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+            std::string expectedStr = getTestString(selectedRecordsNum);
+            int64_t expectedKey = selectedRecordsNum;
+
+            BOOST_CHECK_EQUAL(key, expectedKey);
+
+            BOOST_CHECK_EQUAL(std::string(strField, strFieldLen), expectedStr);
+
+            ++selectedRecordsNum;
+        }
+
+        BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
+    }
+
+    void InsertNonFullBatchSelect(int recordsNum, int splitAt)
+    {
+        Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
+
+        // Inserting values.
+        int inserted = InsertTestBatch(splitAt, recordsNum, recordsNum - splitAt);
+
+        BOOST_REQUIRE_EQUAL(inserted, recordsNum - splitAt);
+
+        inserted = InsertTestBatch(0, recordsNum, splitAt);
+
+        BOOST_REQUIRE_EQUAL(inserted, splitAt);
+
+        int64_t key = 0;
+        char strField[1024] = { 0 };
+        SQLLEN strFieldLen = 0;
+
+        // Binding columns.
+        SQLRETURN ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Binding columns.
+        ret = SQLBindCol(stmt, 2, SQL_C_CHAR, &strField, sizeof(strField), &strFieldLen);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Just selecting everything to make sure everything is OK
+        SQLCHAR selectReq[] = "SELECT _key, strField FROM TestType ORDER BY _key";
+
+        ret = SQLExecDirect(stmt, selectReq, sizeof(selectReq));
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        int selectedRecordsNum = 0;
+
+        ret = SQL_SUCCESS;
+
+        while (ret == SQL_SUCCESS)
+        {
+            ret = SQLFetch(stmt);
+
+            if (ret == SQL_NO_DATA)
+                break;
+
+            if (!SQL_SUCCEEDED(ret))
+                BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+            std::string expectedStr = getTestString(selectedRecordsNum);
+            int64_t expectedKey = selectedRecordsNum;
+
+            BOOST_CHECK_EQUAL(key, expectedKey);
+
+            BOOST_CHECK_EQUAL(std::string(strField, strFieldLen), expectedStr);
+
+            ++selectedRecordsNum;
+        }
+
+        BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
+    }
+
     /** Node started during the test. */
     Ignite grid;
 
@@ -1361,6 +1673,76 @@ BOOST_AUTO_TEST_CASE(TestInsertMergeSelect)
     BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
 }
 
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2)
+{
+    InsertBatchSelect(2);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect100)
+{
+    InsertBatchSelect(100);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1000)
+{
+    InsertBatchSelect(1000);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1023)
+{
+    InsertBatchSelect(1024);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1024)
+{
+    InsertBatchSelect(1024);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1025)
+{
+    InsertBatchSelect(1025);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2000)
+{
+    InsertBatchSelect(2000);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2047)
+{
+    InsertBatchSelect(2048);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2048)
+{
+    InsertBatchSelect(2048);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2049)
+{
+    InsertBatchSelect(2049);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect900)
+{
+    InsertNonFullBatchSelect(900, 42);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect1500)
+{
+    InsertNonFullBatchSelect(1500, 100);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect4500)
+{
+    InsertNonFullBatchSelect(4500, 1500);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect4096)
+{
+    InsertNonFullBatchSelect(4096, 1024);
+}
+
 BOOST_AUTO_TEST_CASE(TestParamsNum)
 {
     Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
diff --git a/modules/platforms/cpp/odbc-test/src/row_test.cpp b/modules/platforms/cpp/odbc-test/src/row_test.cpp
index f38e9c5b58f1d..1ae34c12feadf 100644
--- a/modules/platforms/cpp/odbc-test/src/row_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/row_test.cpp
@@ -82,12 +82,11 @@ void CheckRowData(Row& row, size_t rowIdx)
     char strBuf[1024];
     SQLGUID guidBuf;
     char bitBuf;
-    int* offset = 0;
 
-    ApplicationDataBuffer appLongBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_LONG, &longBuf, sizeof(longBuf), &reslen, &offset);
-    ApplicationDataBuffer appStrBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, &offset);
-    ApplicationDataBuffer appGuidBuf(type_traits::IGNITE_ODBC_C_TYPE_GUID, &guidBuf, sizeof(guidBuf), &reslen, &offset);
-    ApplicationDataBuffer appBitBuf(type_traits::IGNITE_ODBC_C_TYPE_BIT, &bitBuf, sizeof(bitBuf), &reslen, &offset);
+    ApplicationDataBuffer appLongBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_LONG, &longBuf, sizeof(longBuf), &reslen);
+    ApplicationDataBuffer appStrBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
+    ApplicationDataBuffer appGuidBuf(type_traits::IGNITE_ODBC_C_TYPE_GUID, &guidBuf, sizeof(guidBuf), &reslen);
+    ApplicationDataBuffer appBitBuf(type_traits::IGNITE_ODBC_C_TYPE_BIT, &bitBuf, sizeof(bitBuf), &reslen);
 
     // Checking size.
     BOOST_REQUIRE(row.GetSize() == 4);
diff --git a/modules/platforms/cpp/odbc/Makefile.am b/modules/platforms/cpp/odbc/Makefile.am
index b0cc5f87133fc..c2fb42e19edff 100644
--- a/modules/platforms/cpp/odbc/Makefile.am
+++ b/modules/platforms/cpp/odbc/Makefile.am
@@ -52,6 +52,7 @@ libignite_odbc_la_SOURCES = \
     os/linux/src/system/socket_client.cpp \
     src/app/application_data_buffer.cpp \
     src/app/parameter.cpp \
+    src/app/parameter_set.cpp \
     src/common_types.cpp \
     src/config/configuration.cpp \
     src/config/connection_info.cpp \
@@ -68,6 +69,7 @@ libignite_odbc_la_SOURCES = \
     src/dsn_config.cpp \
     src/query/column_metadata_query.cpp \
     src/query/data_query.cpp \
+    src/query/batch_query.cpp \
     src/query/foreign_keys_query.cpp \
     src/query/primary_keys_query.cpp \
     src/query/table_metadata_query.cpp \
@@ -76,6 +78,7 @@ libignite_odbc_la_SOURCES = \
     src/protocol_version.cpp \
     src/result_page.cpp \
     src/row.cpp \
+    src/message.cpp \
     src/column.cpp \
     src/statement.cpp \
     src/type_traits.cpp \
diff --git a/modules/platforms/cpp/odbc/include/Makefile.am b/modules/platforms/cpp/odbc/include/Makefile.am
index 073dcaa1898ba..1408aee3cb52b 100644
--- a/modules/platforms/cpp/odbc/include/Makefile.am
+++ b/modules/platforms/cpp/odbc/include/Makefile.am
@@ -22,6 +22,7 @@ noinst_HEADERS = \
     ignite/odbc/query/table_metadata_query.h \
     ignite/odbc/query/special_columns_query.h \
     ignite/odbc/query/type_info_query.h \
+    ignite/odbc/query/batch_query.h \
     ignite/odbc/query/data_query.h \
     ignite/odbc/query/foreign_keys_query.h \
     ignite/odbc/query/column_metadata_query.h \
@@ -35,6 +36,7 @@ noinst_HEADERS = \
     ignite/odbc/parser.h \
     ignite/odbc/app/application_data_buffer.h \
     ignite/odbc/app/parameter.h \
+    ignite/odbc/app/parameter_set.h \
     ignite/odbc/row.h \
     ignite/odbc/utility.h \
     ignite/odbc/environment.h \
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h b/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h
index 18ac36aead536..b7989c55982fa 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h
@@ -54,10 +54,9 @@ namespace ignite
                  * @param buffer Data buffer pointer.
                  * @param buflen Data buffer length.
                  * @param reslen Resulting data length.
-                 * @param offset Pointer to buffer and reslen offset pointer.
                  */
                 ApplicationDataBuffer(type_traits::IgniteSqlType type, void* buffer,
-                    SqlLen buflen, SqlLen* reslen, int** offset = 0);
+                    SqlLen buflen, SqlLen* reslen);
 
                 /**
                  * Copy constructor.
@@ -80,13 +79,23 @@ namespace ignite
                 ApplicationDataBuffer& operator=(const ApplicationDataBuffer& other);
 
                 /**
-                 * Set pointer to offset pointer.
+                 * Set offset in bytes for all bound pointers.
                  *
-                 * @param offset Pointer to offset pointer.
+                 * @param offset Offset.
                  */
-                void SetPtrToOffsetPtr(int** offset)
+                void SetByteOffset(int offset)
                 {
-                    this->offset = offset;
+                    this->byteOffset = offset;
+                }
+
+                /**
+                 * Set offset in elements for all bound pointers.
+                 *
+                 * @param
+                 */
+                void SetElementOffset(SqlUlen idx)
+                {
+                    this->elementOffset = idx;
                 }
 
                 /**
@@ -313,6 +322,13 @@ namespace ignite
                  */
                 SqlLen GetDataAtExecSize() const;
 
+                /**
+                 * Get single element size.
+                 *
+                 * @return Size of the single element.
+                 */
+                SqlLen GetElementSize() const;
+
                 /**
                  * Get size of the input buffer.
                  *
@@ -392,10 +408,11 @@ namespace ignite
                  * Apply buffer offset to pointer.
                  * Adds offset to pointer if offset pointer is not null.
                  * @param ptr Pointer.
+                 * @param elemSize Element size.
                  * @return Pointer with applied offset.
                  */
                 template<typename T>
-                T* ApplyOffset(T* ptr) const;
+                T* ApplyOffset(T* ptr, size_t elemSize) const;
 
                 /** Underlying data type. */
                 type_traits::IgniteSqlType type;
@@ -409,8 +426,11 @@ namespace ignite
                 /** Result length. */
                 SqlLen* reslen;
 
-                /** Pointer to implementation pointer to application offset */
-                int** offset;
+                /** Current byte offset */
+                int byteOffset;
+
+                /** Current element offset. */
+                SqlUlen elementOffset;
             };
 
             /** Column binging map type alias. */
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h
index 1cf85b514aa59..cdaaead40ee3d 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h
@@ -22,12 +22,10 @@
 
 #include <map>
 
-#include <ignite/guid.h>
 #include <ignite/impl/binary/binary_writer_impl.h>
 #include <ignite/impl/binary/binary_reader_impl.h>
 
 #include "ignite/odbc/app/application_data_buffer.h"
-#include "ignite/odbc/type_traits.h"
 
 namespace ignite
 {
@@ -70,7 +68,7 @@ namespace ignite
                 ~Parameter();
 
                 /**
-                 * Copy assigment operator.
+                 * Assignment operator.
                  *
                  * @param other Other instance.
                  * @return This.
@@ -78,10 +76,12 @@ namespace ignite
                 Parameter& operator=(const Parameter& other);
 
                 /**
-                 * Write request using provided writer.
+                 * Write parameter using provided writer.
                  * @param writer Writer.
+                 * @param offset Offset for the buffer.
+                 * @param idx Index for the array-of-parameters case.
                  */
-                void Write(impl::binary::BinaryWriterImpl& writer) const;
+                void Write(impl::binary::BinaryWriterImpl& writer, int offset = 0, SqlUlen idx = 0) const;
 
                 /**
                  * Get data buffer.
@@ -90,6 +90,13 @@ namespace ignite
                  */
                 ApplicationDataBuffer& GetBuffer();
 
+                /**
+                 * Get data buffer.
+                 *
+                 * @return underlying ApplicationDataBuffer instance.
+                 */
+                const ApplicationDataBuffer& GetBuffer() const;
+
                 /**
                  * Reset stored at-execution data.
                  */
@@ -128,9 +135,6 @@ namespace ignite
                 /** Stored at-execution data. */
                 std::vector<int8_t> storedData;
             };
-
-            /** Parameter binging map type alias. */
-            typedef std::map<uint16_t, Parameter> ParameterBindingMap;
         }
     }
 }
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
new file mode 100644
index 0000000000000..2ab55808c073c
--- /dev/null
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _IGNITE_ODBC_APP_PARAMETER_SET
+#define _IGNITE_ODBC_APP_PARAMETER_SET
+
+#include <stdint.h>
+
+#include <map>
+
+#include <ignite/impl/binary/binary_writer_impl.h>
+#include <ignite/impl/binary/binary_reader_impl.h>
+
+#include "ignite/odbc/app/parameter.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        namespace app
+        {
+            /**
+             * Parameter set.
+             */
+            class ParameterSet
+            {
+                /** Parameter binging map type alias. */
+                typedef std::map<uint16_t, Parameter> ParameterBindingMap;
+
+                /** Parameter meta vector. */
+                typedef std::vector<int8_t> ParameterTypeVector;
+            public:
+                /**
+                 * Default constructor.
+                 */
+                ParameterSet();
+
+                /**
+                 * Destructor.
+                 */
+                ~ParameterSet()
+                {
+                    // No-op.
+                }
+
+                /**
+                 * Set parameters set size.
+                 *
+                 * @param size Size of the parameter set.
+                 */
+                void SetParamSetSize(SqlUlen size);
+
+                /**
+                 * Bind parameter.
+                 * 
+                 * @param paramIdx Parameter index.
+                 * @param param Parameter.
+                 */
+                void BindParameter(uint16_t paramIdx, const Parameter& param);
+
+                /**
+                 * Unbind specified parameter.
+                 *
+                 * @param paramIdx Parameter index.
+                 */
+                void UnbindParameter(uint16_t paramIdx);
+
+                /**
+                 * Unbind all parameters.
+                 */
+                void UnbindAll();
+
+                /**
+                 * Get number of binded parameters.
+                 *
+                 * @return Number of binded parameters.
+                 */
+                uint16_t GetParametersNumber() const;
+
+                /**
+                 * Set parameter binding offset pointer.
+                 *
+                 * @param ptr Parameter binding offset pointer.
+                 */
+                void SetParamBindOffsetPtr(int* ptr);
+
+                /**
+                 * Get parameter binding offset pointer.
+                 *
+                 * @return Parameter binding offset pointer.
+                 */
+                int* GetParamBindOffsetPtr();
+
+                /**
+                 * Prepare parameters set for statement execution.
+                 */
+                void Prepare();
+
+                /**
+                 * Check if the data at-execution is needed.
+                 *
+                 * @return True if the data at execution is needed.
+                 */
+                bool IsDataAtExecNeeded() const;
+
+                /**
+                 * Update parameter types metadata.
+                 *
+                 * @param meta Types metadata.
+                 */
+                void UpdateParamsTypes(const ParameterTypeVector& meta);
+
+                /**
+                 * Get type id of the parameter.
+                 *
+                 * @param idx Parameter index.
+                 * @param dflt Default value to return if the type can not be found.
+                 * @return Type ID of the parameter or dflt, if the type can not be returned.
+                 */
+                int8_t GetParamType(int16_t idx, int8_t dflt);
+
+                /**
+                 * Get expected parameters number.
+                 * Using metadata. If metadata was not updated returns zero.
+                 *
+                 * @return Expected parameters number.
+                 */
+                uint16_t GetExpectedParamNum();
+
+                /**
+                 * Check if the metadata was set for the parameter set.
+                 * 
+                 * @return True if the metadata was set for the parameter set. 
+                 */
+                bool IsMetadataSet() const;
+
+                /**
+                 * Check if the parameter selected for putting data at-execution.
+                 *
+                 * @return True if the parameter selected for putting data at-execution.
+                 */
+                bool IsParameterSelected() const;
+
+                /**
+                 * Get parameter by index.
+                 *
+                 * @param idx Index.
+                 * @return Parameter or null, if parameter is not bound.
+                 */
+                Parameter* GetParameter(uint16_t idx);
+
+                /**
+                 * Get selected parameter.
+                 *
+                 * @return Parameter or null, if parameter is not bound.
+                 */
+                Parameter* GetSelectedParameter();
+
+                /**
+                 * Internally selects next parameter for putting data at-execution.
+                 *
+                 * @return Parameter if found and null otherwise.
+                 */
+                Parameter* SelectNextParameter();
+
+                /**
+                 * Write only first row of the param set using provided writer.
+                 * @param writer Writer.
+                 */
+                void Write(impl::binary::BinaryWriterImpl& writer) const;
+
+                /**
+                 * Write rows of the param set in interval [begin, end) using provided writer.
+                 * @param writer Writer.
+                 * @param begin Beginng of the interval.
+                 * @param end End of the interval.
+                 * @param last Last page flag.
+                 */
+                void Write(impl::binary::BinaryWriterImpl& writer, SqlUlen begin, SqlUlen end, bool last) const;
+
+                /**
+                 * Calculate row length.
+                 *
+                 * @return Row length.
+                 */
+                int32_t CalculateRowLen() const;
+
+                /**
+                 * Get parameter set size.
+                 *
+                 * @return Number of rows in set.
+                 */
+                int32_t GetParamSetSize() const;
+
+                /**
+                 * Set number of parameters processed in batch.
+                 *
+                 * @param processed Processed.
+                 */
+                void SetParamsProcessed(SqlUlen processed) const;
+
+                /**
+                 * Number of processed params should be written using provided address.
+                 *
+                 * @param ptr Pointer.
+                 */
+                void SetParamsProcessedPtr(SqlUlen* ptr);
+
+                /**
+                 * Get pointer to write number of parameters processed in batch.
+                 *
+                 * @return Pointer to write number of parameters processed in batch.
+                 */
+                SqlUlen* GetParamsProcessedPtr();
+
+            private:
+                /**
+                 * Write single row of the param set using provided writer.
+                 * @param writer Writer.
+                 * @param idx Row index.
+                 */
+                void WriteRow(impl::binary::BinaryWriterImpl& writer, SqlUlen idx) const;
+
+                IGNITE_NO_COPY_ASSIGNMENT(ParameterSet);
+
+                /** Parameters. */
+                ParameterBindingMap parameters;
+
+                /** Parameter types. */
+                ParameterTypeVector paramTypes;
+
+                /** Offset added to pointers to change binding of parameters. */
+                int* paramBindOffset;
+
+                /** Processed parameters. */
+                SqlUlen* processedParamRows;
+
+                /** Parameter set size. */
+                SqlUlen paramSetSize;
+
+                /** Current position in parametor set. */
+                SqlUlen paramSetPos;
+
+                /** Index of the parameter, which is currently being set. */
+                uint16_t currentParamIdx;
+
+                /** Parameter types are set. */
+                bool typesSet;
+            };
+        }
+    }
+}
+
+#endif //_IGNITE_ODBC_APP_PARAMETER_SET
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h b/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
index 517fe4e0be17d..408816a6d8f54 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
@@ -132,7 +132,13 @@ namespace ignite
              * The connection timeout period expired before the data source
              * responded to the request.
              */
-            SQL_STATE_HYT01_CONNECTIOIN_TIMEOUT
+            SQL_STATE_HYT01_CONNECTIOIN_TIMEOUT,
+
+            /** Invalid buffer type. */
+            SQL_STATE_HY003_INVALID_APPLICATION_BUFFER_TYPE,
+
+            /** Invalid parameter type. */
+            SQL_STATE_HY105_INVALID_PARAMETER_TYPE
         };
 
         /**
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h b/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
index 6937fcc6f3196..95e4f548c1f9c 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
@@ -45,14 +45,14 @@ namespace ignite
                  *
                  * @return Diagnostic record.
                  */
-                virtual const diagnostic::DiagnosticRecordStorage& GetDiagnosticRecords() const = 0;
+                virtual const DiagnosticRecordStorage& GetDiagnosticRecords() const = 0;
 
                 /**
                  * Get diagnostic record.
                  *
                  * @return Diagnostic record.
                  */
-                virtual diagnostic::DiagnosticRecordStorage& GetDiagnosticRecords() = 0;
+                virtual DiagnosticRecordStorage& GetDiagnosticRecords() = 0;
 
                 /**
                  * Add new status record.
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
index a2bbd99adce3c..cecdac67f9d36 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
@@ -24,11 +24,11 @@
 #include "ignite/impl/binary/binary_writer_impl.h"
 #include "ignite/impl/binary/binary_reader_impl.h"
 
-#include "ignite/odbc/utility.h"
 #include "ignite/odbc/result_page.h"
+#include "ignite/odbc/protocol_version.h"
 #include "ignite/odbc/meta/column_meta.h"
 #include "ignite/odbc/meta/table_meta.h"
-#include "ignite/odbc/app/parameter.h"
+#include "ignite/odbc/app/parameter_set.h"
 
 namespace ignite
 {
@@ -48,7 +48,9 @@ namespace ignite
 
             REQUEST_TYPE_GET_TABLES_METADATA = 6,
 
-            REQUEST_TYPE_GET_PARAMS_METADATA = 7
+            REQUEST_TYPE_GET_PARAMS_METADATA = 7,
+
+            REQUEST_TYPE_EXECUTE_SQL_QUERY_BATCH = 8
         };
 
         enum ResponseStatus
@@ -71,35 +73,18 @@ namespace ignite
              * @param distributedJoins Distributed joins flag.
              * @param enforceJoinOrder Enforce join order flag.
              */
-            HandshakeRequest(int64_t version, bool distributedJoins, bool enforceJoinOrder) :
-                version(version),
-                distributedJoins(distributedJoins),
-                enforceJoinOrder(enforceJoinOrder)
-            {
-                // No-op.
-            }
+            HandshakeRequest(int64_t version, bool distributedJoins, bool enforceJoinOrder);
 
             /**
              * Destructor.
              */
-            ~HandshakeRequest()
-            {
-                // No-op.
-            }
+            ~HandshakeRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_HANDSHAKE);
-
-                writer.WriteInt64(version);
-
-                writer.WriteBool(distributedJoins);
-                writer.WriteBool(enforceJoinOrder);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Protocol version. */
@@ -123,65 +108,80 @@ namespace ignite
              *
              * @param cache Cache name.
              * @param sql SQL query.
-             * @param argsNum Number of arguments.
+             * @param params Number of arguments.
              */
-            QueryExecuteRequest(const std::string& cache, const std::string& sql,
-                const app::ParameterBindingMap& params) :
-                cache(cache),
-                sql(sql),
-                params(params)
-            {
-                // No-op.
-            }
+            QueryExecuteRequest(const std::string& cache, const std::string& sql, const app::ParameterSet& params);
 
             /**
              * Destructor.
              */
-            ~QueryExecuteRequest()
-            {
-                // No-op.
-            }
+            ~QueryExecuteRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_EXECUTE_SQL_QUERY);
-                utility::WriteString(writer, cache);
-                utility::WriteString(writer, sql);
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
-                writer.WriteInt32(static_cast<int32_t>(params.size()));
+        private:
+            /** Cache name. */
+            std::string cache;
 
-                app::ParameterBindingMap::const_iterator i;
-                uint16_t prev = 0;
+            /** SQL query. */
+            std::string sql;
 
-                for (i = params.begin(); i != params.end(); ++i) {
-                    uint16_t current = i->first;
+            /** Parameters bindings. */
+            const app::ParameterSet& params;
+        };
 
-                    while ((current - prev) > 1) {
-                        writer.WriteNull();
-                        ++prev;
-                    }
+        /**
+         * Query execute batch request.
+         */
+        class QueryExecuteBatchtRequest
+        {
+        public:
+            /**
+             * Constructor.
+             *
+             * @param schema Schema.
+             * @param sql SQL query.
+             * @param params Query arguments.
+             * @param begin Beginng of the interval.
+             * @param end End of the interval.
+             */
+            QueryExecuteBatchtRequest(const std::string& schema, const std::string& sql,
+                                          const app::ParameterSet& params, SqlUlen begin, SqlUlen end, bool last);
 
-                    i->second.Write(writer);
+            /**
+             * Destructor.
+             */
+            ~QueryExecuteBatchtRequest();
 
-                    prev = current;
-                }
-            }
+            /**
+             * Write request using provided writer.
+             * @param writer Writer.
+             */
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
-            /** Cache name. */
-            std::string cache;
+            /** Schema name. */
+            std::string schema;
 
             /** SQL query. */
             std::string sql;
 
             /** Parameters bindings. */
-            const app::ParameterBindingMap& params;
-        };
+            const app::ParameterSet& params;
 
+            /** Beginng of the interval. */
+            SqlUlen begin;
+
+            /** End of the interval. */
+            SqlUlen end;
+
+            /** Last page flag. */
+            bool last;
+        };
 
         /**
          * Query close request.
@@ -194,28 +194,18 @@ namespace ignite
              *
              * @param queryId Query ID.
              */
-            QueryCloseRequest(int64_t queryId) : queryId(queryId)
-            {
-                // No-op.
-            }
+            QueryCloseRequest(int64_t queryId);
 
             /**
              * Destructor.
              */
-            ~QueryCloseRequest()
-            {
-                // No-op.
-            }
+            ~QueryCloseRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_CLOSE_SQL_QUERY);
-                writer.WriteInt64(queryId);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Query ID. */
@@ -234,31 +224,18 @@ namespace ignite
              * @param queryId Query ID.
              * @param pageSize Required page size.
              */
-            QueryFetchRequest(int64_t queryId, int32_t pageSize) :
-                queryId(queryId),
-                pageSize(pageSize)
-            {
-                // No-op.
-            }
+            QueryFetchRequest(int64_t queryId, int32_t pageSize);
 
             /**
              * Destructor.
              */
-            ~QueryFetchRequest()
-            {
-                // No-op.
-            }
+            ~QueryFetchRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_FETCH_SQL_QUERY);
-                writer.WriteInt64(queryId);
-                writer.WriteInt32(pageSize);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Query ID. */
@@ -281,34 +258,18 @@ namespace ignite
              * @param table Table name.
              * @param column Column name.
              */
-            QueryGetColumnsMetaRequest(const std::string& schema, const std::string& table, const std::string& column) :
-                schema(schema),
-                table(table),
-                column(column)
-            {
-                // No-op.
-            }
+            QueryGetColumnsMetaRequest(const std::string& schema, const std::string& table, const std::string& column);
 
             /**
              * Destructor.
              */
-            ~QueryGetColumnsMetaRequest()
-            {
-                // No-op.
-            }
+            ~QueryGetColumnsMetaRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_GET_COLUMNS_METADATA);
-                
-                utility::WriteString(writer, schema);
-                utility::WriteString(writer, table);
-                utility::WriteString(writer, column);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Schema search pattern. */
@@ -336,36 +297,18 @@ namespace ignite
              * @param tableTypes Table types search pattern.
              */
             QueryGetTablesMetaRequest(const std::string& catalog, const std::string& schema,
-                                      const std::string& table, const std::string& tableTypes) :
-                catalog(catalog),
-                schema(schema),
-                table(table),
-                tableTypes(tableTypes)
-            {
-                // No-op.
-            }
+                                      const std::string& table, const std::string& tableTypes);
 
             /**
              * Destructor.
              */
-            ~QueryGetTablesMetaRequest()
-            {
-                // No-op.
-            }
+            ~QueryGetTablesMetaRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_GET_TABLES_METADATA);
-
-                utility::WriteString(writer, catalog);
-                utility::WriteString(writer, schema);
-                utility::WriteString(writer, table);
-                utility::WriteString(writer, tableTypes);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Column search pattern. */
@@ -414,13 +357,7 @@ namespace ignite
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_GET_PARAMS_METADATA);
-
-                utility::WriteString(writer, cacheName);
-                utility::WriteString(writer, sqlQuery);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Cache name. */
@@ -439,33 +376,19 @@ namespace ignite
             /**
              * Constructor.
              */
-            Response() : status(RESPONSE_STATUS_FAILED), error()
-            {
-                // No-op.
-            }
+            Response();
 
             /**
              * Destructor.
              */
-            virtual ~Response()
-            {
-                // No-op.
-            }
+            virtual ~Response();
 
             /**
              * Read response using provided reader.
              * @param reader Reader.
              */
-            void Read(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                status = reader.ReadInt8();
+            void Read(impl::binary::BinaryReaderImpl& reader);
 
-                if (status == RESPONSE_STATUS_SUCCESS)
-                    ReadOnSuccess(reader);
-                else
-                    utility::ReadString(reader, error);;
-            }
-            
             /**
              * Get request processing status.
              * @return Status.
@@ -488,10 +411,7 @@ namespace ignite
             /**
              * Read data if response status is RESPONSE_STATUS_SUCCESS.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl&)
-            {
-                // No-op.
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl&);
 
         private:
             /** Request processing status. */
@@ -510,21 +430,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            HandshakeResponse() :
-                accepted(false),
-                protoVerSince(),
-                currentVer()
-            {
-                // No-op.
-            }
+            HandshakeResponse();
 
             /**
              * Destructor.
              */
-            ~HandshakeResponse()
-            {
-                // No-op.
-            }
+            ~HandshakeResponse();
 
             /**
              * Check if the handshake has been accepted.
@@ -558,16 +469,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                accepted = reader.ReadBool();
-
-                if (!accepted)
-                {
-                    utility::ReadString(reader, protoVerSince);
-                    utility::ReadString(reader, currentVer);
-                }
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Handshake accepted. */
             bool accepted;
@@ -588,18 +490,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryCloseResponse() : queryId(0)
-            {
-                // No-op.
-            }
+            QueryCloseResponse();
 
             /**
              * Destructor.
              */
-            ~QueryCloseResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryCloseResponse();
 
             /**
              * Get query ID.
@@ -615,10 +511,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                queryId = reader.ReadInt64();
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Query ID. */
             int64_t queryId;
@@ -633,18 +526,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryExecuteResponse() : queryId(0), meta()
-            {
-                // No-op.
-            }
+            QueryExecuteResponse();
 
             /**
              * Destructor.
              */
-            ~QueryExecuteResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryExecuteResponse();
 
             /**
              * Get query ID.
@@ -669,12 +556,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                queryId = reader.ReadInt64();
-
-                meta::ReadColumnMetaVector(reader, meta);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Query ID. */
             int64_t queryId;
@@ -684,28 +566,82 @@ namespace ignite
         };
 
         /**
-         * Query fetch response.
+         * Query execute batch start response.
          */
-        class QueryFetchResponse : public Response
+        class QueryExecuteBatchResponse : public Response
         {
         public:
             /**
              * Constructor.
-             * @param resultPage Result page.
              */
-            QueryFetchResponse(ResultPage& resultPage) : queryId(0), resultPage(resultPage)
+            QueryExecuteBatchResponse();
+
+            /**
+             * Destructor.
+             */
+            virtual ~QueryExecuteBatchResponse();
+
+            /**
+             * Affected rows.
+             * @return Affected rows.
+             */
+            int64_t GetAffectedRows() const
             {
-                // No-op.
+                return affectedRows;
             }
 
             /**
-             * Destructor.
+             * Get index of the set which caused an error.
+             * @return Index of the set which caused an error.
              */
-            ~QueryFetchResponse()
+            int64_t GetErrorSetIdx() const
             {
-                // No-op.
+                return affectedRows;
+            }
+
+            /**
+             * Get error message.
+             * @return Error message.
+             */
+            const std::string& GetErrorMessage() const
+            {
+                return errorMessage;
             }
 
+        private:
+            /**
+             * Read response using provided reader.
+             * @param reader Reader.
+             */
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
+
+            /** Affected rows. */
+            int64_t affectedRows;
+
+            /** Index of the set which caused an error. */
+            int64_t errorSetIdx;
+
+            /** Error message. */
+            std::string errorMessage;
+        };
+
+        /**
+         * Query fetch response.
+         */
+        class QueryFetchResponse : public Response
+        {
+        public:
+            /**
+             * Constructor.
+             * @param resultPage Result page.
+             */
+            QueryFetchResponse(ResultPage& resultPage);
+
+            /**
+             * Destructor.
+             */
+            virtual ~QueryFetchResponse();
+
             /**
              * Get query ID.
              * @return Query ID.
@@ -720,12 +656,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                queryId = reader.ReadInt64();
-
-                resultPage.Read(reader);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Query ID. */
             int64_t queryId;
@@ -743,18 +674,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryGetColumnsMetaResponse()
-            {
-                // No-op.
-            }
+            QueryGetColumnsMetaResponse();
 
             /**
              * Destructor.
              */
-            ~QueryGetColumnsMetaResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryGetColumnsMetaResponse();
 
             /**
              * Get column metadata.
@@ -770,10 +695,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                meta::ReadColumnMetaVector(reader, meta);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Columns metadata. */
             meta::ColumnMetaVector meta;
@@ -788,18 +710,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryGetTablesMetaResponse()
-            {
-                // No-op.
-            }
+            QueryGetTablesMetaResponse();
 
             /**
              * Destructor.
              */
-            ~QueryGetTablesMetaResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryGetTablesMetaResponse();
 
             /**
              * Get column metadata.
@@ -815,10 +731,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                meta::ReadTableMetaVector(reader, meta);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Columns metadata. */
             meta::TableMetaVector meta;
@@ -833,18 +746,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryGetParamsMetaResponse()
-            {
-                // No-op.
-            }
+            QueryGetParamsMetaResponse();
 
             /**
              * Destructor.
              */
-            ~QueryGetParamsMetaResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryGetParamsMetaResponse();
 
             /**
              * Get parameter type IDs.
@@ -860,10 +767,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                utility::ReadByteArray(reader, typeIds);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Columns metadata. */
             std::vector<int8_t> typeIds;
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
new file mode 100644
index 0000000000000..a691f7351d956
--- /dev/null
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _IGNITE_ODBC_QUERY_BATCH_QUERY
+#define _IGNITE_ODBC_QUERY_BATCH_QUERY
+
+#include "ignite/odbc/query/query.h"
+#include "ignite/odbc/app/parameter_set.h"
+#include "ignite/odbc/cursor.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        /** Connection forward-declaration. */
+        class Connection;
+
+        namespace query
+        {
+            /**
+             * Query.
+             */
+            class BatchQuery : public Query
+            {
+            public:
+                /**
+                 * Constructor.
+                 *
+                 * @param diag Diagnostics collector.
+                 * @param connection Associated connection.
+                 * @param sql SQL query string.
+                 * @param params SQL params.
+                 */
+                BatchQuery(diagnostic::Diagnosable& diag, Connection& connection,
+                    const std::string& sql, const app::ParameterSet& params);
+
+                /**
+                 * Destructor.
+                 */
+                virtual ~BatchQuery();
+
+                /**
+                 * Execute query.
+                 *
+                 * @return True on success.
+                 */
+                virtual SqlResult Execute();
+
+                /**
+                 * Get column metadata.
+                 *
+                 * @return Column metadata.
+                 */
+                virtual const meta::ColumnMetaVector& GetMeta() const;
+
+                /**
+                 * Fetch next result row to application buffers.
+                 *
+                 * @param columnBindings Application buffers to put data to.
+                 * @return Operation result.
+                 */
+                virtual SqlResult FetchNextRow(app::ColumnBindingMap& columnBindings);
+                
+                /**
+                 * Get data of the specified column in the result set.
+                 *
+                 * @param columnIdx Column index.
+                 * @param buffer Buffer to put column data to.
+                 * @return Operation result.
+                 */
+                virtual SqlResult GetColumn(uint16_t columnIdx, app::ApplicationDataBuffer& buffer);
+
+                /**
+                 * Close query.
+                 *
+                 * @return Result.
+                 */
+                virtual SqlResult Close();
+
+                /**
+                 * Check if data is available.
+                 *
+                 * @return True if data is available.
+                 */
+                virtual bool DataAvailable() const;
+
+                /**
+                 * Get number of rows affected by the statement.
+                 *
+                 * @return Number of rows affected by the statement.
+                 */
+                virtual int64_t AffectedRows() const;
+
+                /**
+                 * Get SQL query string.
+                 *
+                 * @return SQL query string.
+                 */
+                const std::string& GetSql() const
+                {
+                    return sql;
+                }
+
+            private:
+                IGNITE_NO_COPY_ASSIGNMENT(BatchQuery);
+
+                /**
+                 * Make query execute request and use response to set internal
+                 * state.
+                 *
+                 * @param begin Paramset interval beginning.
+                 * @param end Paramset interval end.
+                 * @param last Last page flag.
+                 * @return Result.
+                 */
+                SqlResult MakeRequestExecuteBatch(SqlUlen begin, SqlUlen end, bool last);
+
+                /** Connection associated with the statement. */
+                Connection& connection;
+
+                /** SQL Query. */
+                std::string sql;
+
+                /** Parameter bindings. */
+                const app::ParameterSet& params;
+
+                /** Columns metadata. */
+                meta::ColumnMetaVector resultMeta;
+
+                /** Number of rows affected. */
+                int64_t rowsAffected;
+
+                /** Number of parameter sets successfully processed. */
+                int64_t setsProcessed;
+
+                /** Query executed. */
+                bool executed;
+
+                /** Data retrieved. */
+                bool dataRetrieved;
+            };
+        }
+    }
+}
+
+#endif //_IGNITE_ODBC_QUERY_BATCH_QUERY
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
index 68bb8776417c8..0424bf8364a6d 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
@@ -19,7 +19,7 @@
 #define _IGNITE_ODBC_QUERY_DATA_QUERY
 
 #include "ignite/odbc/query/query.h"
-#include "ignite/odbc/app/parameter.h"
+#include "ignite/odbc/app/parameter_set.h"
 #include "ignite/odbc/cursor.h"
 
 namespace ignite
@@ -46,7 +46,7 @@ namespace ignite
                  * @param params SQL params.
                  */
                 DataQuery(diagnostic::Diagnosable& diag, Connection& connection,
-                    const std::string& sql, const app::ParameterBindingMap& params);
+                    const std::string& sql, const app::ParameterSet& params);
 
                 /**
                  * Destructor.
@@ -122,24 +122,32 @@ namespace ignite
                  * Make query execute request and use response to set internal
                  * state.
                  *
-                 * @return True on success.
+                 * @return Result.
                  */
                 SqlResult MakeRequestExecute();
 
                 /**
                  * Make query close request.
                  *
-                 * @return True on success.
+                 * @return Result.
                  */
                 SqlResult MakeRequestClose();
 
                 /**
                  * Make data fetch request and use response to set internal state.
                  *
-                 * @return True on success.
+                 * @return Result.
                  */
                 SqlResult MakeRequestFetch();
 
+                /**
+                 * Close query.
+                 * Non-virtual implementation.
+                 *
+                 * @return True on success.
+                 */
+                SqlResult InternalClose();
+
                 /** Connection associated with the statement. */
                 Connection& connection;
 
@@ -147,7 +155,7 @@ namespace ignite
                 std::string sql;
 
                 /** Parameter bindings. */
-                const app::ParameterBindingMap& params;
+                const app::ParameterSet& params;
 
                 /** Columns metadata. */
                 meta::ColumnMetaVector resultMeta;
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
index 40be1ed4c8bd6..d8103af182a30 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
@@ -47,6 +47,9 @@ namespace ignite
                     /** Data query type. */
                     DATA,
 
+                    /** Batch query type. */
+                    BATCH,
+
                     /** Foreign keys query type. */
                     FOREIGN_KEYS,
 
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
index d337d0311b972..ee37bb012bb33 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
@@ -95,7 +95,7 @@ namespace ignite
                  * @return Number of rows affected by the statement.
                  */
                 virtual int64_t AffectedRows() const;
-                
+
             private:
                 IGNITE_NO_COPY_ASSIGNMENT(TypeInfoQuery);
 
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
index 1ee56197ce81b..170f59998e461 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
@@ -23,19 +23,12 @@
 #include <map>
 #include <memory>
 
-#include <ignite/impl/interop/interop_output_stream.h>
-#include <ignite/impl/interop/interop_input_stream.h>
-#include <ignite/impl/binary/binary_writer_impl.h>
-
 #include "ignite/odbc/meta/column_meta.h"
-#include "ignite/odbc/meta/table_meta.h"
 #include "ignite/odbc/query/query.h"
 #include "ignite/odbc/app/application_data_buffer.h"
-#include "ignite/odbc/app/parameter.h"
+#include "ignite/odbc/app/parameter_set.h"
 #include "ignite/odbc/diagnostic/diagnosable_adapter.h"
 #include "ignite/odbc/common_types.h"
-#include "ignite/odbc/cursor.h"
-#include "ignite/odbc/utility.h"
 
 namespace ignite
 {
@@ -96,14 +89,23 @@ namespace ignite
              * @return Columns number.
              */
             int32_t GetColumnNumber();
-
+            
             /**
              * Bind parameter.
              *
              * @param paramIdx Parameter index.
-             * @param param Parameter.
+             * @param ioType Type of the parameter (input/output).
+             * @param bufferType The data type of the parameter.
+             * @param paramSqlType The SQL data type of the parameter.
+             * @param columnSize  The size of the column or expression of the corresponding parameter marker.
+             * @param decDigits  The decimal digits of the column or expression of the corresponding parameter marker.
+             * @param buffer A pointer to a buffer for the parameter's data.
+             * @param bufferLen Length of the ParameterValuePtr buffer in bytes.
+             * @param resLen A pointer to a buffer for the parameter's length.
+             * @return Operation result.
              */
-            void BindParameter(uint16_t paramIdx, const app::Parameter& param);
+            void BindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType, int16_t paramSqlType,
+                SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen);
 
             /**
              * Unbind specified parameter.
@@ -150,13 +152,6 @@ namespace ignite
              */
             void SetParamBindOffsetPtr(int* ptr);
 
-            /**
-             * Get parameter binding offset pointer.
-             *
-             * @return Parameter binding offset pointer.
-             */
-            int* GetParamBindOffsetPtr();
-
             /**
              * Get value of the column in the result set.
              *
@@ -171,7 +166,7 @@ namespace ignite
              * @param query SQL query.
              */
             void PrepareSqlQuery(const std::string& query);
-            
+
             /**
              * Execute SQL query.
              *
@@ -366,10 +361,18 @@ namespace ignite
              * Bind parameter.
              *
              * @param paramIdx Parameter index.
-             * @param param Parameter.
+             * @param ioType Type of the parameter (input/output).
+             * @param bufferType The data type of the parameter.
+             * @param paramSqlType The SQL data type of the parameter.
+             * @param columnSize  The size of the column or expression of the corresponding parameter marker.
+             * @param decDigits  The decimal digits of the column or expression of the corresponding parameter marker.
+             * @param buffer A pointer to a buffer for the parameter's data.
+             * @param bufferLen Length of the ParameterValuePtr buffer in bytes.
+             * @param resLen A pointer to a buffer for the parameter's length.
              * @return Operation result.
              */
-            SqlResult InternalBindParameter(uint16_t paramIdx, const app::Parameter& param);
+            SqlResult InternalBindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType, int16_t paramSqlType,
+                SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen);
 
             /**
              * Set statement attribute.
@@ -425,7 +428,7 @@ namespace ignite
              * @return Operation result.
              */
             SqlResult InternalPrepareSqlQuery(const std::string& query);
-            
+
             /**
              * Execute SQL query.
              *
@@ -611,12 +614,6 @@ namespace ignite
             /** Column bindings. */
             app::ColumnBindingMap columnBindings;
 
-            /** Parameter bindings. */
-            app::ParameterBindingMap paramBindings;
-
-            /** Parameter meta. */
-            std::vector<int8_t> paramTypes;
-
             /** Underlying query. */
             std::auto_ptr<query::Query> currentQuery;
 
@@ -626,14 +623,10 @@ namespace ignite
             /** Array to store statuses of rows fetched by the last fetch. */
             uint16_t* rowStatuses;
 
-            /** Offset added to pointers to change binding of parameters. */
-            int* paramBindOffset;
-            
             /** Offset added to pointers to change binding of column data. */
             int* columnBindOffset;
 
-            /** Index of the parameter, which is currently being set. */
-            uint16_t currentParamIdx;
+            app::ParameterSet parameters;
         };
     }
 }
diff --git a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
index 56358c5f8bf4f..280665984e50f 100644
--- a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
+++ b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
@@ -160,6 +160,7 @@
     <ClCompile Include="..\..\os\win\src\system\ui\window.cpp" />
     <ClCompile Include="..\..\src\app\application_data_buffer.cpp" />
     <ClCompile Include="..\..\src\app\parameter.cpp" />
+    <ClCompile Include="..\..\src\app\parameter_set.cpp" />
     <ClCompile Include="..\..\src\column.cpp" />
     <ClCompile Include="..\..\src\common_types.cpp" />
     <ClCompile Include="..\..\src\config\configuration.cpp" />
@@ -172,10 +173,12 @@
     <ClCompile Include="..\..\src\dsn_config.cpp" />
     <ClCompile Include="..\..\src\entry_points.cpp" />
     <ClCompile Include="..\..\src\environment.cpp" />
+    <ClCompile Include="..\..\src\message.cpp" />
     <ClCompile Include="..\..\src\meta\column_meta.cpp" />
     <ClCompile Include="..\..\src\meta\table_meta.cpp" />
     <ClCompile Include="..\..\src\odbc.cpp" />
     <ClCompile Include="..\..\src\protocol_version.cpp" />
+    <ClCompile Include="..\..\src\query\batch_query.cpp" />
     <ClCompile Include="..\..\src\query\data_query.cpp" />
     <ClCompile Include="..\..\src\query\column_metadata_query.cpp" />
     <ClCompile Include="..\..\src\query\foreign_keys_query.cpp" />
@@ -196,6 +199,7 @@
     <ClInclude Include="..\..\include\ignite\odbc.h" />
     <ClInclude Include="..\..\include\ignite\odbc\app\application_data_buffer.h" />
     <ClInclude Include="..\..\include\ignite\odbc\app\parameter.h" />
+    <ClInclude Include="..\..\include\ignite\odbc\app\parameter_set.h" />
     <ClInclude Include="..\..\include\ignite\odbc\column.h" />
     <ClInclude Include="..\..\include\ignite\odbc\common_types.h" />
     <ClInclude Include="..\..\include\ignite\odbc\config\configuration.h" />
@@ -214,6 +218,7 @@
     <ClInclude Include="..\..\include\ignite\odbc\meta\table_meta.h" />
     <ClInclude Include="..\..\include\ignite\odbc\parser.h" />
     <ClInclude Include="..\..\include\ignite\odbc\protocol_version.h" />
+    <ClInclude Include="..\..\include\ignite\odbc\query\batch_query.h" />
     <ClInclude Include="..\..\include\ignite\odbc\query\data_query.h" />
     <ClInclude Include="..\..\include\ignite\odbc\query\column_metadata_query.h" />
     <ClInclude Include="..\..\include\ignite\odbc\query\foreign_keys_query.h" />
diff --git a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
index 58764e4043871..9aefc0c616ea1 100644
--- a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
+++ b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
@@ -136,6 +136,15 @@
     <ClCompile Include="..\..\src\protocol_version.cpp">
       <Filter>Code</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\app\parameter_set.cpp">
+      <Filter>Code\app</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\query\batch_query.cpp">
+      <Filter>Code\query</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\message.cpp">
+      <Filter>Code</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="module.def">
@@ -260,5 +269,11 @@
     <ClInclude Include="..\..\include\ignite\odbc\protocol_version.h">
       <Filter>Code</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\ignite\odbc\app\parameter_set.h">
+      <Filter>Code\app</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\include\ignite\odbc\query\batch_query.h">
+      <Filter>Code\query</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp b/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp
index 71c5f39025796..c13857e815c51 100644
--- a/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp
+++ b/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp
@@ -33,35 +33,38 @@ namespace ignite
     {
         namespace app
         {
-            using ignite::impl::binary::BinaryUtils;
+            using impl::binary::BinaryUtils;
 
             ApplicationDataBuffer::ApplicationDataBuffer() :
                 type(type_traits::IGNITE_ODBC_C_TYPE_UNSUPPORTED),
                 buffer(0),
                 buflen(0),
                 reslen(0),
-                offset(0)
+                byteOffset(0),
+                elementOffset(0)
             {
                 // No-op.
             }
 
-            ApplicationDataBuffer::ApplicationDataBuffer(type_traits::IgniteSqlType type, 
-                void* buffer, SqlLen buflen, SqlLen* reslen, int** offset) :
+            ApplicationDataBuffer::ApplicationDataBuffer(type_traits::IgniteSqlType type,
+                void* buffer, SqlLen buflen, SqlLen* reslen) :
                 type(type),
                 buffer(buffer),
                 buflen(buflen),
                 reslen(reslen),
-                offset(offset)
+                byteOffset(0),
+                elementOffset(0)
             {
                 // No-op.
             }
 
-            ApplicationDataBuffer::ApplicationDataBuffer(const ApplicationDataBuffer & other) :
+            ApplicationDataBuffer::ApplicationDataBuffer(const ApplicationDataBuffer& other) :
                 type(other.type),
                 buffer(other.buffer),
                 buflen(other.buflen),
                 reslen(other.reslen),
-                offset(other.offset)
+                byteOffset(other.byteOffset),
+                elementOffset(other.elementOffset)
             {
                 // No-op.
             }
@@ -77,7 +80,8 @@ namespace ignite
                 buffer = other.buffer;
                 buflen = other.buflen;
                 reslen = other.reslen;
-                offset = other.offset;
+                byteOffset = other.byteOffset;
+                elementOffset = other.elementOffset;
 
                 return *this;
             }
@@ -1072,22 +1076,22 @@ namespace ignite
 
             const void* ApplicationDataBuffer::GetData() const
             {
-                return ApplyOffset(buffer);
+                return ApplyOffset(buffer, GetElementSize());
             }
 
             const SqlLen* ApplicationDataBuffer::GetResLen() const
             {
-                return ApplyOffset(reslen);
+                return ApplyOffset(reslen, sizeof(*reslen));
             }
 
-            void* ApplicationDataBuffer::GetData() 
+            void* ApplicationDataBuffer::GetData()
             {
-                return ApplyOffset(buffer);
+                return ApplyOffset(buffer, GetElementSize());
             }
 
             SqlLen* ApplicationDataBuffer::GetResLen()
             {
-                return ApplyOffset(reslen);
+                return ApplyOffset(reslen, sizeof(*reslen));
             }
 
             template<typename T>
@@ -1407,12 +1411,12 @@ namespace ignite
             }
 
             template<typename T>
-            T* ApplicationDataBuffer::ApplyOffset(T* ptr) const
+            T* ApplicationDataBuffer::ApplyOffset(T* ptr, size_t elemSize) const
             {
-                if (!ptr || !offset || !*offset)
+                if (!ptr)
                     return ptr;
 
-                return utility::GetPointerWithOffset(ptr, **offset);
+                return utility::GetPointerWithOffset(ptr, byteOffset + elemSize * elementOffset);
             }
 
             bool ApplicationDataBuffer::IsDataAtExec() const
@@ -1502,6 +1506,64 @@ namespace ignite
                 return 0;
             }
 
+            SqlLen ApplicationDataBuffer::GetElementSize() const
+            {
+                using namespace type_traits;
+
+                switch (type)
+                {
+                    case IGNITE_ODBC_C_TYPE_WCHAR:
+                    case IGNITE_ODBC_C_TYPE_CHAR:
+                    case IGNITE_ODBC_C_TYPE_BINARY:
+                        return buflen;
+
+                    case IGNITE_ODBC_C_TYPE_SIGNED_SHORT:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT:
+                        return static_cast<SqlLen>(sizeof(short));
+
+                    case IGNITE_ODBC_C_TYPE_SIGNED_LONG:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_LONG:
+                        return static_cast<SqlLen>(sizeof(long));
+
+                    case IGNITE_ODBC_C_TYPE_FLOAT:
+                        return static_cast<SqlLen>(sizeof(float));
+
+                    case IGNITE_ODBC_C_TYPE_DOUBLE:
+                        return static_cast<SqlLen>(sizeof(double));
+
+                    case IGNITE_ODBC_C_TYPE_BIT:
+                    case IGNITE_ODBC_C_TYPE_SIGNED_TINYINT:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_TINYINT:
+                        return static_cast<SqlLen>(sizeof(char));
+
+                    case IGNITE_ODBC_C_TYPE_SIGNED_BIGINT:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT:
+                        return static_cast<SqlLen>(sizeof(SQLBIGINT));
+
+                    case IGNITE_ODBC_C_TYPE_TDATE:
+                        return static_cast<SqlLen>(sizeof(SQL_DATE_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_TTIME:
+                        return static_cast<SqlLen>(sizeof(SQL_TIME_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_TTIMESTAMP:
+                        return static_cast<SqlLen>(sizeof(SQL_TIMESTAMP_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_NUMERIC:
+                        return static_cast<SqlLen>(sizeof(SQL_NUMERIC_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_GUID:
+                        return static_cast<SqlLen>(sizeof(SQLGUID));
+
+                    case IGNITE_ODBC_C_TYPE_DEFAULT:
+                    case IGNITE_ODBC_C_TYPE_UNSUPPORTED:
+                    default:
+                        break;
+                }
+
+                return 0;
+            }
+
             SqlLen ApplicationDataBuffer::GetInputSize() const
             {
                 if (!IsDataAtExec())
diff --git a/modules/platforms/cpp/odbc/src/app/parameter.cpp b/modules/platforms/cpp/odbc/src/app/parameter.cpp
index 937ef58e7c6c3..5ee132c1b7107 100644
--- a/modules/platforms/cpp/odbc/src/app/parameter.cpp
+++ b/modules/platforms/cpp/odbc/src/app/parameter.cpp
@@ -16,8 +16,6 @@
  */
 
 #include <algorithm>
-#include <string>
-#include <sstream>
 
 #include "ignite/odbc/system/odbc_constants.h"
 #include "ignite/odbc/app/parameter.h"
@@ -78,7 +76,7 @@ namespace ignite
                 return *this;
             }
 
-            void Parameter::Write(ignite::impl::binary::BinaryWriterImpl& writer) const
+            void Parameter::Write(impl::binary::BinaryWriterImpl& writer, int offset, SqlUlen idx) const
             {
                 if (buffer.GetInputSize() == SQL_NULL_DATA)
                 {
@@ -89,6 +87,8 @@ namespace ignite
 
                 // Buffer to use to get data.
                 ApplicationDataBuffer buf(buffer);
+                buf.SetByteOffset(offset);
+                buf.SetElementOffset(idx);
 
                 SqlLen storedDataLen = static_cast<SqlLen>(storedData.size());
 
@@ -150,12 +150,14 @@ namespace ignite
                         break;
                     }
 
+                    case SQL_TYPE_DATE:
                     case SQL_DATE:
                     {
                         writer.WriteDate(buf.GetDate());
                         break;
                     }
 
+                    case SQL_TYPE_TIMESTAMP:
                     case SQL_TIMESTAMP:
                     {
                         writer.WriteTimestamp(buf.GetTimestamp());
@@ -207,6 +209,11 @@ namespace ignite
                 return buffer;
             }
 
+            const ApplicationDataBuffer& Parameter::GetBuffer() const
+            {
+                return buffer;
+            }
+
             void Parameter::ResetStoredData()
             {
                 storedData.clear();
diff --git a/modules/platforms/cpp/odbc/src/app/parameter_set.cpp b/modules/platforms/cpp/odbc/src/app/parameter_set.cpp
new file mode 100644
index 0000000000000..c110d05dc5b38
--- /dev/null
+++ b/modules/platforms/cpp/odbc/src/app/parameter_set.cpp
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ignite/odbc/app/parameter_set.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        namespace app
+        {
+            ParameterSet::ParameterSet():
+                parameters(),
+                paramTypes(),
+                paramBindOffset(0),
+                processedParamRows(0),
+                paramSetSize(1),
+                paramSetPos(0),
+                currentParamIdx(0),
+                typesSet(false)
+            {
+                // No-op.
+            }
+
+            void ParameterSet::SetParamSetSize(SqlUlen size)
+            {
+                paramSetSize = size;
+            }
+
+            void ParameterSet::BindParameter(uint16_t paramIdx, const Parameter& param)
+            {
+                parameters[paramIdx] = param;
+            }
+
+            void ParameterSet::UnbindParameter(uint16_t paramIdx)
+            {
+                parameters.erase(paramIdx);
+            }
+
+            void ParameterSet::UnbindAll()
+            {
+                parameters.clear();
+            }
+
+            uint16_t ParameterSet::GetParametersNumber() const
+            {
+                return static_cast<uint16_t>(parameters.size());
+            }
+
+            void ParameterSet::SetParamBindOffsetPtr(int* ptr)
+            {
+                paramBindOffset = ptr;
+            }
+
+            int* ParameterSet::GetParamBindOffsetPtr()
+            {
+                return paramBindOffset;
+            }
+
+            void ParameterSet::Prepare()
+            {
+                paramTypes.clear();
+
+                typesSet = false;
+
+                paramSetPos = 0;
+
+                for (ParameterBindingMap::iterator it = parameters.begin(); it != parameters.end(); ++it)
+                    it->second.ResetStoredData();
+            }
+
+            bool ParameterSet::IsDataAtExecNeeded() const
+            {
+                for (ParameterBindingMap::const_iterator it = parameters.begin(); it != parameters.end(); ++it)
+                {
+                    if (!it->second.IsDataReady())
+                        return true;
+                }
+
+                return false;
+            }
+
+            void ParameterSet::SetParamsProcessedPtr(SqlUlen* ptr)
+            {
+                processedParamRows = ptr;
+            }
+
+            SqlUlen* ParameterSet::GetParamsProcessedPtr()
+            {
+                return processedParamRows;
+            }
+
+            void ParameterSet::SetParamsProcessed(SqlUlen processed) const
+            {
+                if (processedParamRows)
+                    *processedParamRows = processed;
+            }
+
+            void ParameterSet::UpdateParamsTypes(const ParameterTypeVector& meta)
+            {
+                paramTypes = meta;
+
+                typesSet = true;
+            }
+
+            int8_t ParameterSet::GetParamType(int16_t idx, int8_t dflt)
+            {
+                if (idx > 0 && static_cast<size_t>(idx) <= paramTypes.size())
+                    return paramTypes[idx - 1];
+
+                return dflt;
+            }
+
+            uint16_t ParameterSet::GetExpectedParamNum()
+            {
+                return static_cast<uint16_t>(paramTypes.size());
+            }
+
+            bool ParameterSet::IsMetadataSet() const
+            {
+                return typesSet;
+            }
+
+            bool ParameterSet::IsParameterSelected() const
+            {
+                return currentParamIdx != 0;
+            }
+
+            Parameter* ParameterSet::GetParameter(uint16_t idx)
+            {
+                ParameterBindingMap::iterator it = parameters.find(currentParamIdx);
+
+                if (it != parameters.end())
+                    return &it->second;
+
+                return 0;
+            }
+
+            Parameter* ParameterSet::GetSelectedParameter()
+            {
+                return GetParameter(currentParamIdx);
+            }
+
+            Parameter* ParameterSet::SelectNextParameter()
+            {
+                for (ParameterBindingMap::iterator it = parameters.begin(); it != parameters.end(); ++it)
+                {
+                    uint16_t paramIdx = it->first;
+                    Parameter& param = it->second;
+
+                    if (!param.IsDataReady())
+                    {
+                        currentParamIdx = paramIdx;
+
+                        return &param;
+                    }
+                }
+
+                return 0;
+            }
+
+            void ParameterSet::Write(impl::binary::BinaryWriterImpl& writer) const
+            {
+                writer.WriteInt32(CalculateRowLen());
+
+                WriteRow(writer, 0);
+            }
+
+            void ParameterSet::Write(impl::binary::BinaryWriterImpl& writer, SqlUlen begin, SqlUlen end, bool last) const
+            {
+                int32_t rowLen = CalculateRowLen();
+
+                writer.WriteInt32(rowLen);
+
+                SqlUlen intervalEnd = std::min(paramSetSize, end);
+
+                assert(begin < intervalEnd);
+
+                int32_t intervalLen = static_cast<int32_t>(intervalEnd - begin);
+
+                writer.WriteInt32(intervalLen);
+                writer.WriteBool(last);
+
+                if (rowLen)
+                {
+                    for (SqlUlen i = begin; i < intervalEnd; ++i)
+                        WriteRow(writer, i);
+                }
+            }
+
+            void ParameterSet::WriteRow(impl::binary::BinaryWriterImpl& writer, SqlUlen idx) const
+            {
+                uint16_t prev = 0;
+
+                int appOffset = paramBindOffset ? *paramBindOffset : 0;
+
+                for (ParameterBindingMap::const_iterator it = parameters.begin(); it != parameters.end(); ++it)
+                {
+                    uint16_t paramIdx = it->first;
+                    const Parameter& param = it->second;
+
+                    while ((paramIdx - prev) > 1)
+                    {
+                        writer.WriteNull();
+                        ++prev;
+                    }
+
+                    param.Write(writer, appOffset, idx);
+
+                    prev = paramIdx;
+                }
+            }
+
+            int32_t ParameterSet::CalculateRowLen() const
+            {
+                if (!parameters.empty())
+                    return static_cast<int32_t>(parameters.rbegin()->first);
+
+                return  0;
+            }
+
+            int32_t ParameterSet::GetParamSetSize() const
+            {
+                return static_cast<int32_t>(paramSetSize);
+            }
+        }
+    }
+}
diff --git a/modules/platforms/cpp/odbc/src/config/connection_info.cpp b/modules/platforms/cpp/odbc/src/config/connection_info.cpp
index 341ab7fea2863..6fe03f24dcf40 100644
--- a/modules/platforms/cpp/odbc/src/config/connection_info.cpp
+++ b/modules/platforms/cpp/odbc/src/config/connection_info.cpp
@@ -29,11 +29,11 @@
 
 #ifndef SQL_ASYNC_NOTIFICATION_NOT_CAPABLE
 #define SQL_ASYNC_NOTIFICATION_NOT_CAPABLE      0x00000000L
-#endif 
+#endif
 
 #ifndef SQL_ASYNC_NOTIFICATION_CAPABLE
 #define SQL_ASYNC_NOTIFICATION_CAPABLE          0x00000001L
-#endif 
+#endif
 
 namespace ignite
 {
@@ -95,7 +95,9 @@ namespace ignite
                     DBG_STR_CASE(SQL_SQL92_VALUE_EXPRESSIONS);
                     DBG_STR_CASE(SQL_STATIC_CURSOR_ATTRIBUTES1);
                     DBG_STR_CASE(SQL_STATIC_CURSOR_ATTRIBUTES2);
-                default: 
+                    DBG_STR_CASE(SQL_PARAM_ARRAY_ROW_COUNTS);
+                    DBG_STR_CASE(SQL_PARAM_ARRAY_SELECTS);
+                default:
                     break;
                 }
                 return "<< UNKNOWN TYPE >>";
@@ -117,7 +119,7 @@ namespace ignite
                 strParams[SQL_DBMS_VER]        = "03.00";
 
 #ifdef SQL_DRIVER_VER
-                // Driver version. At a minimum, the version is of the form 
+                // Driver version. At a minimum, the version is of the form
                 // ##.##.####, where the first two digits are the major version,
                 // the next two digits are the minor version, and the last four
                 // digits are the release version.
@@ -125,7 +127,7 @@ namespace ignite
 #endif // SQL_DRIVER_VER
 
 #ifdef SQL_COLUMN_ALIAS
-                // A character string: "Y" if the data source supports column 
+                // A character string: "Y" if the data source supports column
                 // aliases; otherwise, "N".
                 strParams[SQL_COLUMN_ALIAS] = "Y";
 #endif // SQL_COLUMN_ALIAS
@@ -168,7 +170,7 @@ namespace ignite
 #endif // SQL_TABLE_TERM
 
 #ifdef SQL_SCHEMA_TERM
-                // A character string with the data source vendor's name for 
+                // A character string with the data source vendor's name for
                 // a schema; for example, "owner", "Authorization ID", or "Schema".
                 strParams[SQL_SCHEMA_TERM] = "schema";
 #endif // SQL_SCHEMA_TERM
@@ -194,9 +196,9 @@ namespace ignite
 
 #ifdef SQL_ASYNC_NOTIFICATION
                 // Indicates if the driver supports asynchronous notification.
-                // SQL_ASYNC_NOTIFICATION_CAPABLE  = Asynchronous execution 
+                // SQL_ASYNC_NOTIFICATION_CAPABLE  = Asynchronous execution
                 // notification is supported by the driver.
-                // SQL_ASYNC_NOTIFICATION_NOT_CAPABLE Asynchronous execution 
+                // SQL_ASYNC_NOTIFICATION_NOT_CAPABLE Asynchronous execution
                 // notification is not supported by the driver.
                 intParams[SQL_ASYNC_NOTIFICATION] = SQL_ASYNC_NOTIFICATION_NOT_CAPABLE;
 #endif // SQL_ASYNC_NOTIFICATION
@@ -207,7 +209,7 @@ namespace ignite
 #endif // SQL_GETDATA_EXTENSIONS
 
 #ifdef SQL_ODBC_INTERFACE_CONFORMANCE
-                // Indicates the level of the ODBC 3.x interface that the driver 
+                // Indicates the level of the ODBC 3.x interface that the driver
                 // complies with.
                 intParams[SQL_ODBC_INTERFACE_CONFORMANCE] = SQL_OIC_CORE;
 #endif // SQL_ODBC_INTERFACE_CONFORMANCE
@@ -229,7 +231,7 @@ namespace ignite
 #endif // SQL_SCHEMA_USAGE
 
 #ifdef SQL_MAX_IDENTIFIER_LEN
-                // Indicates the maximum size in characters that the data source 
+                // Indicates the maximum size in characters that the data source
                 // supports for user-defined names.
                 intParams[SQL_MAX_IDENTIFIER_LEN] = 128;
 #endif // SQL_MAX_IDENTIFIER_LEN
@@ -243,7 +245,7 @@ namespace ignite
 #ifdef SQL_NUMERIC_FUNCTIONS
                 // Bitmask enumerating the scalar numeric functions supported by
                 // the driver and associated data source.
-                intParams[SQL_NUMERIC_FUNCTIONS] = SQL_FN_NUM_ABS | SQL_FN_NUM_ACOS | SQL_FN_NUM_ASIN | 
+                intParams[SQL_NUMERIC_FUNCTIONS] = SQL_FN_NUM_ABS | SQL_FN_NUM_ACOS | SQL_FN_NUM_ASIN |
                     SQL_FN_NUM_ATAN | SQL_FN_NUM_ATAN2 | SQL_FN_NUM_CEILING | SQL_FN_NUM_COS | SQL_FN_NUM_COT |
                     SQL_FN_NUM_EXP | SQL_FN_NUM_FLOOR | SQL_FN_NUM_LOG | SQL_FN_NUM_MOD | SQL_FN_NUM_SIGN |
                     SQL_FN_NUM_SIN | SQL_FN_NUM_SQRT | SQL_FN_NUM_TAN | SQL_FN_NUM_PI | SQL_FN_NUM_RAND |
@@ -273,7 +275,7 @@ namespace ignite
 #endif // SQL_TIMEDATE_FUNCTIONS
 
 #ifdef SQL_TIMEDATE_ADD_INTERVALS
-                // Bitmask enumerating timestamp intervals supported by the driver 
+                // Bitmask enumerating timestamp intervals supported by the driver
                 // and associated data source for the TIMESTAMPADD scalar function.
                 intParams[SQL_TIMEDATE_ADD_INTERVALS] = 0;
 #endif // SQL_TIMEDATE_ADD_INTERVALS
@@ -303,7 +305,7 @@ namespace ignite
 #endif // SQL_CONVERT_FUNCTIONS
 
 #ifdef SQL_OJ_CAPABILITIES
-                // Bitmask enumerating the types of outer joins supported by the 
+                // Bitmask enumerating the types of outer joins supported by the
                 // driver and data source.
                 intParams[SQL_OJ_CAPABILITIES] = SQL_OJ_LEFT | SQL_OJ_NOT_ORDERED | SQL_OJ_ALL_COMPARISON_OPS;
 #endif // SQL_OJ_CAPABILITIES
@@ -333,7 +335,7 @@ namespace ignite
 #ifdef SQL_SQL92_VALUE_EXPRESSIONS
                 // Bitmask enumerating the value expressions supported,
                 // as defined in SQL-92.
-                intParams[SQL_SQL92_VALUE_EXPRESSIONS] = SQL_SVE_CASE | 
+                intParams[SQL_SQL92_VALUE_EXPRESSIONS] = SQL_SVE_CASE |
                     SQL_SVE_CAST | SQL_SVE_COALESCE | SQL_SVE_NULLIF;
 #endif // SQL_SQL92_VALUE_EXPRESSIONS
 
@@ -369,6 +371,40 @@ namespace ignite
                 intParams[SQL_STATIC_CURSOR_ATTRIBUTES2] = 0;
 #endif //SQL_STATIC_CURSOR_ATTRIBUTES2
 
+#ifdef SQL_PARAM_ARRAY_ROW_COUNTS
+                // Enumerating the driver's properties regarding the availability of row counts in a parameterized
+                // execution. Has the following values:
+                //
+                // SQL_PARC_BATCH = Individual row counts are available for each set of parameters. This is conceptually
+                //     equivalent to the driver generating a batch of SQL statements, one for each parameter set in the
+                //     array. Extended error information can be retrieved by using the SQL_PARAM_STATUS_PTR descriptor
+                //     field.
+                //
+                // SQL_PARC_NO_BATCH = There is only one row count available, which is the cumulative row count
+                //     resulting from the execution of the statement for the entire array of parameters. This is
+                //     conceptually equivalent to treating the statement together with the complete parameter array as
+                //     one atomic unit. Errors are handled the same as if one statement were executed.
+                intParams[SQL_PARAM_ARRAY_ROW_COUNTS] = SQL_PARC_NO_BATCH;
+#endif //SQL_PARAM_ARRAY_ROW_COUNTS
+
+#ifdef SQL_PARAM_ARRAY_SELECTS
+                // Enumerating the driver's properties regarding the availability of result sets in a parameterized
+                // execution. Has the following values:
+                //
+                // SQL_PAS_BATCH = There is one result set available per set of parameters. This is conceptually
+                //     equivalent to the driver generating a batch of SQL statements, one for each parameter set in
+                //     the array.
+                //
+                // SQL_PAS_NO_BATCH = There is only one result set available, which represents the cumulative result set
+                //     resulting from the execution of the statement for the complete array of parameters. This is
+                //     conceptually equivalent to treating the statement together with the complete parameter array as
+                //     one atomic unit.
+                //
+                // SQL_PAS_NO_SELECT = A driver does not allow a result - set generating statement to be executed with
+                //     an array of parameters.
+                intParams[SQL_PARAM_ARRAY_SELECTS] = SQL_PAS_NO_SELECT;
+#endif //SQL_PARAM_ARRAY_SELECTS
+
                 //======================= Short Params ========================
 #ifdef SQL_MAX_CONCURRENT_ACTIVITIES
                 // The maximum number of active statements that the driver can
@@ -412,10 +448,10 @@ namespace ignite
 
                 StringInfoMap::const_iterator itStr = strParams.find(type);
 
-                if (itStr != strParams.cend()) 
+                if (itStr != strParams.cend())
                 {
                     unsigned short strlen = static_cast<short>(
-                        utility::CopyStringToBuffer(itStr->second, 
+                        utility::CopyStringToBuffer(itStr->second,
                             reinterpret_cast<char*>(buf), buflen));
 
                     if (reslen)
diff --git a/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp b/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp
index 215d77f5c249f..6e873c296f1b8 100644
--- a/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp
+++ b/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp
@@ -75,6 +75,12 @@ namespace
 
     /** SQL state HYT01 constant. */
     const std::string STATE_HYT01 = "HYT01";
+
+    /** SQL state HY003 constant. */
+    const std::string STATE_HY003 = "HY003";
+
+    /** SQL state HY105 constant. */
+    const std::string STATE_HY105 = "HY105";
 }
 
 namespace ignite
@@ -246,6 +252,12 @@ namespace ignite
                     case SQL_STATE_HYT01_CONNECTIOIN_TIMEOUT:
                         return STATE_HYT01;
 
+                    case SQL_STATE_HY003_INVALID_APPLICATION_BUFFER_TYPE:
+                        return STATE_HY003;
+
+                    case SQL_STATE_HY105_INVALID_PARAMETER_TYPE:
+                        return STATE_HY105;
+
                     default:
                         break;
                 }
diff --git a/modules/platforms/cpp/odbc/src/message.cpp b/modules/platforms/cpp/odbc/src/message.cpp
new file mode 100644
index 0000000000000..741540526fb19
--- /dev/null
+++ b/modules/platforms/cpp/odbc/src/message.cpp
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ignite/odbc/message.h"
+#include "ignite/odbc/utility.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        HandshakeRequest::HandshakeRequest(int64_t version, bool distributedJoins, bool enforceJoinOrder):
+            version(version),
+            distributedJoins(distributedJoins),
+            enforceJoinOrder(enforceJoinOrder)
+        {
+            // No-op.
+        }
+
+        HandshakeRequest::~HandshakeRequest()
+        {
+            // No-op.
+        }
+
+        void HandshakeRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_HANDSHAKE);
+
+            writer.WriteInt64(version);
+
+            writer.WriteBool(distributedJoins);
+            writer.WriteBool(enforceJoinOrder);
+        }
+
+        QueryExecuteRequest::QueryExecuteRequest(const std::string& schema, const std::string& sql, const app::ParameterSet& params):
+            cache(schema),
+            sql(sql),
+            params(params)
+        {
+            // No-op.
+        }
+
+        QueryExecuteRequest::~QueryExecuteRequest()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_EXECUTE_SQL_QUERY);
+
+            if (cache.empty())
+                writer.WriteNull();
+            else
+                writer.WriteObject<std::string>(cache);
+
+            writer.WriteObject<std::string>(sql);
+
+            params.Write(writer);
+        }
+
+        QueryExecuteBatchtRequest::QueryExecuteBatchtRequest(const std::string& schema, const std::string& sql,
+            const app::ParameterSet& params, SqlUlen begin, SqlUlen end, bool last):
+            schema(schema),
+            sql(sql),
+            params(params),
+            begin(begin),
+            end(end),
+            last(last)
+        {
+            // No-op.
+        }
+
+        QueryExecuteBatchtRequest::~QueryExecuteBatchtRequest()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteBatchtRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_EXECUTE_SQL_QUERY_BATCH);
+
+            if (schema.empty())
+                writer.WriteNull();
+            else
+                writer.WriteObject<std::string>(schema);
+
+            writer.WriteObject<std::string>(sql);
+
+            params.Write(writer, begin, end, last);
+        }
+
+        QueryCloseRequest::QueryCloseRequest(int64_t queryId): queryId(queryId)
+        {
+            // No-op.
+        }
+
+        QueryCloseRequest::~QueryCloseRequest()
+        {
+            // No-op.
+        }
+
+        void QueryCloseRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_CLOSE_SQL_QUERY);
+            writer.WriteInt64(queryId);
+        }
+
+        QueryFetchRequest::QueryFetchRequest(int64_t queryId, int32_t pageSize):
+            queryId(queryId),
+            pageSize(pageSize)
+        {
+            // No-op.
+        }
+
+        QueryFetchRequest::~QueryFetchRequest()
+        {
+            // No-op.
+        }
+
+        void QueryFetchRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_FETCH_SQL_QUERY);
+            writer.WriteInt64(queryId);
+            writer.WriteInt32(pageSize);
+        }
+
+        QueryGetColumnsMetaRequest::QueryGetColumnsMetaRequest(const std::string& schema, const std::string& table, const std::string& column):
+            schema(schema),
+            table(table),
+            column(column)
+        {
+            // No-op.
+        }
+
+        QueryGetColumnsMetaRequest::~QueryGetColumnsMetaRequest()
+        {
+            // No-op.
+        }
+
+        void QueryGetColumnsMetaRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_GET_COLUMNS_METADATA);
+
+            utility::WriteString(writer, schema);
+            utility::WriteString(writer, table);
+            utility::WriteString(writer, column);
+        }
+
+        QueryGetTablesMetaRequest::QueryGetTablesMetaRequest(const std::string& catalog, const std::string& schema, const std::string& table, const std::string& tableTypes):
+            catalog(catalog),
+            schema(schema),
+            table(table),
+            tableTypes(tableTypes)
+        {
+            // No-op.
+        }
+
+        QueryGetTablesMetaRequest::~QueryGetTablesMetaRequest()
+        {
+            // No-op.
+        }
+
+        void QueryGetTablesMetaRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_GET_TABLES_METADATA);
+
+            utility::WriteString(writer, catalog);
+            utility::WriteString(writer, schema);
+            utility::WriteString(writer, table);
+            utility::WriteString(writer, tableTypes);
+        }
+
+        void QueryGetParamsMetaRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_GET_PARAMS_METADATA);
+
+            utility::WriteString(writer, cacheName);
+            utility::WriteString(writer, sqlQuery);
+        }
+
+        Response::Response():
+            status(RESPONSE_STATUS_FAILED),
+            error()
+        {
+            // No-op.
+        }
+
+        Response::~Response()
+        {
+            // No-op.
+        }
+
+        void Response::Read(impl::binary::BinaryReaderImpl& reader)
+        {
+            status = reader.ReadInt8();
+
+            if (status == RESPONSE_STATUS_SUCCESS)
+                ReadOnSuccess(reader);
+            else
+                utility::ReadString(reader, error);
+        }
+
+        void Response::ReadOnSuccess(impl::binary::BinaryReaderImpl&)
+        {
+            // No-op.
+        }
+
+        HandshakeResponse::HandshakeResponse():
+            accepted(false),
+            protoVerSince(),
+            currentVer()
+        {
+            // No-op.
+        }
+
+        HandshakeResponse::~HandshakeResponse()
+        {
+            // No-op.
+        }
+
+        void HandshakeResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            accepted = reader.ReadBool();
+
+            if (!accepted)
+            {
+                utility::ReadString(reader, protoVerSince);
+                utility::ReadString(reader, currentVer);
+            }
+        }
+
+        QueryCloseResponse::QueryCloseResponse(): queryId(0)
+        {
+            // No-op.
+        }
+
+        QueryCloseResponse::~QueryCloseResponse()
+        {
+            // No-op.
+        }
+
+        void QueryCloseResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            queryId = reader.ReadInt64();
+        }
+
+        QueryExecuteResponse::QueryExecuteResponse(): queryId(0), meta()
+        {
+            // No-op.
+        }
+
+        QueryExecuteResponse::~QueryExecuteResponse()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            queryId = reader.ReadInt64();
+
+            meta::ReadColumnMetaVector(reader, meta);
+        }
+
+        QueryExecuteBatchResponse::QueryExecuteBatchResponse():
+            affectedRows(0),
+            errorSetIdx(-1),
+            errorMessage()
+        {
+            // No-op.
+        }
+
+        QueryExecuteBatchResponse::~QueryExecuteBatchResponse()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteBatchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            bool success = reader.ReadBool();
+            affectedRows = reader.ReadInt64();
+
+            if (!success)
+            {
+                errorSetIdx = reader.ReadInt64();
+                errorMessage = reader.ReadObject<std::string>();
+            }
+        }
+
+        QueryFetchResponse::QueryFetchResponse(ResultPage& resultPage): queryId(0), resultPage(resultPage)
+        {
+            // No-op.
+        }
+
+        QueryFetchResponse::~QueryFetchResponse()
+        {
+            // No-op.
+        }
+
+        void QueryFetchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            queryId = reader.ReadInt64();
+
+            resultPage.Read(reader);
+        }
+
+        QueryGetColumnsMetaResponse::QueryGetColumnsMetaResponse()
+        {
+            // No-op.
+        }
+
+        QueryGetColumnsMetaResponse::~QueryGetColumnsMetaResponse()
+        {
+            // No-op.
+        }
+
+        void QueryGetColumnsMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            meta::ReadColumnMetaVector(reader, meta);
+        }
+
+        QueryGetTablesMetaResponse::QueryGetTablesMetaResponse()
+        {
+            // No-op.
+        }
+
+        QueryGetTablesMetaResponse::~QueryGetTablesMetaResponse()
+        {
+            // No-op.
+        }
+
+        void QueryGetTablesMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            meta::ReadTableMetaVector(reader, meta);
+        }
+
+        QueryGetParamsMetaResponse::QueryGetParamsMetaResponse()
+        {
+            // No-op.
+        }
+
+        QueryGetParamsMetaResponse::~QueryGetParamsMetaResponse()
+        {
+            // No-op.
+        }
+
+        void QueryGetParamsMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            utility::ReadByteArray(reader, typeIds);
+        }
+    }
+}
+
diff --git a/modules/platforms/cpp/odbc/src/odbc.cpp b/modules/platforms/cpp/odbc/src/odbc.cpp
index caf31d91860b1..5f4db6597bfea 100644
--- a/modules/platforms/cpp/odbc/src/odbc.cpp
+++ b/modules/platforms/cpp/odbc/src/odbc.cpp
@@ -630,27 +630,8 @@ namespace ignite
         if (!statement)
             return SQL_INVALID_HANDLE;
 
-        if (ioType != SQL_PARAM_INPUT)
-            return SQL_ERROR;
-
-        if (!IsSqlTypeSupported(paramSqlType))
-            return SQL_ERROR;
-
-        IgniteSqlType driverType = ToDriverType(bufferType);
-
-        if (driverType == IGNITE_ODBC_C_TYPE_UNSUPPORTED)
-            return SQL_ERROR;
-
-        if (buffer)
-        {
-            ApplicationDataBuffer dataBuffer(driverType, buffer, bufferLen, resLen);
-
-            Parameter param(dataBuffer, paramSqlType, columnSize, decDigits);
-
-            statement->BindParameter(paramIdx, param);
-        }
-        else
-            statement->UnbindParameter(paramIdx);
+        statement->BindParameter(paramIdx, ioType, bufferType, paramSqlType,
+            columnSize, decDigits, buffer, bufferLen, resLen);
 
         return statement->GetDiagnosticRecords().GetReturnCode();
     }
diff --git a/modules/platforms/cpp/odbc/src/query/batch_query.cpp b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
new file mode 100644
index 0000000000000..11ddd93e19302
--- /dev/null
+++ b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ignite/odbc/connection.h"
+#include "ignite/odbc/message.h"
+#include "ignite/odbc/query/batch_query.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        namespace query
+        {
+            BatchQuery::BatchQuery(diagnostic::Diagnosable& diag, Connection& connection,
+                const std::string& sql, const app::ParameterSet& params) :
+                Query(diag, BATCH),
+                connection(connection),
+                sql(sql),
+                params(params),
+                resultMeta(),
+                rowsAffected(0),
+                setsProcessed(0),
+                executed(false),
+                dataRetrieved(false)
+            {
+                // No-op.
+            }
+
+            BatchQuery::~BatchQuery()
+            {
+                // No-op.
+            }
+
+            SqlResult BatchQuery::Execute()
+            {
+                if (executed)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query cursor is in open state already.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                int32_t maxPageSize = connection.GetConfiguration().GetPageSize();
+                int32_t rowNum = params.GetParamSetSize();
+                SqlResult res;
+
+                int32_t processed = 0;
+
+                do {
+                    int32_t currentPageSize = std::min(maxPageSize, rowNum - processed);
+                    bool lastPage = currentPageSize == rowNum - processed;
+
+                    res = MakeRequestExecuteBatch(processed, processed + currentPageSize, lastPage);
+
+                    processed += currentPageSize;
+                } while (res == SQL_RESULT_SUCCESS && processed < rowNum);
+
+                params.SetParamsProcessed(static_cast<SqlUlen>(setsProcessed));
+
+                return res;
+            }
+
+            const meta::ColumnMetaVector& BatchQuery::GetMeta() const
+            {
+                return resultMeta;
+            }
+
+            SqlResult BatchQuery::FetchNextRow(app::ColumnBindingMap& columnBindings)
+            {
+                if (!executed)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query was not executed.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                if (dataRetrieved)
+                    return SQL_RESULT_NO_DATA;
+
+                app::ColumnBindingMap::iterator it = columnBindings.find(1);
+
+                if (it != columnBindings.end())
+                    it->second.PutInt64(rowsAffected);
+
+                dataRetrieved = true;
+
+                return SQL_RESULT_SUCCESS;
+            }
+
+            SqlResult BatchQuery::GetColumn(uint16_t columnIdx, app::ApplicationDataBuffer& buffer)
+            {
+                if (!executed)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query was not executed.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                if (dataRetrieved)
+                    return SQL_RESULT_NO_DATA;
+
+                if (columnIdx != 1)
+                {
+                    std::stringstream builder;
+                    builder << "Column with id " << columnIdx << " is not available in result set.";
+
+                    diag.AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR, builder.str());
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                buffer.PutInt64(rowsAffected);
+
+                return SQL_RESULT_SUCCESS;
+            }
+
+            SqlResult BatchQuery::Close()
+            {
+                return SQL_RESULT_SUCCESS;
+            }
+
+            bool BatchQuery::DataAvailable() const
+            {
+                return false;
+            }
+
+            int64_t BatchQuery::AffectedRows() const
+            {
+                return rowsAffected;
+            }
+
+            SqlResult BatchQuery::MakeRequestExecuteBatch(SqlUlen begin, SqlUlen end, bool last)
+            {
+                const std::string& schema = connection.GetCache();
+
+                QueryExecuteBatchtRequest req(schema, sql, params, begin, end, last);
+                QueryExecuteBatchResponse rsp;
+
+                try
+                {
+                    connection.SyncMessage(req, rsp);
+                }
+                catch (const IgniteError& err)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                if (rsp.GetStatus() != RESPONSE_STATUS_SUCCESS)
+                {
+                    LOG_MSG("Error: " << rsp.GetError());
+
+                    diag.AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR, rsp.GetError());
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                rowsAffected += rsp.GetAffectedRows();
+                LOG_MSG("rowsAffected: " << rowsAffected);
+
+                if (!rsp.GetErrorMessage().empty())
+                {
+                    LOG_MSG("Error: " << rsp.GetErrorMessage());
+
+                    setsProcessed += rsp.GetErrorSetIdx();
+                    LOG_MSG("setsProcessed: " << setsProcessed);
+
+                    diag.AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR, rsp.GetErrorMessage(),
+                        static_cast<int32_t>(setsProcessed), 0);
+
+                    return SQL_RESULT_SUCCESS_WITH_INFO;
+                }
+
+                setsProcessed += end - begin;
+                LOG_MSG("setsProcessed: " << setsProcessed);
+
+                return SQL_RESULT_SUCCESS;
+            }
+        }
+    }
+}
+
diff --git a/modules/platforms/cpp/odbc/src/query/data_query.cpp b/modules/platforms/cpp/odbc/src/query/data_query.cpp
index 5b1b758bd4281..f4402d41f7895 100644
--- a/modules/platforms/cpp/odbc/src/query/data_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/data_query.cpp
@@ -18,6 +18,7 @@
 #include "ignite/odbc/connection.h"
 #include "ignite/odbc/message.h"
 #include "ignite/odbc/query/data_query.h"
+#include "ignite/odbc/query/batch_query.h"
 
 namespace ignite
 {
@@ -25,9 +26,8 @@ namespace ignite
     {
         namespace query
         {
-            DataQuery::DataQuery(diagnostic::Diagnosable& diag,
-                Connection& connection, const std::string& sql,
-                const app::ParameterBindingMap& params) :
+            DataQuery::DataQuery(diagnostic::Diagnosable& diag, Connection& connection,
+                const std::string& sql, const app::ParameterSet& params) :
                 Query(diag, DATA),
                 connection(connection),
                 sql(sql),
@@ -38,9 +38,9 @@ namespace ignite
 
             DataQuery::~DataQuery()
             {
-                Close();
+                InternalClose();
             }
-            
+
             SqlResult DataQuery::Execute()
             {
                 if (cursor.get())
@@ -139,6 +139,11 @@ namespace ignite
             }
 
             SqlResult DataQuery::Close()
+            {
+                return InternalClose();
+            }
+
+            SqlResult DataQuery::InternalClose()
             {
                 if (!cursor.get())
                     return SQL_RESULT_SUCCESS;
diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp
index 9aca8c9164209..6154f9126568e 100644
--- a/modules/platforms/cpp/odbc/src/statement.cpp
+++ b/modules/platforms/cpp/odbc/src/statement.cpp
@@ -16,6 +16,7 @@
  */
 
 #include "ignite/odbc/system/odbc_constants.h"
+#include "ignite/odbc/query/batch_query.h"
 #include "ignite/odbc/query/data_query.h"
 #include "ignite/odbc/query/column_metadata_query.h"
 #include "ignite/odbc/query/table_metadata_query.h"
@@ -38,9 +39,8 @@ namespace ignite
             currentQuery(),
             rowsFetched(0),
             rowStatuses(0),
-            paramBindOffset(0),
             columnBindOffset(0),
-            currentParamIdx(0)
+            parameters()
         {
             // No-op.
         }
@@ -55,8 +55,6 @@ namespace ignite
             IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
 
             columnBindings[columnIdx] = buffer;
-
-            columnBindings[columnIdx].SetPtrToOffsetPtr(&columnBindOffset);
         }
 
         void Statement::UnbindColumn(uint16_t columnIdx)
@@ -108,25 +106,72 @@ namespace ignite
             return SQL_RESULT_SUCCESS;
         }
 
-        void Statement::BindParameter(uint16_t paramIdx, const app::Parameter& param)
+        void Statement::BindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType, int16_t paramSqlType,
+            SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen)
         {
-            IGNITE_ODBC_API_CALL(InternalBindParameter(paramIdx, param));
+            IGNITE_ODBC_API_CALL(InternalBindParameter(paramIdx, ioType, bufferType, paramSqlType,
+                columnSize, decDigits, buffer, bufferLen, resLen));
         }
 
-
-        SqlResult Statement::InternalBindParameter(uint16_t paramIdx, const app::Parameter& param)
+        SqlResult Statement::InternalBindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType,
+            int16_t paramSqlType, SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen)
         {
+            using namespace type_traits;
+            using app::ApplicationDataBuffer;
+            using app::Parameter;
+
             if (paramIdx == 0)
             {
-                AddStatusRecord(SQL_STATE_24000_INVALID_CURSOR_STATE,
-                    "The value specified for the argument ParameterNumber was less than 1.");
+                std::stringstream builder;
+                builder << "The value specified for the argument ParameterNumber was less than 1. [ParameterNumber=" << paramIdx << ']';
+
+                AddStatusRecord(SQL_STATE_24000_INVALID_CURSOR_STATE, builder.str());
+
+                return SQL_RESULT_ERROR;
+            }
+
+            if (ioType != SQL_PARAM_INPUT)
+            {
+                std::stringstream builder;
+                builder << "The value specified for the argument InputOutputType was not SQL_PARAM_INPUT. [ioType=" << ioType << ']';
+
+                AddStatusRecord(SQL_STATE_HY105_INVALID_PARAMETER_TYPE, builder.str());
+
+                return SQL_RESULT_ERROR;
+            }
+
+            if (!IsSqlTypeSupported(paramSqlType))
+            {
+                std::stringstream builder;
+                builder << "Data type is not supported. [typeId=" << paramSqlType << ']';
+
+                AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED, builder.str());
+
+                return SQL_RESULT_ERROR;
+            }
+
+            IgniteSqlType driverType = ToDriverType(bufferType);
+
+            if (driverType == IGNITE_ODBC_C_TYPE_UNSUPPORTED)
+            {
+                std::stringstream builder;
+                builder << "The argument TargetType was not a valid data type. [TargetType=" << bufferType << ']';
+
+                AddStatusRecord(SQL_STATE_HY003_INVALID_APPLICATION_BUFFER_TYPE, builder.str());
 
                 return SQL_RESULT_ERROR;
             }
 
-            paramBindings[paramIdx] = param;
+            if (buffer)
+            {
+                ApplicationDataBuffer dataBuffer(driverType, buffer, bufferLen, resLen);
+
+                Parameter param(dataBuffer, paramSqlType, columnSize, decDigits);
 
-            paramBindings[paramIdx].GetBuffer().SetPtrToOffsetPtr(&paramBindOffset);
+                parameters.BindParameter(paramIdx, param);
+            }
+            else
+                parameters.UnbindParameter(paramIdx);
 
             return SQL_RESULT_SUCCESS;
         }
@@ -135,14 +180,14 @@ namespace ignite
         {
             IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
 
-            paramBindings.erase(paramIdx);
+            parameters.UnbindParameter(paramIdx);
         }
 
         void Statement::UnbindAllParameters()
         {
             IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
 
-            paramBindings.clear();
+            parameters.UnbindAll();
         }
 
         void Statement::SetAttribute(int attr, void* value, SQLINTEGER valueLen)
@@ -156,7 +201,7 @@ namespace ignite
             {
                 case SQL_ATTR_ROW_ARRAY_SIZE:
                 {
-                    SQLULEN val = reinterpret_cast<SQLULEN>(value);
+                    SqlUlen val = reinterpret_cast<SqlUlen>(value);
 
                     LOG_MSG("SQL_ATTR_ROW_ARRAY_SIZE: %d\n", val);
 
@@ -199,6 +244,20 @@ namespace ignite
                     break;
                 }
 
+                case SQL_ATTR_PARAMSET_SIZE:
+                {
+                    parameters.SetParamSetSize(reinterpret_cast<SqlUlen>(value));
+
+                    break;
+                }
+
+                case SQL_ATTR_PARAMS_PROCESSED_PTR:
+                {
+                    parameters.SetParamsProcessedPtr(reinterpret_cast<SqlUlen*>(value));
+
+                    break;
+                }
+
                 default:
                 {
                     AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED,
@@ -236,6 +295,9 @@ namespace ignite
 
                     *val = static_cast<SQLPOINTER>(this);
 
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
+
                     break;
                 }
 
@@ -245,14 +307,20 @@ namespace ignite
 
                     *val = static_cast<SQLINTEGER>(1);
 
+                    if (valueLen)
+                        *valueLen = SQL_IS_INTEGER;
+
                     break;
                 }
 
                 case SQL_ATTR_ROWS_FETCHED_PTR:
                 {
-                    SQLULEN** val = reinterpret_cast<SQLULEN**>(buf);
+                    SqlUlen** val = reinterpret_cast<SqlUlen**>(buf);
+
+                    *val = reinterpret_cast<SqlUlen*>(GetRowsFetchedPtr());
 
-                    *val = reinterpret_cast<SQLULEN*>(GetRowsFetchedPtr());
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
 
                     break;
                 }
@@ -263,6 +331,9 @@ namespace ignite
 
                     *val = reinterpret_cast<SQLUSMALLINT*>(GetRowStatusesPtr());
 
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
+
                     break;
                 }
 
@@ -270,16 +341,46 @@ namespace ignite
                 {
                     SQLULEN** val = reinterpret_cast<SQLULEN**>(buf);
 
-                    *val = reinterpret_cast<SQLULEN*>(GetParamBindOffsetPtr());
+                    *val = reinterpret_cast<SQLULEN*>(parameters.GetParamBindOffsetPtr());
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
 
                     break;
                 }
 
                 case SQL_ATTR_ROW_BIND_OFFSET_PTR:
                 {
-                    SQLULEN** val = reinterpret_cast<SQLULEN**>(buf);
+                    SqlUlen** val = reinterpret_cast<SqlUlen**>(buf);
+
+                    *val = reinterpret_cast<SqlUlen*>(GetColumnBindOffsetPtr());
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
+
+                    break;
+                }
+
+                case SQL_ATTR_PARAMSET_SIZE:
+                {
+                    SqlUlen* val = reinterpret_cast<SqlUlen*>(buf);
+
+                    *val = static_cast<SqlUlen>(parameters.GetParamSetSize());
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_UINTEGER;
+
+                    break;
+                }
+
+                case SQL_ATTR_PARAMS_PROCESSED_PTR:
+                {
+                    SqlUlen** val = reinterpret_cast<SqlUlen**>(buf);
+
+                    *val = parameters.GetParamsProcessedPtr();
 
-                    *val = reinterpret_cast<SQLULEN*>(GetColumnBindOffsetPtr());
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
 
                     break;
                 }
@@ -317,7 +418,7 @@ namespace ignite
                 return SQL_RESULT_SUCCESS;
             }
 
-            if (paramTypes.empty())
+            if (!parameters.IsMetadataSet())
             {
                 SqlResult res = UpdateParamsMeta();
 
@@ -325,7 +426,7 @@ namespace ignite
                     return res;
             }
 
-            paramNum = static_cast<uint16_t>(paramTypes.size());
+            paramNum = parameters.GetExpectedParamNum();
 
             return SQL_RESULT_SUCCESS;
         }
@@ -334,12 +435,7 @@ namespace ignite
         {
             IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
 
-            paramBindOffset = ptr;
-        }
-
-        int* Statement::GetParamBindOffsetPtr()
-        {
-            return paramBindOffset;
+            parameters.SetParamBindOffsetPtr(ptr);
         }
 
         void Statement::GetColumnData(uint16_t columnIdx, app::ApplicationDataBuffer& buffer)
@@ -373,10 +469,10 @@ namespace ignite
             if (currentQuery.get())
                 currentQuery->Close();
 
-            currentQuery.reset(new query::DataQuery(*this, connection, query, paramBindings));
-
             // Resetting parameters types as we are changing the query.
-            paramTypes.clear();
+            parameters.Prepare();
+
+            currentQuery.reset(new query::DataQuery(*this, connection, query, parameters));
 
             return SQL_RESULT_SUCCESS;
         }
@@ -410,20 +506,31 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            bool paramDataReady = true;
-
-            app::ParameterBindingMap::iterator it;
-            for (it = paramBindings.begin(); it != paramBindings.end(); ++it)
+            if (parameters.GetParamSetSize() > 1 && currentQuery->GetType() == query::Query::DATA)
             {
-                app::Parameter& param = it->second;
+                query::DataQuery& qry = static_cast<query::DataQuery&>(*currentQuery);
 
-                param.ResetStoredData();
+                currentQuery.reset(new query::BatchQuery(*this, connection, qry.GetSql(), parameters));
+            }
+            else if (parameters.GetParamSetSize() == 1 && currentQuery->GetType() == query::Query::BATCH)
+            {
+                query::BatchQuery& qry = static_cast<query::BatchQuery&>(*currentQuery);
 
-                paramDataReady &= param.IsDataReady();
+                currentQuery.reset(new query::DataQuery(*this, connection, qry.GetSql(), parameters));
             }
 
-            if (!paramDataReady)
+            if (parameters.IsDataAtExecNeeded())
+            {
+                if (currentQuery->GetType() == query::Query::BATCH)
+                {
+                    AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED,
+                        "Data-at-execution is not supported together with batching.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
                 return SQL_RESULT_NEED_DATA;
+            }
 
             return currentQuery->Execute();
         }
@@ -549,8 +656,10 @@ namespace ignite
         {
             if (!type_traits::IsSqlTypeSupported(sqlType))
             {
-                AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED,
-                    "Data type is not supported.");
+                std::stringstream builder;
+                builder << "Data type is not supported. [typeId=" << sqlType << ']';
+
+                AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED, builder.str());
 
                 return SQL_RESULT_ERROR;
             }
@@ -572,7 +681,7 @@ namespace ignite
         {
             if (!currentQuery.get())
                 return SQL_RESULT_SUCCESS;
-            
+
             SqlResult result = currentQuery->Close();
 
             if (result == SQL_RESULT_SUCCESS)
@@ -599,6 +708,12 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
+            if (columnBindOffset)
+            {
+                for (app::ColumnBindingMap::iterator it = columnBindings.begin(); it != columnBindings.end(); ++it)
+                    it->second.SetByteOffset(*columnBindOffset);
+            }
+
             SqlResult res = currentQuery->FetchNextRow(columnBindings);
 
             if (res == SQL_RESULT_SUCCESS)
@@ -767,35 +882,24 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            app::ParameterBindingMap::iterator it;
+            app::Parameter *selected = parameters.GetSelectedParameter();
 
-            if (currentParamIdx)
+            if (selected && !selected->IsDataReady())
             {
-                it = paramBindings.find(currentParamIdx);
+                AddStatusRecord(SQL_STATE_22026_DATA_LENGTH_MISMATCH,
+                    "Less data was sent for a parameter than was specified with "
+                    "the StrLen_or_IndPtr argument in SQLBindParameter.");
 
-                if (it != paramBindings.end() && !it->second.IsDataReady())
-                {
-                    AddStatusRecord(SQL_STATE_22026_DATA_LENGTH_MISMATCH,
-                        "Less data was sent for a parameter than was specified with "
-                        "the StrLen_or_IndPtr argument in SQLBindParameter.");
-
-                    return SQL_RESULT_ERROR;
-                }
+                return SQL_RESULT_ERROR;
             }
 
-            for (it = paramBindings.begin(); it != paramBindings.end(); ++it)
-            {
-                uint16_t paramIdx = it->first;
-                app::Parameter& param = it->second;
-
-                if (!param.IsDataReady())
-                {
-                    *paramPtr = param.GetBuffer().GetData();
+            selected = parameters.SelectNextParameter();
 
-                    currentParamIdx = paramIdx;
+            if (selected)
+            {
+                *paramPtr = selected->GetBuffer().GetData();
 
-                    return SQL_RESULT_NEED_DATA;
-                }
+                return SQL_RESULT_NEED_DATA;
             }
 
             SqlResult res = currentQuery->Execute();
@@ -822,7 +926,7 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            if (currentParamIdx == 0)
+            if (!parameters.IsParameterSelected())
             {
                 AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR,
                     "Parameter is not selected with the SQLParamData.");
@@ -830,9 +934,9 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            app::ParameterBindingMap::iterator it = paramBindings.find(currentParamIdx);
+            app::Parameter* param = parameters.GetSelectedParameter();
 
-            if (it == paramBindings.end())
+            if (!param)
             {
                 AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR,
                     "Selected parameter has been unbound.");
@@ -840,9 +944,7 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            app::Parameter& param = it->second;
-
-            param.PutData(data, len);
+            param->PutData(data, len);
 
             return SQL_RESULT_SUCCESS;
         }
@@ -872,10 +974,7 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            int8_t type = 0;
-
-            if (paramNum > 0 && static_cast<size_t>(paramNum) <= paramTypes.size())
-                type = paramTypes[paramNum - 1];
+            int8_t type = parameters.GetParamType(paramNum, 0);
 
             LOG_MSG("Type: %d\n", type);
 
@@ -886,10 +985,7 @@ namespace ignite
                 if (res != SQL_RESULT_SUCCESS)
                     return res;
 
-                if (paramNum < 1 || static_cast<size_t>(paramNum) > paramTypes.size())
-                    type = impl::binary::IGNITE_HDR_NULL;
-                else
-                    type = paramTypes[paramNum - 1];
+                type = parameters.GetParamType(paramNum, impl::binary::IGNITE_HDR_NULL);
             }
 
             if (dataType)
@@ -942,10 +1038,12 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            paramTypes = rsp.GetTypeIds();
+            parameters.UpdateParamsTypes(rsp.GetTypeIds());
 
-            for (size_t i = 0; i < paramTypes.size(); ++i)
-                LOG_MSG("[%zu] Parameter type: %u\n", i, paramTypes[i]);
+            for (size_t i = 0; i < rsp.GetTypeIds().size(); ++i)
+            {
+                LOG_MSG("[" << i << "] Parameter type: " << rsp.GetTypeIds()[i]);
+            }
 
             return SQL_RESULT_SUCCESS;
         }

From 7fbaecc67f1b204162bda4595d6c118ddd45f963 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 16 Jun 2017 20:01:49 +0300
Subject: [PATCH 296/446] IGNITE-5527: Prevent starvation in stripe pool on
 unstable topology.

---
 .../processors/cache/GridDeferredAckMessageSender.java         | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java
index 7145dc2cd7fee..967404a1ccf3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java
@@ -173,7 +173,8 @@ private DeferredAckMessageBuffer(UUID nodeId) {
          * @return {@code True} if request was handled, {@code false} if this buffer is filled and cannot be used.
          */
         public boolean add(GridCacheVersion ver) {
-            readLock().lock();
+            if(!readLock().tryLock())
+                return false; // Here, writeLock is help by another thread and guard is already true.
 
             boolean snd = false;
 

From f81964f59b0ea5b8dfdc8eb2acc34d2a5b8fee07 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 10 Jan 2017 16:59:17 +0300
Subject: [PATCH 297/446] Do not evict removed entries, otherwise removes can
 be lost.

(cherry picked from commit 55ac6e7)
---
 .../internal/processors/cache/GridCacheMapEntry.java     | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 58b4ae3130c9d..ea01bca4bd50b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -29,7 +29,6 @@
 import javax.cache.expiry.ExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.EntryProcessorResult;
-
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -4300,6 +4299,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                     if (obsoleteVersionExtras() != null)
                         return true;
 
+                    // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                    if (cctx.deferredDelete() && deletedUnlocked())
+                        return false;
+
                     CacheObject prev = saveOldValueUnlocked(false);
 
                     if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {
@@ -4358,6 +4361,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                             // Version has changed since entry passed the filter. Do it again.
                             continue;
 
+                        // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                        if (cctx.deferredDelete() && deletedUnlocked())
+                            return false;
+
                         CacheObject prevVal = saveValueForIndexUnlocked();
 
                         if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {

From 5dd74ff635de50ff9561ccdb51bdeb620f60c3db Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 10 Jan 2017 16:59:17 +0300
Subject: [PATCH 298/446] Do not evict removed entries, otherwise removes can
 be lost.

(cherry picked from commit 55ac6e7)
---
 .../internal/processors/cache/GridCacheMapEntry.java     | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 58b4ae3130c9d..ea01bca4bd50b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -29,7 +29,6 @@
 import javax.cache.expiry.ExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.EntryProcessorResult;
-
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -4300,6 +4299,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                     if (obsoleteVersionExtras() != null)
                         return true;
 
+                    // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                    if (cctx.deferredDelete() && deletedUnlocked())
+                        return false;
+
                     CacheObject prev = saveOldValueUnlocked(false);
 
                     if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {
@@ -4358,6 +4361,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                             // Version has changed since entry passed the filter. Do it again.
                             continue;
 
+                        // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                        if (cctx.deferredDelete() && deletedUnlocked())
+                            return false;
+
                         CacheObject prevVal = saveValueForIndexUnlocked();
 
                         if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {

From 5fb5c7e3b54ae4efb7a6a1832ba647677d93e0cd Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 22 Jun 2017 09:43:03 +0300
Subject: [PATCH 299/446] IGNITE-5399 Manual cache rebalancing feature is
 broken

---
 .../java/org/apache/ignite/IgniteCache.java   |   3 +-
 .../GridCachePartitionExchangeManager.java    |   6 +-
 .../processors/cache/GridCachePreloader.java  |   4 +-
 .../cache/GridCachePreloaderAdapter.java      |   4 +-
 .../preloader/GridDhtPartitionDemander.java   |  17 +-
 .../GridDhtPartitionsExchangeFuture.java      |   7 +-
 .../dht/preloader/GridDhtPreloader.java       |   2 +-
 .../CacheManualRebalancingTest.java           | 178 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |   4 +-
 9 files changed, 201 insertions(+), 24 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
index d7bccf58941b1..ef2c4b6a00166 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
@@ -872,7 +872,8 @@ public <T> Map<K, EntryProcessorResult<T>> invokeAll(Set<? extends K> keys,
      * See {@link CacheConfiguration#getRebalanceDelay()} for more information on how to configure
      * rebalance re-partition delay.
      * <p>
-     * @return Future that will be completed when rebalancing is finished.
+     * @return Future that will be completed when rebalancing is finished. Future.get() returns true
+     *      when rebalance was successfully finished.
      */
     public IgniteFuture<?> rebalance();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index c175c43ae2f2a..07805f3f1b50b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -80,6 +80,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridListSet;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.IgnitePair;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -724,7 +725,7 @@ public void forceDummyExchange(boolean reassign,
      * @param exchFut Exchange future.
      */
     public IgniteInternalFuture<Boolean> forceRebalance(GridDhtPartitionsExchangeFuture exchFut) {
-        GridFutureAdapter<Boolean> fut = new GridFutureAdapter<>();
+        GridCompoundFuture<Boolean, Boolean> fut = new GridCompoundFuture<>(CU.boolReducer());
 
         exchWorker.addFuture(
             new GridDhtPartitionsExchangeFuture(cctx, exchFut.discoveryEvent(), exchFut.exchangeId(), fut));
@@ -1830,6 +1831,9 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                             }
                         }
 
+                        if (exchFut.forcedRebalanceFuture() != null)
+                            exchFut.forcedRebalanceFuture().markInitialized();
+
                         if (assignsCancelled) { // Pending exchange.
                             U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
                                 "[top=" + exchFut.topologyVersion() + ", evt=" + exchFut.discoveryEvent().name() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index 0c2869101aa79..110a53b3d2154 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -28,7 +28,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
@@ -92,7 +92,7 @@ public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
         boolean forcePreload,
         int cnt,
         Runnable next,
-        @Nullable GridFutureAdapter<Boolean> forcedRebFut);
+        @Nullable GridCompoundFuture<Boolean, Boolean> forcedRebFut);
 
     /**
      * @param p Preload predicate.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index 8ae67215dabc9..fe4859b2dc4b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -30,8 +30,8 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
@@ -167,7 +167,7 @@ public GridCachePreloaderAdapter(GridCacheContext<?, ?> cctx) {
 
     /** {@inheritDoc} */
     @Override public Runnable addAssignments(GridDhtPreloaderAssignments assignments, boolean forcePreload,
-        int cnt, Runnable next, @Nullable GridFutureAdapter<Boolean> forcedRebFut) {
+        int cnt, Runnable next, @Nullable GridCompoundFuture<Boolean, Boolean> forcedRebFut) {
         return null;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 9ece00cd01049..5c6eb2b9a53d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -54,6 +54,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -301,7 +302,7 @@ Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
         boolean force,
         int cnt,
         final Runnable next,
-        @Nullable final GridFutureAdapter<Boolean> forcedRebFut) {
+        @Nullable final GridCompoundFuture<Boolean, Boolean> forcedRebFut) {
         if (log.isDebugEnabled())
             log.debug("Adding partition assignments: " + assigns);
 
@@ -324,18 +325,8 @@ Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
                 });
             }
 
-            if (forcedRebFut != null) {
-                fut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
-                    @Override public void apply(IgniteInternalFuture<Boolean> future) {
-                        try {
-                            forcedRebFut.onDone(future.get());
-                        }
-                        catch (Exception e) {
-                            forcedRebFut.onDone(e);
-                        }
-                    }
-                });
-            }
+            if (forcedRebFut != null)
+                forcedRebFut.add(fut);
 
             rebalanceFut = fut;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 2245d17e83523..d3e3701936786 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -62,6 +62,7 @@
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -198,7 +199,7 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     private boolean centralizedAff;
 
     /** Forced Rebalance future. */
-    private GridFutureAdapter<Boolean> forcedRebFut;
+    private GridCompoundFuture<Boolean, Boolean> forcedRebFut;
 
     /**
      * Dummy future created to trigger reassignments if partition
@@ -236,7 +237,7 @@ public GridDhtPartitionsExchangeFuture(
      * @param forcedRebFut Forced Rebalance future.
      */
     public GridDhtPartitionsExchangeFuture(GridCacheSharedContext cctx, DiscoveryEvent discoEvt,
-        GridDhtPartitionExchangeId exchId, GridFutureAdapter<Boolean> forcedRebFut) {
+        GridDhtPartitionExchangeId exchId, GridCompoundFuture<Boolean, Boolean> forcedRebFut) {
         dummy = false;
         forcePreload = true;
 
@@ -417,7 +418,7 @@ public GridDhtPartitionExchangeId exchangeId() {
     /**
      * @return Forced Rebalance future.
      */
-    @Nullable public GridFutureAdapter<Boolean> forcedRebalanceFuture() {
+    @Nullable public GridCompoundFuture<Boolean, Boolean> forcedRebalanceFuture() {
         return forcedRebFut;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 4aff4d5a71f42..2efaed859b714 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -413,7 +413,7 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
 
     /** {@inheritDoc} */
     @Override public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
-        boolean forcePreload, int cnt, Runnable next,  @Nullable GridFutureAdapter<Boolean> forcedRebFut) {
+        boolean forcePreload, int cnt, Runnable next,  @Nullable GridCompoundFuture<Boolean, Boolean> forcedRebFut) {
         return demander.addAssignments(assignments, forcePreload, cnt, next, forcedRebFut);
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java
new file mode 100644
index 0000000000000..363e07b890711
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/** */
+public class CacheManualRebalancingTest extends GridCommonAbstractTest {
+    /** */
+    private static final String MYCACHE = "mycache";
+
+    /** */
+    public static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    public static final int NODES_CNT = 2;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setCacheConfiguration(cacheConfiguration());
+
+        return cfg;
+    }
+
+    /**
+     * @return Cache configuration.
+     */
+    private static CacheConfiguration cacheConfiguration() {
+        return new CacheConfiguration(MYCACHE)
+            .setAtomicityMode(ATOMIC)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setWriteSynchronizationMode(FULL_SYNC)
+            .setRebalanceMode(ASYNC)
+            .setRebalanceDelay(-1)
+            .setBackups(1)
+            .setCopyOnRead(true)
+            .setReadFromBackup(true);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrids(NODES_CNT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 400_000;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRebalance() throws Exception {
+        // Fill cache with large dataset to make rebalancing slow.
+        try (IgniteDataStreamer<Object, Object> streamer = grid(0).dataStreamer(MYCACHE)) {
+            for (int i = 0; i < 100_000; i++)
+                streamer.addData(i, i);
+        }
+
+        // Start new node.
+        final IgniteEx newNode = startGrid(NODES_CNT);
+
+        int newNodeCacheSize;
+
+        // Start manual rebalancing.
+        IgniteCompute compute = newNode.compute().withAsync();
+
+        compute.broadcast(new MyCallable());
+
+        final ComputeTaskFuture<Object> rebalanceTaskFuture = compute.future();
+
+        boolean rebalanceFinished = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return rebalanceTaskFuture.isDone();
+            }
+        }, 10_000);
+
+        assertTrue(rebalanceFinished);
+
+        assertTrue(newNode.context().cache().cache(MYCACHE).context().preloader().rebalanceFuture().isDone());
+
+        newNodeCacheSize = newNode.cache(MYCACHE).localSize(CachePeekMode.ALL);
+
+        System.out.println("New node cache local size: " + newNodeCacheSize);
+
+        assertTrue(newNodeCacheSize > 0);
+
+    }
+
+    /** */
+    public static class MyCallable implements IgniteRunnable {
+        /** */
+        @IgniteInstanceResource
+        Ignite localNode;
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            IgniteLogger log = localNode.log();
+
+            log.info("Start local rebalancing caches");
+
+            for (String cacheName : localNode.cacheNames()) {
+                IgniteCache<?, ?> cache = localNode.cache(cacheName);
+
+                assertNotNull(cache);
+
+                boolean finished;
+                
+                log.info("Start rebalancing cache: " + cacheName + ", size: " + cache.localSize());
+
+                do {
+                    IgniteFuture<?> rebalance = cache.rebalance();
+
+                    log.info("Wait rebalancing cache: " + cacheName + " - " + rebalance);
+
+                    finished = (Boolean)rebalance.get();
+
+                    log.info("Rebalancing cache: " + cacheName + " - " + rebalance);
+
+                    if (finished) {
+                        log.info("Finished rebalancing cache: " + cacheName + ", size: " +
+                            cache.localSize(CachePeekMode.PRIMARY) + cache.localSize(CachePeekMode.BACKUP));
+                    } else
+                        log.info("Rescheduled rebalancing cache: " + cacheName + ", size: " + cache.localSize());
+                }
+                while (!finished);
+            }
+
+            log.info("Finished local rebalancing caches");
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 58a26e76ddd2f..464ded85c8587 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -31,8 +31,9 @@
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentFairAffinityTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest;
 import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest;
 
 /**
@@ -66,6 +67,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheOffHeapCleanupTest.class);
 
         suite.addTestSuite(CacheRebalancingSelfTest.class);
+        suite.addTestSuite(CacheManualRebalancingTest.class);
 
         return suite;
     }

From 01d41b72ecc3e81dfc8966cc0e395c247037241c Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Wed, 21 Jun 2017 13:48:15 +0300
Subject: [PATCH 300/446] GG-12256 H2Indexes are not deleted if key class
 implements Externalizable

---
 .../processors/query/GridQueryProcessor.java  |  26 ++-
 .../cache/SqlQueryAfterCacheClearedTest.java  | 163 ++++++++++++++++++
 2 files changed, 187 insertions(+), 2 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 0bde688ded248..c79a65a6f1347 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -268,6 +268,17 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                         else
                             desc.keyClass(Object.class);
                     }
+                    else if (binaryEnabled) {
+                        if (!valMustDeserialize && !SQL_TYPES.contains(valCls))
+                            desc.valueClass(Object.class);
+                        else
+                            desc.valueClass(valCls);
+
+                        if (!keyMustDeserialize && !SQL_TYPES.contains(keyCls))
+                            desc.keyClass(Object.class);
+                        else
+                            desc.keyClass(keyCls);
+                    }
                     else {
                         if (keyCls == null)
                             throw new IgniteCheckedException("Failed to find key class in the node classpath " +
@@ -298,7 +309,7 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     TypeId typeId;
                     TypeId altTypeId = null;
 
-                    if (valCls == null || (binaryEnabled && !keyOrValMustDeserialize)) {
+                    if (valCls == null || (binaryEnabled && (!valMustDeserialize))) {
                         processBinaryMeta(qryEntity, desc);
 
                         typeId = new TypeId(ccfg.getName(), ctx.cacheObjects().typeId(qryEntity.getValueType()));
@@ -385,6 +396,17 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                         else
                             desc.keyClass(Object.class);
                     }
+                    else if (binaryEnabled) {
+                        if (!valMustDeserialize && !SQL_TYPES.contains(valCls))
+                            desc.valueClass(Object.class);
+                        else
+                            desc.valueClass(valCls);
+
+                        if (!keyMustDeserialize && !SQL_TYPES.contains(keyCls))
+                            desc.keyClass(Object.class);
+                        else
+                            desc.keyClass(keyCls);
+                    }
                     else {
                         desc.valueClass(valCls);
                         desc.keyClass(keyCls);
@@ -407,7 +429,7 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     TypeId typeId;
                     TypeId altTypeId = null;
 
-                    if (valCls == null || (binaryEnabled && !keyOrValMustDeserialize)) {
+                    if (valCls == null || (binaryEnabled && !valMustDeserialize)) {
                         processBinaryMeta(meta, desc);
 
                         typeId = new TypeId(ccfg.getName(), ctx.cacheObjects().typeId(meta.getValueType()));
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java
new file mode 100644
index 0000000000000..fa2d9ca341030
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.affinity.AffinityKey;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Test;
+
+import java.util.LinkedHashMap;
+
+import static java.util.Collections.singletonList;
+import static org.apache.ignite.cache.CacheMemoryMode.OFFHEAP_TIERED;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+public class SqlQueryAfterCacheClearedTest extends GridCommonAbstractTest {
+
+
+    /** */
+    public static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    private final static String CACHE_NAME = "propertyCache";
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrids(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        return new IgniteConfiguration()
+            .setGridName(gridName)
+            .setPeerClassLoadingEnabled(false)
+            .setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER))
+            .setCacheConfiguration(cacheCfg());
+    }
+
+    /** */
+    private static CacheConfiguration cacheCfg() {
+        final CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME)
+        .setCacheMode(PARTITIONED)
+        .setMemoryMode(OFFHEAP_TIERED)
+        .setQueryEntities(singletonList(createQueryEntityConfig()));
+        return ccfg;
+    }
+
+    @Test
+    public void testQueryCacheWasCleared() throws InterruptedException {
+        IgniteCache<PropertyAffinityKey, Property> cache = grid(0).cache(CACHE_NAME);
+
+        Property property1 = new Property(1, 2);
+        Property property2 = new Property(2, 2);
+
+        cache.put(property1.getKey(), property1);
+        cache.put(property2.getKey(), property2);
+
+        assertEquals(cache.size(),2);
+        assertEquals(cache.query(selectAllQuery()).getAll().size(), 2);
+
+        cache.clear();
+
+        assertEquals(0, cache.size());
+        assertEquals(0, cache.query(selectAllQuery()).getAll().size());
+    }
+
+    @Test
+    public void testQueryEntriesWereRemoved() {
+        IgniteCache<PropertyAffinityKey, Property> cache = grid(0).cache(CACHE_NAME);
+
+        Property property1 = new Property(1, 2);
+        Property property2 = new Property(2, 2);
+
+        cache.put(property1.getKey(), property1);
+        cache.put(property2.getKey(), property2);
+
+        assertEquals(cache.size(),2);
+        assertEquals(cache.query(selectAllQuery()).getAll().size(), 2);
+
+        cache.remove(new PropertyAffinityKey(1, 2));
+        cache.remove(new PropertyAffinityKey(2, 2));
+
+        assertEquals(0, cache.size());
+        assertEquals(0, cache.query(selectAllQuery()).getAll().size());
+    }
+
+    @NotNull
+    private SqlQuery<PropertyAffinityKey, Property> selectAllQuery() {
+        return new SqlQuery<>(Property.class, "from Property");
+    }
+
+    private static QueryEntity createQueryEntityConfig() {
+        QueryEntity queryEntity = new QueryEntity();
+        queryEntity.setKeyType(PropertyAffinityKey.class.getName());
+        queryEntity.setValueType(Property.class.getName());
+        queryEntity.setFields(getMapOfFields());
+        return queryEntity;
+    }
+
+    @NotNull
+    private static LinkedHashMap<String, String> getMapOfFields() {
+        LinkedHashMap<String, String> mapOfFields = new LinkedHashMap<>();
+        mapOfFields.put("id", Integer.class.getName());
+        mapOfFields.put("region", Integer.class.getName());
+        mapOfFields.put("key", PropertyAffinityKey.class.getName());
+        return mapOfFields;
+    }
+
+    static class Property {
+        private final int id;
+        private final int region;
+        private final String someData = "Some attributes";
+
+        Property(int id, int region) {
+            this.id = id;
+            this.region = region;
+        }
+
+        public PropertyAffinityKey getKey() {
+            return new PropertyAffinityKey(id, region);
+        }
+
+        public int getId() {
+            return id;
+        }
+    }
+
+    static class PropertyAffinityKey extends AffinityKey<Integer> {
+        public PropertyAffinityKey(final int thirdPartyPropertyId, final int region) {
+            super(thirdPartyPropertyId, region);
+        }
+
+        public PropertyAffinityKey() {
+            // Required by Ignite
+        }
+    }
+}

From 5ac9afc719138e37a7d97d9d9db05243eee9a942 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 22 Jun 2017 12:36:14 +0300
Subject: [PATCH 301/446] IGNITE-5399 add test to testsuite

---
 .../ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java       | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java
index c4fcdacb603c8..2bcfd1dbec332 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.SqlQueryAfterCacheClearedTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFailoverAtomicPrimaryWriteOrderSelfTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFailoverTxReplicatedSelfTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFailoverTxSelfTest;
@@ -50,6 +51,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheContinuousQueryAsyncFailoverAtomicPrimaryWriteOrderSelfTest.class);
         suite.addTestSuite(CacheContinuousQueryAsyncFailoverTxReplicatedSelfTest.class);
         suite.addTestSuite(CacheContinuousQueryAsyncFailoverTxSelfTest.class);
+        suite.addTestSuite(SqlQueryAfterCacheClearedTest.class);
 
         return suite;
     }

From a935d40a80e2f928a84a145aba540a45b156687f Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 22 Jun 2017 15:10:32 +0300
Subject: [PATCH 302/446] GG-12256 Minor fixes

---
 .../processors/query/GridQueryProcessor.java  | 28 +------
 .../cache/SqlQueryAfterCacheClearedTest.java  | 75 +++++++++++--------
 2 files changed, 46 insertions(+), 57 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index c79a65a6f1347..7511a991472c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -256,19 +256,7 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
 
                     desc.tableName(qryEntity.getTableName());
 
-                    if (binaryEnabled && !keyOrValMustDeserialize) {
-                        // Safe to check null.
-                        if (SQL_TYPES.contains(valCls))
-                            desc.valueClass(valCls);
-                        else
-                            desc.valueClass(Object.class);
-
-                        if (SQL_TYPES.contains(keyCls))
-                            desc.keyClass(keyCls);
-                        else
-                            desc.keyClass(Object.class);
-                    }
-                    else if (binaryEnabled) {
+                    if (binaryEnabled) {
                         if (!valMustDeserialize && !SQL_TYPES.contains(valCls))
                             desc.valueClass(Object.class);
                         else
@@ -384,19 +372,7 @@ else if (binaryEnabled) {
 
                     desc.name(simpleValType);
 
-                    if (binaryEnabled && !keyOrValMustDeserialize) {
-                        // Safe to check null.
-                        if (SQL_TYPES.contains(valCls))
-                            desc.valueClass(valCls);
-                        else
-                            desc.valueClass(Object.class);
-
-                        if (SQL_TYPES.contains(keyCls))
-                            desc.keyClass(keyCls);
-                        else
-                            desc.keyClass(Object.class);
-                    }
-                    else if (binaryEnabled) {
+                    if (binaryEnabled) {
                         if (!valMustDeserialize && !SQL_TYPES.contains(valCls))
                             desc.valueClass(Object.class);
                         else
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java
index fa2d9ca341030..669161fb98ac5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java
@@ -27,7 +27,6 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.NotNull;
-import org.junit.Test;
 
 import java.util.LinkedHashMap;
 
@@ -35,13 +34,15 @@
 import static org.apache.ignite.cache.CacheMemoryMode.OFFHEAP_TIERED;
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 
+/**
+ *
+ */
 public class SqlQueryAfterCacheClearedTest extends GridCommonAbstractTest {
-
-
     /** */
     public static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
-    private final static String CACHE_NAME = "propertyCache";
+    /** */
+    private static final String CACHE_NAME = "propertyCache";
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
@@ -64,22 +65,21 @@ public class SqlQueryAfterCacheClearedTest extends GridCommonAbstractTest {
 
     /** */
     private static CacheConfiguration cacheCfg() {
-        final CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME)
+        return new CacheConfiguration(CACHE_NAME)
         .setCacheMode(PARTITIONED)
         .setMemoryMode(OFFHEAP_TIERED)
         .setQueryEntities(singletonList(createQueryEntityConfig()));
-        return ccfg;
     }
 
-    @Test
+    /** */
     public void testQueryCacheWasCleared() throws InterruptedException {
         IgniteCache<PropertyAffinityKey, Property> cache = grid(0).cache(CACHE_NAME);
 
-        Property property1 = new Property(1, 2);
-        Property property2 = new Property(2, 2);
+        Property prop1 = new Property(1, 2);
+        Property prop2 = new Property(2, 2);
 
-        cache.put(property1.getKey(), property1);
-        cache.put(property2.getKey(), property2);
+        cache.put(prop1.getKey(), prop1);
+        cache.put(prop2.getKey(), prop2);
 
         assertEquals(cache.size(),2);
         assertEquals(cache.query(selectAllQuery()).getAll().size(), 2);
@@ -90,15 +90,15 @@ public void testQueryCacheWasCleared() throws InterruptedException {
         assertEquals(0, cache.query(selectAllQuery()).getAll().size());
     }
 
-    @Test
+    /** */
     public void testQueryEntriesWereRemoved() {
         IgniteCache<PropertyAffinityKey, Property> cache = grid(0).cache(CACHE_NAME);
 
-        Property property1 = new Property(1, 2);
-        Property property2 = new Property(2, 2);
+        Property prop1 = new Property(1, 2);
+        Property prop2 = new Property(2, 2);
 
-        cache.put(property1.getKey(), property1);
-        cache.put(property2.getKey(), property2);
+        cache.put(prop1.getKey(), prop1);
+        cache.put(prop2.getKey(), prop2);
 
         assertEquals(cache.size(),2);
         assertEquals(cache.query(selectAllQuery()).getAll().size(), 2);
@@ -110,21 +110,22 @@ public void testQueryEntriesWereRemoved() {
         assertEquals(0, cache.query(selectAllQuery()).getAll().size());
     }
 
-    @NotNull
-    private SqlQuery<PropertyAffinityKey, Property> selectAllQuery() {
+    /** */
+    @NotNull private SqlQuery<PropertyAffinityKey, Property> selectAllQuery() {
         return new SqlQuery<>(Property.class, "from Property");
     }
 
+    /** */
     private static QueryEntity createQueryEntityConfig() {
-        QueryEntity queryEntity = new QueryEntity();
-        queryEntity.setKeyType(PropertyAffinityKey.class.getName());
-        queryEntity.setValueType(Property.class.getName());
-        queryEntity.setFields(getMapOfFields());
-        return queryEntity;
+        QueryEntity qryEntity = new QueryEntity();
+        qryEntity.setKeyType(PropertyAffinityKey.class.getName());
+        qryEntity.setValueType(Property.class.getName());
+        qryEntity.setFields(getMapOfFields());
+        return qryEntity;
     }
 
-    @NotNull
-    private static LinkedHashMap<String, String> getMapOfFields() {
+    /** */
+    @NotNull private static LinkedHashMap<String, String> getMapOfFields() {
         LinkedHashMap<String, String> mapOfFields = new LinkedHashMap<>();
         mapOfFields.put("id", Integer.class.getName());
         mapOfFields.put("region", Integer.class.getName());
@@ -132,32 +133,44 @@ private static LinkedHashMap<String, String> getMapOfFields() {
         return mapOfFields;
     }
 
-    static class Property {
+    /**
+     *
+     */
+    private static class Property {
+        /** Id. */
         private final int id;
+
+        /** Region. */
         private final int region;
-        private final String someData = "Some attributes";
 
+        /** */
         Property(int id, int region) {
             this.id = id;
             this.region = region;
         }
 
+        /** */
         public PropertyAffinityKey getKey() {
             return new PropertyAffinityKey(id, region);
         }
 
+        /** */
         public int getId() {
             return id;
         }
     }
 
-    static class PropertyAffinityKey extends AffinityKey<Integer> {
-        public PropertyAffinityKey(final int thirdPartyPropertyId, final int region) {
-            super(thirdPartyPropertyId, region);
+    /**
+     *
+     */
+    private static class PropertyAffinityKey extends AffinityKey<Integer> {
+        /** */
+        PropertyAffinityKey(final int thirdPartyPropId, final int region) {
+            super(thirdPartyPropId, region);
         }
 
+        /** */
         public PropertyAffinityKey() {
-            // Required by Ignite
         }
     }
 }

From 7e2468770a4eb47a4f61204d8c2000b6ab67c967 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Thu, 22 Jun 2017 16:13:01 +0300
Subject: [PATCH 303/446] IGNITE-GG-12197 Fixed "Ignore events for discarded
 update in CLOCK mode".

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../processors/cache/GridCacheMapEntry.java   |  10 +-
 .../CacheContinuousQueryHandler.java          | 128 +++++++-----
 .../CacheContinuousQueryManager.java          |   5 +-
 .../cache/version/GridCacheVersion.java       |   9 +
 ...tinuousQueryClockModeConflictSelfTest.java | 196 ++++++++++++++++++
 ...ntinuousQueryFailoverAbstractSelfTest.java |  14 +-
 ...CacheContinuousQueryOrderingEventTest.java | 131 ++++++++----
 .../IgniteCacheQuerySelfTestSuite3.java       |   2 +
 8 files changed, 392 insertions(+), 103 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryClockModeConflictSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index ea01bca4bd50b..e8c449994a253 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -2280,16 +2280,12 @@ else if (ttl != CU.TTL_ZERO)
                             if (updateCntr != null)
                                 updateCntr0 = updateCntr;
 
-                            cctx.continuousQueries().onEntryUpdated(
+                            cctx.continuousQueries().skipUpdateEvent(
+                                lsnrs,
                                 key,
-                                evtVal,
-                                prevVal,
-                                isInternal() || !context().userCache(),
                                 partition(),
-                                primary,
-                                false,
                                 updateCntr0,
-                                null,
+                                primary,
                                 topVer);
                         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 17f4308c0709b..db8575c5ff89e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -137,6 +137,9 @@ public class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler
     /** */
     private boolean locCache;
 
+    /** Local query. */
+    private transient boolean localQuery;
+
     /** */
     private transient boolean keepBinary;
 
@@ -242,6 +245,13 @@ public void localCache(boolean locCache) {
         this.locCache = locCache;
     }
 
+    /**
+     * @param loc Local query.
+     */
+    public void localQuery(boolean loc) {
+        this.localQuery = loc;
+    }
+
     /**
      * @param taskHash Task hash.
      */
@@ -334,13 +344,13 @@ public void keepBinary(boolean keepBinary) {
             }
         }
 
-        entryBufs = new ConcurrentHashMap<>();
-
-        backupQueue = new ConcurrentLinkedDeque8<>();
-
-        ackBuf = new AcknowledgeBuffer();
-
-        rcvs = new ConcurrentHashMap<>();
+        // Not need to support Fault Tolerance for local queries or local cache.
+        if (!isQueryOnlyLocal()) {
+            entryBufs = new ConcurrentHashMap<>();
+            backupQueue = new ConcurrentLinkedDeque8<>();
+            ackBuf = new AcknowledgeBuffer();
+            rcvs = new ConcurrentHashMap<>();
+        }
 
         this.nodeId = nodeId;
 
@@ -403,7 +413,7 @@ public void keepBinary(boolean keepBinary) {
                         primary,
                         evt,
                         recordIgniteEvt,
-                        fut);
+                        isQueryOnlyLocal() ? null : fut);
 
                     ctx.asyncCallbackPool().execute(clsr, evt.partitionId());
                 }
@@ -415,7 +425,7 @@ public void keepBinary(boolean keepBinary) {
                             + ", notify=" + notify + ']');
 
                     if (primary || skipPrimaryCheck) {
-                        if (fut == null)
+                        if (fut == null || isQueryOnlyLocal())
                             onEntryUpdate(evt, notify, loc, recordIgniteEvt);
                         else {
                             fut.addContinuousQueryClosure(new CI1<Boolean>() {
@@ -437,57 +447,66 @@ public void keepBinary(boolean keepBinary) {
             }
 
             @Override public void cleanupBackupQueue(Map<Integer, Long> updateCntrs) {
-                Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
+                if (!isQueryOnlyLocal()) {
+                    Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
 
-                if (backupQueue0 != null) {
-                    Iterator<CacheContinuousQueryEntry> it = backupQueue0.iterator();
+                    if (backupQueue0 != null) {
+                        Iterator<CacheContinuousQueryEntry> it = backupQueue0.iterator();
 
-                    while (it.hasNext()) {
-                        CacheContinuousQueryEntry backupEntry = it.next();
+                        while (it.hasNext()) {
+                            CacheContinuousQueryEntry backupEntry = it.next();
 
-                        Long updateCntr = updateCntrs.get(backupEntry.partition());
+                            Long updateCntr = updateCntrs.get(backupEntry.partition());
 
-                        if (updateCntr != null && backupEntry.updateCounter() <= updateCntr)
-                            it.remove();
+                            if (updateCntr != null && backupEntry.updateCounter() <= updateCntr)
+                                it.remove();
+                        }
                     }
                 }
             }
 
             @Override public void flushBackupQueue(GridKernalContext ctx, AffinityTopologyVersion topVer) {
-                Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
+                if (!isQueryOnlyLocal()) {
+                    Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
 
-                if (backupQueue0 == null)
-                    return;
+                    if (backupQueue0 == null)
+                        return;
 
-                try {
-                    ClusterNode nodeId0 = ctx.discovery().node(nodeId);
+                    try {
+                        ClusterNode nodeId0 = ctx.discovery().node(nodeId);
 
-                    if (nodeId0 != null) {
-                        GridCacheContext<K, V> cctx = cacheContext(ctx);
+                        if (nodeId0 != null) {
+                            GridCacheContext<K, V> cctx = cacheContext(ctx);
 
-                        for (CacheContinuousQueryEntry e : backupQueue0) {
-                            if (!e.isFiltered())
-                                prepareEntry(cctx, nodeId, e);
+                            for (CacheContinuousQueryEntry e : backupQueue0) {
+                                if (!e.isFiltered())
+                                    prepareEntry(cctx, nodeId, e);
+
+                                e.topologyVersion(topVer);
+                            }
 
-                            e.topologyVersion(topVer);
+                            ctx.continuous().addBackupNotification(nodeId, routineId, backupQueue0, topic);
                         }
+                        else
+                            // Node which start CQ leave topology. Not needed to put data to backup queue.
+                            backupQueue = null;
 
-                        ctx.continuous().addBackupNotification(nodeId, routineId, backupQueue0, topic);
+                        backupQueue0.clear();
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY),
+                            "Failed to send backup event notification to node: " + nodeId, e);
                     }
-                    else
-                        // Node which start CQ leave topology. Not needed to put data to backup queue.
-                        backupQueue = null;
-
-                    backupQueue0.clear();
-                }
-                catch (IgniteCheckedException e) {
-                    U.error(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY),
-                        "Failed to send backup event notification to node: " + nodeId, e);
                 }
+                else
+                    assert backupQueue == null; // For local CQ backup queue should be null.
             }
 
             @Override public void acknowledgeBackupOnTimeout(GridKernalContext ctx) {
-                sendBackupAcknowledge(ackBuf.acknowledgeOnTimeout(), routineId, ctx);
+                if (!localQuery)
+                    sendBackupAcknowledge(ackBuf.acknowledgeOnTimeout(), routineId, ctx);
+                else
+                    assert ackBuf == null; // For local CQ ack buffer should be null.
             }
 
             @Override public void skipUpdateEvent(CacheContinuousQueryEvent<K, V> evt,
@@ -502,14 +521,18 @@ public void keepBinary(boolean keepBinary) {
             }
 
             @Override public void onPartitionEvicted(int part) {
-                Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
+                if (!isQueryOnlyLocal()) {
+                    Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
 
-                if (backupQueue0 != null) {
-                    for (Iterator<CacheContinuousQueryEntry> it = backupQueue0.iterator(); it.hasNext(); ) {
-                        if (it.next().partition() == part)
-                            it.remove();
+                    if (backupQueue0 != null) {
+                        for (Iterator<CacheContinuousQueryEntry> it = backupQueue0.iterator(); it.hasNext(); ) {
+                            if (it.next().partition() == part)
+                                it.remove();
+                        }
                     }
                 }
+                else
+                    assert backupQueue == null; // For local CQ backup queue should be null.
             }
 
             @Override public boolean oldValueRequired() {
@@ -788,7 +811,11 @@ private void onEntryUpdate(CacheContinuousQueryEvent evt, boolean notify, boolea
             final CacheContinuousQueryEntry entry = evt.entry();
 
             if (loc) {
-                if (!locCache) {
+                if (isQueryOnlyLocal()) {
+                    if (!entry.isFiltered())
+                        locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
+                }
+                else {
                     Collection<CacheEntryEvent<? extends K, ? extends V>> evts = handleEvent(ctx, entry);
 
                     if (!evts.isEmpty())
@@ -797,10 +824,6 @@ private void onEntryUpdate(CacheContinuousQueryEvent evt, boolean notify, boolea
                     if (!internal && !skipPrimaryCheck)
                         sendBackupAcknowledge(ackBuf.onAcknowledged(entry), routineId, ctx);
                 }
-                else {
-                    if (!entry.isFiltered())
-                        locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
-                }
             }
             else {
                 if (!entry.isFiltered())
@@ -848,6 +871,13 @@ private void onEntryUpdate(CacheContinuousQueryEvent evt, boolean notify, boolea
         }
     }
 
+    /**
+     * @return {@code True} if query deployed only localy (local query or local cache), otherwise {@code false}.
+     */
+    private boolean isQueryOnlyLocal() {
+        return locCache || localQuery;
+    }
+
     /**
      * @return Task name.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 91c199184a2de..9a4ca90269727 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -641,6 +641,7 @@ private UUID executeQuery0(CacheEntryUpdatedListener locLsnr,
         hnd.internal(internal);
         hnd.keepBinary(keepBinary);
         hnd.localCache(cctx.isLocal());
+        hnd.localQuery(loc);
 
         IgnitePredicate<ClusterNode> pred = (loc || cctx.config().getCacheMode() == CacheMode.LOCAL) ?
             F.nodeForNodeId(cctx.localNodeId()) : cctx.config().getNodeFilter();
@@ -649,14 +650,14 @@ private UUID executeQuery0(CacheEntryUpdatedListener locLsnr,
 
         UUID id = cctx.kernalContext().continuous().startRoutine(
             hnd,
-            internal && loc,
+            loc,
             bufSize,
             timeInterval,
             autoUnsubscribe,
             pred).get();
 
         try {
-            if (hnd.isQuery() && cctx.userCache() && !onStart)
+            if (hnd.isQuery() && cctx.userCache() && !onStart && !loc)
                 hnd.waitTopologyFuture(cctx.kernalContext());
         }
         catch (IgniteCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersion.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersion.java
index 95aab7407116c..7796dd5f9cdbc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersion.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersion.java
@@ -190,6 +190,15 @@ public IgniteUuid asGridUuid() {
         return new IgniteUuid(new UUID(((long)topVer << 32) | nodeOrderDrId, globalTime), order);
     }
 
+    /**
+     * For testing purpose!!!
+     *
+     * @param globalTime Global time.
+     */
+    public void globalTime(long globalTime) {
+        this.globalTime = globalTime;
+    }
+
     /** {@inheritDoc} */
     @Override public void onAckReceived() {
         // No-op.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryClockModeConflictSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryClockModeConflictSelfTest.java
new file mode 100644
index 0000000000000..9de738346090c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryClockModeConflictSelfTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryListenerException;
+import javax.cache.event.CacheEntryUpdatedListener;
+import javax.cache.event.EventType;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicWriteOrderMode;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicFullUpdateRequest;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test coverages case when update discarded
+ */
+public class CacheContinuousQueryClockModeConflictSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    public static final int KEYS = 0;
+
+    /** */
+    public static final int ITERATION_CNT = 50;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+        cfg.setCommunicationSpi(new TestCommunicationSpi());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        // Node should start exactly in the same order.
+        startGrid(0);
+        startGrid(1);
+        startGrid(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testIgniteLocalContinuousQuery() throws Exception {
+        for (int i = 0; i < ITERATION_CNT; i++) {
+            String cacheName = "test-cache-" + i;
+
+            IgniteCache cache2 = ignite(2).createCache(cacheConfiguration(cacheName));
+
+            try {
+                IgniteCache cache0 = ignite(0).cache(cacheName);
+                IgniteCache cache1 = ignite(1).cache(cacheName);
+
+                final BlockingQueue<CacheEntryEvent<Integer, Integer>> evts0 = new LinkedBlockingQueue<>();
+                QueryCursor query = registerQuery(cache0, evts0);
+
+                final BlockingQueue<CacheEntryEvent<Integer, Integer>> evts1 = new LinkedBlockingQueue<>();
+                QueryCursor query1 = registerQuery(cache1, evts1);
+
+                final BlockingQueue<CacheEntryEvent<Integer, Integer>> evts2 = new LinkedBlockingQueue<>();
+                QueryCursor query2 = registerQuery(cache2, evts2);
+
+                Integer key = keyForNode(affinity(cache1), new AtomicInteger(0),
+                    grid(0).cluster().localNode());
+
+                // This update should CacheVersion with orderId == 3.
+                cache2.put(key, 42);
+
+                // The update will be have orderId == 2 and will be discarded.
+                cache1.put(key, 41);
+
+                assertEquals("Entry was updated.", 42, cache1.get(key));
+
+                cache0.remove(key);
+
+                assertNull("Entry was not removed.", cache1.get(key));
+
+                CacheEntryEvent<Integer, Integer> e = evts0.poll(200, TimeUnit.MILLISECONDS);
+
+                assertNotNull("Missed events.", e);
+                assertEquals("Invalid event.", EventType.CREATED, e.getEventType());
+                assertNull("Invalid event.", e.getOldValue());
+                assertEquals("Invalid event.", Integer.valueOf(42), e.getValue());
+
+                e = evts0.poll(200, TimeUnit.MILLISECONDS);
+
+                assertNotNull("Missed events.", e);
+                assertEquals("Invalid event.", EventType.REMOVED, e.getEventType());
+                assertEquals("Invalid event.", Integer.valueOf(42), e.getOldValue());
+
+                assertTrue("Received extra events.", evts1.isEmpty());
+                assertTrue("Received extra events.", evts2.isEmpty());
+
+                query.close();
+                query1.close();
+                query2.close();
+            }
+            finally {
+                grid(0).destroyCache(cacheName);
+            }
+        }
+    }
+
+    private QueryCursor registerQuery(IgniteCache cache, final BlockingQueue<CacheEntryEvent<Integer, Integer>> evts) {
+        ContinuousQuery qry = new ContinuousQuery();
+
+        qry.setLocalListener(new CacheEntryUpdatedListener() {
+            @Override public void onUpdated(Iterable iterable) throws CacheEntryListenerException {
+                for (Object o : iterable)
+                    evts.add((CacheEntryEvent)o);
+            }
+        });
+
+        qry.setLocal(true);
+
+        return cache.query(qry);
+    }
+
+    /**
+     * @param name Cache name.
+     * @return Cache configuration.
+     */
+    private CacheConfiguration cacheConfiguration(String name) {
+        return new CacheConfiguration(name)
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setAtomicWriteOrderMode(CacheAtomicWriteOrderMode.CLOCK)
+            .setBackups(0);
+    }
+
+    /**
+     *
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC)
+            throws IgniteSpiException {
+            Object msg0 = ((GridIoMessage)msg).message();
+
+            if (msg0 instanceof GridNearAtomicFullUpdateRequest) {
+                GridCacheVersion ver = ((GridNearAtomicFullUpdateRequest)msg0).updateVersion();
+
+                ver.globalTime(142);
+            }
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
index d2cb710edbeb5..2df38d767299b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
@@ -258,15 +258,15 @@ public void testFirstFilteredEvent() throws Exception {
                 qryClnCache.put(key, -1);
 
             qryClnCache.put(keys.get(0), 100);
-        }
 
-        GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override public boolean apply() {
-                return lsnr.evts.size() == 1;
-            }
-        }, 5000);
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return lsnr.evts.size() == 1;
+                }
+            }, 5000);
 
-        assertEquals(lsnr.evts.size(), 1);
+            assertEquals(1, lsnr.evts.size());
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
index 7d975f2a28209..2460a8045cd1f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
@@ -145,7 +145,37 @@ public void testAtomicOnheapTwoBackup() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicOnheapTwoBackupLocalQuery() throws Exception {
+        CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
+            ONHEAP_TIERED, PRIMARY_SYNC);
+
+        doOrderingTest(ccfg, false, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicOnheapTwoBackupLocalQueryAsync() throws Exception {
+        CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
+            ONHEAP_TIERED, PRIMARY_SYNC);
+
+        doOrderingTest(ccfg, true, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicOnheapTwoBackupLocalQueryAsyncFullSync() throws Exception {
+        CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
+            ONHEAP_TIERED, FULL_SYNC);
+
+        doOrderingTest(ccfg, false, true);
     }
 
     /**
@@ -155,7 +185,7 @@ public void testAtomicOffheapTwoBackup() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     /**
@@ -165,7 +195,7 @@ public void testAtomicOffheapValuesTwoBackup() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_VALUES, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     /**
@@ -175,7 +205,7 @@ public void testAtomicReplicatedOffheap() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(REPLICATED, 0, ATOMIC,
             OFFHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     /**
@@ -185,7 +215,7 @@ public void testTxOnheapTwoBackup() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, TRANSACTIONAL,
             ONHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     /**
@@ -195,7 +225,7 @@ public void testTxOnheapWithoutBackup() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 0, TRANSACTIONAL,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     /**
@@ -205,7 +235,7 @@ public void testTxOnheapWithoutBackupFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 0, TRANSACTIONAL,
             ONHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     // ASYNC
@@ -217,7 +247,7 @@ public void testAtomicOnheapTwoBackupAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -227,7 +257,7 @@ public void testAtomicOnheapTwoBackupAsyncFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             ONHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -237,7 +267,7 @@ public void testAtomicOffheapTwoBackupAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -247,7 +277,7 @@ public void testAtomicOffheapTwoBackupAsyncFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -257,7 +287,7 @@ public void testAtomicOffheapValuesTwoBackupAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_VALUES, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -267,7 +297,7 @@ public void testAtomicOffheapValuesTwoBackupAsyncFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_VALUES, FULL_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -277,7 +307,7 @@ public void testAtomicReplicatedAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(REPLICATED, 0, ATOMIC,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -287,7 +317,7 @@ public void testAtomicReplicatedAsyncFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(REPLICATED, 0, ATOMIC,
             ONHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -297,7 +327,7 @@ public void testAtomicReplicatedOffheapAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(REPLICATED, 0, ATOMIC,
             OFFHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -307,7 +337,7 @@ public void testAtomicOnheapWithoutBackupAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 0, ATOMIC,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -317,7 +347,7 @@ public void testTxOnheapTwoBackupAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, TRANSACTIONAL,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -327,7 +357,7 @@ public void testTxOnheapAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 0, TRANSACTIONAL,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -337,17 +367,17 @@ public void testTxOnheapAsyncFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 0, TRANSACTIONAL,
             ONHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
      * @param ccfg Cache configuration.
      * @param async Async filter.
+     * @param localQry Start local query.
+     *
      * @throws Exception If failed.
      */
-    protected void doOrderingTest(
-        final CacheConfiguration ccfg,
-        final boolean async)
+    protected void doOrderingTest(final CacheConfiguration ccfg, final boolean async, final boolean localQry)
         throws Exception {
         ignite(0).createCache(ccfg);
 
@@ -372,15 +402,17 @@ protected void doOrderingTest(
                         qry.setLocalListener(new TestCacheAsyncEventListener(queue, qryCntr));
 
                         qry.setRemoteFilterFactory(FactoryBuilder.factoryOf(
-                            new CacheTestRemoteFilterAsync(ccfg.getName())));
+                            new CacheTestRemoteFilterAsync(ccfg.getName(), localQry)));
                     }
                     else {
                         qry.setLocalListener(new TestCacheEventListener(queue, qryCntr));
 
                         qry.setRemoteFilterFactory(FactoryBuilder.factoryOf(
-                            new CacheTestRemoteFilter(ccfg.getName())));
+                            new CacheTestRemoteFilter(ccfg.getName(), localQry)));
                     }
 
+                    qry.setLocal(localQry);
+
                     rcvdEvts.add(queue);
 
                     IgniteCache<Object, Object> cache = grid(idx).cache(ccfg.getName());
@@ -448,14 +480,17 @@ protected void doOrderingTest(
 
             f.get(15, TimeUnit.SECONDS);
 
-            GridTestUtils.waitForCondition(new PA() {
+            boolean res = GridTestUtils.waitForCondition(new PA() {
                 @Override public boolean apply() {
-                    return qryCntr.get() >= ITERATION_CNT * threadCnt * LISTENER_CNT * NODES;
+                    return qryCntr.get() >= ITERATION_CNT * threadCnt * LISTENER_CNT *
+                        (localQry ? 1 : NODES);
                 }
             }, 1000L);
 
+            assertTrue("Failed to wait all events.", res);
+
             for (BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>> queue : rcvdEvts)
-                checkEvents(queue, ITERATION_CNT * threadCnt);
+                checkEvents(queue, ITERATION_CNT * threadCnt, localQry);
 
             assertFalse("Ordering invocations of filter broken.", fail);
         }
@@ -469,9 +504,12 @@ protected void doOrderingTest(
 
     /**
      * @param queue Event queue.
+     * @param localQry Local query flag.
      * @throws Exception If failed.
      */
-    private void checkEvents(BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>> queue, int expCnt)
+    private void checkEvents(BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>> queue,
+        int expCnt,
+        boolean localQry)
         throws Exception {
         CacheEntryEvent<QueryTestKey, QueryTestValue> evt;
         int cnt = 0;
@@ -483,11 +521,17 @@ private void checkEvents(BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestVa
 
             Integer preVal = vals.get(evt.getKey());
 
-            if (preVal == null)
-                assertEquals(new QueryTestValue(0), evt.getValue());
+            if (!localQry) {
+                if (preVal == null)
+                    assertEquals(new QueryTestValue(0), evt.getValue());
+                else {
+                    if (!new QueryTestValue(preVal + 1).equals(evt.getValue()))
+                        assertEquals("Key event: " + evt.getKey(), new QueryTestValue(preVal + 1), evt.getValue());
+                }
+            }
             else {
-                if (!new QueryTestValue(preVal + 1).equals(evt.getValue()))
-                    assertEquals("Key event: " + evt.getKey(), new QueryTestValue(preVal + 1), evt.getValue());
+                if (preVal != null)
+                    assertTrue("Evt out of order.", preVal < evt.getValue().val1);
             }
 
             vals.put(evt.getKey(), evt.getValue().val1);
@@ -495,7 +539,8 @@ private void checkEvents(BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestVa
             ++cnt;
         }
 
-        assertEquals(expCnt, cnt);
+        if (!localQry)
+            assertEquals(expCnt, cnt);
     }
 
     /** {@inheritDoc} */
@@ -510,9 +555,10 @@ private void checkEvents(BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestVa
     private static class CacheTestRemoteFilterAsync extends CacheTestRemoteFilter {
         /**
          * @param cacheName Cache name.
+         * @param localQry Local query.
          */
-        public CacheTestRemoteFilterAsync(String cacheName) {
-            super(cacheName);
+        public CacheTestRemoteFilterAsync(String cacheName, boolean localQry) {
+            super(cacheName, localQry);
         }
     }
 
@@ -531,11 +577,16 @@ private static class CacheTestRemoteFilter implements
         /** */
         private String cacheName;
 
+        /** */
+        private boolean localQry;
+
         /**
          * @param cacheName Cache name.
+         * @param localQry Local query flag.
          */
-        public CacheTestRemoteFilter(String cacheName) {
+        public CacheTestRemoteFilter(String cacheName, boolean localQry) {
             this.cacheName = cacheName;
+            this.localQry = localQry;
         }
 
         /** {@inheritDoc} */
@@ -544,7 +595,11 @@ public CacheTestRemoteFilter(String cacheName) {
                 QueryTestValue prevVal = prevVals.put(e.getKey(), e.getValue());
 
                 if (prevVal != null) {
-                    if (!new QueryTestValue(prevVal.val1 + 1).equals(e.getValue()))
+                    if (localQry) {
+                        if (prevVal.val1 > e.getValue().val1)
+                            fail = true;
+                    }
+                    else if (!new QueryTestValue(prevVal.val1 + 1).equals(e.getValue()))
                         fail = true;
                 }
             }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
index 6b2fea0a5ab55..3767bafe97199 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchAckTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchForceServerModeAckTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFilterListenerTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryClockModeConflictSelfTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryExecuteInPrimaryTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryAsyncFilterRandomOperationTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryFilterRandomOperationTest;
@@ -125,6 +126,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheContinuousQueryBackupQueueTest.class);
         suite.addTestSuite(IgniteCacheContinuousQueryNoUnsubscribeTest.class);
         suite.addTestSuite(ContinuousQueryPeerClassLoadingTest.class);
+        suite.addTestSuite(CacheContinuousQueryClockModeConflictSelfTest.class);
 
         return suite;
     }

From 5858efd406bb54352de14a0a7e7f21c2ac7bf899 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 16 Dec 2016 19:23:29 +0300
Subject: [PATCH 304/446] IGNITE-2412 - Do not acquire asyncSemaphore for
 synchronous operations (cherry-picked from master)

(cherry picked from commit 82b4073)
---
 .../processors/cache/GridCacheAdapter.java    | 703 +++++++-----------
 .../dht/atomic/GridDhtAtomicCache.java        | 469 +++++++-----
 .../dht/colocated/GridDhtColocatedCache.java  |  13 -
 .../distributed/near/GridNearAtomicCache.java |  65 +-
 .../local/atomic/GridLocalAtomicCache.java    | 173 +----
 ...bledMultiNodeLongTxTimeoutFullApiTest.java |   2 +-
 ...ckMessageSystemPoolStarvationSelfTest.java |  13 +-
 7 files changed, 596 insertions(+), 842 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index ac1d2682d5f2e..47c9998239777 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -248,16 +248,16 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
 
     /** Grid configuration. */
     @GridToStringExclude
-    protected IgniteConfiguration gridCfg;
+    private IgniteConfiguration gridCfg;
 
     /** Cache metrics. */
     protected CacheMetricsImpl metrics;
 
     /** Cache localMxBean. */
-    protected CacheMetricsMXBean localMxBean;
+    private CacheMetricsMXBean locMxBean;
 
     /** Cache mxBean. */
-    protected CacheMetricsMXBean clusterMxBean;
+    private CacheMetricsMXBean clusterMxBean;
 
     /** Logger. */
     protected IgniteLogger log;
@@ -331,7 +331,7 @@ protected GridCacheAdapter(final GridCacheContext<K, V> ctx, @Nullable GridCache
 
         metrics = new CacheMetricsImpl(ctx);
 
-        localMxBean = new CacheLocalMetricsMXBeanImpl(this);
+        locMxBean = new CacheLocalMetricsMXBeanImpl(this);
         clusterMxBean = new CacheClusterMetricsMXBeanImpl(this);
 
         FileSystemConfiguration[] igfsCfgs = gridCfg.getFileSystemConfiguration();
@@ -456,49 +456,49 @@ public boolean isDht() {
     public abstract GridCachePreloader preloader();
 
     /** {@inheritDoc} */
-    @Override public Affinity<K> affinity() {
+    @Override public final Affinity<K> affinity() {
         return aff;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings({"unchecked", "RedundantCast"})
-    @Override public <K1, V1> IgniteInternalCache<K1, V1> cache() {
+    @Override public final <K1, V1> IgniteInternalCache<K1, V1> cache() {
         return (IgniteInternalCache<K1, V1>)this;
     }
 
     /** {@inheritDoc} */
-    @Override public GridCacheProxyImpl<K, V> forSubjectId(UUID subjId) {
+    @Override public final GridCacheProxyImpl<K, V> forSubjectId(UUID subjId) {
         CacheOperationContext opCtx = new CacheOperationContext(false, subjId, false, null, false, null);
 
         return new GridCacheProxyImpl<>(ctx, this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Override public boolean skipStore() {
+    @Override public final boolean skipStore() {
         return false;
     }
 
     /** {@inheritDoc} */
-    @Override public GridCacheProxyImpl<K, V> setSkipStore(boolean skipStore) {
+    @Override public final GridCacheProxyImpl<K, V> setSkipStore(boolean skipStore) {
         CacheOperationContext opCtx = new CacheOperationContext(true, null, false, null, false, null);
 
         return new GridCacheProxyImpl<>(ctx, this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Override public <K1, V1> GridCacheProxyImpl<K1, V1> keepBinary() {
+    @Override public final <K1, V1> GridCacheProxyImpl<K1, V1> keepBinary() {
         CacheOperationContext opCtx = new CacheOperationContext(false, null, true, null, false, null);
 
         return new GridCacheProxyImpl<>((GridCacheContext<K1, V1>)ctx, (GridCacheAdapter<K1, V1>)this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public ExpiryPolicy expiry() {
+    @Nullable @Override public final ExpiryPolicy expiry() {
         return null;
     }
 
     /** {@inheritDoc} */
-    @Override public GridCacheProxyImpl<K, V> withExpiryPolicy(ExpiryPolicy plc) {
+    @Override public final GridCacheProxyImpl<K, V> withExpiryPolicy(ExpiryPolicy plc) {
         assert !CU.isUtilityCache(ctx.name());
         assert !CU.isAtomicsCache(ctx.name());
         assert !CU.isMarshallerCache(ctx.name());
@@ -509,14 +509,14 @@ public boolean isDht() {
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalCache<K, V> withNoRetries() {
+    @Override public final IgniteInternalCache<K, V> withNoRetries() {
         CacheOperationContext opCtx = new CacheOperationContext(false, null, false, null, true, null);
 
         return new GridCacheProxyImpl<>(ctx, this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Override public CacheConfiguration configuration() {
+    @Override public final CacheConfiguration configuration() {
         return ctx.config();
     }
 
@@ -617,7 +617,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean isEmpty() {
+    @Override public final boolean isEmpty() {
         try {
             return localSize(CachePeekModes.ONHEAP_ONLY) == 0;
         }
@@ -627,7 +627,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean containsKey(K key) {
+    @Override public final boolean containsKey(K key) {
         try {
             return containsKeyAsync(key).get();
         }
@@ -654,7 +654,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean containsKeys(Collection<? extends K> keys) {
+    @Override public final boolean containsKeys(Collection<? extends K> keys) {
         try {
             return containsKeysAsync(keys).get();
         }
@@ -664,7 +664,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> containsKeysAsync(final Collection<? extends K> keys) {
+    @Override public final IgniteInternalFuture<Boolean> containsKeysAsync(final Collection<? extends K> keys) {
         A.notNull(keys, "keys");
 
         return getAllAsync(
@@ -695,7 +695,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public Iterable<Cache.Entry<K, V>> localEntries(CachePeekMode[] peekModes) throws IgniteCheckedException {
+    @Override public final Iterable<Cache.Entry<K, V>> localEntries(CachePeekMode[] peekModes) throws IgniteCheckedException {
         assert peekModes != null;
 
         ctx.checkSecurity(SecurityPermission.CACHE_READ);
@@ -752,7 +752,7 @@ public String toString() {
 
     /** {@inheritDoc} */
     @SuppressWarnings("ForLoopReplaceableByForEach")
-    @Nullable @Override public V localPeek(K key,
+    @Nullable @Override public final V localPeek(K key,
         CachePeekMode[] peekModes,
         @Nullable IgniteCacheExpiryPolicy plc)
         throws IgniteCheckedException {
@@ -894,7 +894,7 @@ public String toString() {
      *
      * @param ldr Class loader to undeploy.
      */
-    public void onUndeploy(ClassLoader ldr) {
+    public final void onUndeploy(ClassLoader ldr) {
         ctx.deploy().onUndeploy(ldr, context());
     }
 
@@ -903,7 +903,7 @@ public void onUndeploy(ClassLoader ldr) {
      * @param key Entry key.
      * @return Entry or <tt>null</tt>.
      */
-    @Nullable public GridCacheEntryEx peekEx(KeyCacheObject key) {
+    @Nullable public final GridCacheEntryEx peekEx(KeyCacheObject key) {
         return entry0(key, ctx.affinity().affinityTopologyVersion(), false, false);
     }
 
@@ -912,7 +912,7 @@ public void onUndeploy(ClassLoader ldr) {
      * @param key Entry key.
      * @return Entry or <tt>null</tt>.
      */
-    @Nullable public GridCacheEntryEx peekEx(Object key) {
+    @Nullable public final GridCacheEntryEx peekEx(Object key) {
         return entry0(ctx.toCacheKeyObject(key), ctx.affinity().affinityTopologyVersion(), false, false);
     }
 
@@ -920,7 +920,7 @@ public void onUndeploy(ClassLoader ldr) {
      * @param key Entry key.
      * @return Entry (never {@code null}).
      */
-    public GridCacheEntryEx entryEx(Object key) {
+    public final GridCacheEntryEx entryEx(Object key) {
         return entryEx(ctx.toCacheKeyObject(key), false);
     }
 
@@ -928,7 +928,7 @@ public GridCacheEntryEx entryEx(Object key) {
      * @param key Entry key.
      * @return Entry (never {@code null}).
      */
-    public GridCacheEntryEx entryEx(KeyCacheObject key) {
+    public final GridCacheEntryEx entryEx(KeyCacheObject key) {
         return entryEx(key, false);
     }
 
@@ -983,24 +983,24 @@ public GridCacheEntryEx entryEx(KeyCacheObject key, AffinityTopologyVersion topV
     /**
      * @return Set of internal cached entry representations.
      */
-    public Iterable<? extends GridCacheEntryEx> entries() {
+    public final Iterable<? extends GridCacheEntryEx> entries() {
         return allEntries();
     }
 
     /**
      * @return Set of internal cached entry representations.
      */
-    public Iterable<? extends GridCacheEntryEx> allEntries() {
+    public final Iterable<? extends GridCacheEntryEx> allEntries() {
         return map.entries();
     }
 
     /** {@inheritDoc} */
-    @Override public Set<Cache.Entry<K, V>> entrySet() {
+    @Override public final Set<Cache.Entry<K, V>> entrySet() {
         return entrySet((CacheEntryPredicate[])null);
     }
 
     /** {@inheritDoc} */
-    @Override public Set<Cache.Entry<K, V>> entrySetx(final CacheEntryPredicate... filter) {
+    @Override public final Set<Cache.Entry<K, V>> entrySetx(final CacheEntryPredicate... filter) {
         boolean keepBinary = ctx.keepBinary();
 
         return new EntrySet(map.entrySet(filter), keepBinary);
@@ -1012,17 +1012,17 @@ public Iterable<? extends GridCacheEntryEx> allEntries() {
     }
 
     /** {@inheritDoc} */
-    @Override public Set<K> keySet() {
+    @Override public final Set<K> keySet() {
         return new KeySet(map.entrySet());
     }
 
     /** {@inheritDoc} */
-    @Override public Set<K> keySetx() {
+    @Override public final Set<K> keySetx() {
         return keySet();
     }
 
     /** {@inheritDoc} */
-    @Override public Set<K> primaryKeySet() {
+    @Override public final Set<K> primaryKeySet() {
         return new KeySet(map.entrySet(CU.cachePrimary(ctx.grid().affinity(ctx.name()), ctx.localNode())));
     }
 
@@ -1044,7 +1044,7 @@ public Iterable<? extends GridCacheEntryEx> allEntries() {
      * @param filter Filters.
      * @return Collection of cached values.
      */
-    public Iterable<V> values(final CacheEntryPredicate... filter) {
+    public final Iterable<V> values(final CacheEntryPredicate... filter) {
         return new Iterable<V>() {
             @Override public Iterator<V> iterator() {
                 return new Iterator<V>() {
@@ -1070,12 +1070,12 @@ public Iterable<V> values(final CacheEntryPredicate... filter) {
      *
      * @param key Entry key.
      */
-    public void removeIfObsolete(KeyCacheObject key) {
+    public final void removeIfObsolete(KeyCacheObject key) {
         assert key != null;
 
         GridCacheMapEntry entry = map.getEntry(key);
 
-        if (entry.obsolete())
+        if (entry != null && entry.obsolete())
             removeEntry(entry);
     }
 
@@ -1259,11 +1259,11 @@ public void clearLocally(Collection<KeyCacheObject> keys, boolean readers) {
     /**
      * @param entry Removes entry from cache if currently mapped value is the same as passed.
      */
-    public void removeEntry(GridCacheEntryEx entry) {
-        boolean removed = map.removeEntry(entry);
+    public final void removeEntry(GridCacheEntryEx entry) {
+        boolean rmvd = map.removeEntry(entry);
 
         if (log.isDebugEnabled()) {
-            if (removed)
+            if (rmvd)
                 log.debug("Removed entry from cache: " + entry);
             else
                 log.debug("Remove will not be done for key (entry got replaced or removed): " + entry.key());
@@ -1298,7 +1298,7 @@ private boolean evictx(K key, GridCacheVersion ver,
     }
 
     /** {@inheritDoc} */
-    @Override public V getForcePrimary(K key) throws IgniteCheckedException {
+    @Override public final V getForcePrimary(K key) throws IgniteCheckedException {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(
@@ -1315,7 +1315,7 @@ private boolean evictx(K key, GridCacheVersion ver,
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getForcePrimaryAsync(final K key) {
+    @Override public final IgniteInternalFuture<V> getForcePrimaryAsync(final K key) {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(
@@ -1336,7 +1336,7 @@ private boolean evictx(K key, GridCacheVersion ver,
     }
 
     /** {@inheritDoc} */
-    public V getTopologySafe(K key) throws IgniteCheckedException {
+    public final V getTopologySafe(K key) throws IgniteCheckedException {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(
@@ -1353,12 +1353,12 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public Map<K, V> getAllOutTx(Set<? extends K> keys) throws IgniteCheckedException {
+    @Nullable @Override public final Map<K, V> getAllOutTx(Set<? extends K> keys) throws IgniteCheckedException {
         return getAllOutTxAsync(keys).get();
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Map<K, V>> getAllOutTxAsync(Set<? extends K> keys) {
+    @Override public final IgniteInternalFuture<Map<K, V>> getAllOutTxAsync(Set<? extends K> keys) {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(keys,
@@ -1372,15 +1372,6 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
             false);
     }
 
-    /**
-     * @param key Key.
-     * @param topVer Topology version.
-     * @return Entry.
-     */
-    @Nullable protected GridCacheEntryEx entryExSafe(KeyCacheObject key, AffinityTopologyVersion topVer) {
-        return entryEx(key);
-    }
-
     /** {@inheritDoc} */
     @Nullable @Override public V get(K key) throws IgniteCheckedException {
         A.notNull(key, "key");
@@ -1521,14 +1512,14 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
     }
 
     /** {@inheritDoc} */
-    @Override public Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
+    @Override public final Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
         boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         long start = statsEnabled ? System.nanoTime() : 0L;
 
-        Map<K, V> map = getAll(keys, !ctx.keepBinary(), false);
+        Map<K, V> map = getAll0(keys, !ctx.keepBinary(), false);
 
         if (ctx.config().getInterceptor() != null)
             map = interceptGet(keys, map);
@@ -1548,7 +1539,7 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
 
         long start = statsEnabled ? System.nanoTime() : 0L;
 
-        Map<K, EntryGetResult> map = (Map<K, EntryGetResult>)getAll(keys, !ctx.keepBinary(), true);
+        Map<K, EntryGetResult> map = (Map<K, EntryGetResult>)getAll0(keys, !ctx.keepBinary(), true);
 
         Collection<CacheEntry<K, V>> res = new HashSet<>();
 
@@ -2156,7 +2147,7 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndPut(K key, V val) throws IgniteCheckedException {
+    @Override public final V getAndPut(K key, V val) throws IgniteCheckedException {
         return getAndPut(key, val, null);
     }
 
@@ -2178,7 +2169,24 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
         if (keyCheck)
             validateCacheKey(key);
 
-        V prevVal = syncOp(new SyncOp<V>(true) {
+        V prevVal = getAndPut0(key, val, filter);
+
+        if (statsEnabled)
+            metrics0().addPutAndGetTimeNanos(System.nanoTime() - start);
+
+        return prevVal;
+    }
+
+    /**
+     * @param key Key.
+     * @param val Value.
+     * @param filter Optional filter.
+     * @return Previous value.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected V getAndPut0(final K key, final V val, @Nullable final CacheEntryPredicate filter)
+        throws IgniteCheckedException {
+        return syncOp(new SyncOp<V>(true) {
             @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return (V)tx.putAsync(ctx, null, key, val, true, filter).get().value();
             }
@@ -2187,15 +2195,10 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                 return "put [key=" + key + ", val=" + val + ", filter=" + filter + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addPutAndGetTimeNanos(System.nanoTime() - start);
-
-        return prevVal;
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutAsync(K key, V val) {
+    @Override public final IgniteInternalFuture<V> getAndPutAsync(K key, V val) {
         return getAndPutAsync(key, val, null);
     }
 
@@ -2205,11 +2208,16 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
      * @param filter Filter.
      * @return Put operation future.
      */
-    public IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+    protected final IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
         final boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
 
+        A.notNull(key, "key", val, "val");
+
+        if (keyCheck)
+            validateCacheKey(key);
+
         IgniteInternalFuture<V> fut = getAndPutAsync0(key, val, filter);
 
         if (statsEnabled)
@@ -2224,13 +2232,10 @@ public IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntry
      * @param filter Optional filter.
      * @return Put operation future.
      */
-    public IgniteInternalFuture<V> getAndPutAsync0(final K key, final V val,
-        @Nullable final CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
+    public IgniteInternalFuture<V> getAndPutAsync0(final K key,
+        final V val,
+        @Nullable final CacheEntryPredicate filter)
+    {
         return asyncOp(new AsyncOp<V>() {
             @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAsync(ctx, readyTopVer, key, val, true, filter)
@@ -2244,7 +2249,7 @@ public IgniteInternalFuture<V> getAndPutAsync0(final K key, final V val,
     }
 
     /** {@inheritDoc} */
-    @Override public boolean put(final K key, final V val) throws IgniteCheckedException {
+    @Override public final boolean put(final K key, final V val) throws IgniteCheckedException {
         return put(key, val, null);
     }
 
@@ -2268,7 +2273,26 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
         if (keyCheck)
             validateCacheKey(key);
 
-        Boolean stored = syncOp(new SyncOp<Boolean>(true) {
+        boolean stored = put0(key, val, filter);
+
+        if (statsEnabled && stored)
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
+
+        return stored;
+    }
+
+    /**
+     * @param key Key.
+     * @param val Value.
+     * @param filter Filter.
+     * @return {@code True} if optional filter passed and value was stored in cache,
+     *      {@code false} otherwise. Note that this method will return {@code true} if filter is not
+     *      specified.
+     * @throws IgniteCheckedException If put operation failed.
+     */
+    protected boolean put0(final K key, final V val, final CacheEntryPredicate filter)
+        throws IgniteCheckedException {
+        Boolean res = syncOp(new SyncOp<Boolean>(true) {
             @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return tx.putAsync(ctx, null, key, val, false, filter).get().success();
             }
@@ -2278,10 +2302,9 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
             }
         });
 
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
+        assert res != null;
 
-        return stored;
+        return res;
     }
 
     /** {@inheritDoc} */
@@ -2323,7 +2346,7 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public <T> EntryProcessorResult<T> invoke(@Nullable AffinityTopologyVersion topVer,
+    @Nullable @Override public final <T> EntryProcessorResult<T> invoke(@Nullable AffinityTopologyVersion topVer,
         K key,
         EntryProcessor<K, V, T> entryProcessor,
         Object... args) throws IgniteCheckedException {
@@ -2556,7 +2579,7 @@ private <T> EntryProcessorResult<T> invoke0(
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putAsync(K key, V val) {
+    @Override public final IgniteInternalFuture<Boolean> putAsync(K key, V val) {
         return putAsync(key, val, null);
     }
 
@@ -2566,7 +2589,12 @@ private <T> EntryProcessorResult<T> invoke0(
      * @param filter Filter.
      * @return Put future.
      */
-    public IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+    public final IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+        A.notNull(key, "key", val, "val");
+
+        if (keyCheck)
+            validateCacheKey(key);
+
         final boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
@@ -2587,11 +2615,6 @@ public IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntry
      */
     public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         @Nullable final CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
         return asyncOp(new AsyncOp<Boolean>() {
             @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAsync(ctx,
@@ -2619,267 +2642,82 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public V getAndPutIfAbsent(final K key, final V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return syncOp(new SyncOp<V>(true) {
-            @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return (V)tx.putAsync(ctx, null, key, val, true, ctx.noVal()).get().value();
-            }
-
-            @Override public String toString() {
-                return "putIfAbsent [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Nullable @Override public final V getAndPutIfAbsent(final K key, final V val) throws IgniteCheckedException {
+        return getAndPut(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
-            @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx, readyTopVer, key, val, true, ctx.noVal())
-                    .chain((IgniteClosure<IgniteInternalFuture<GridCacheReturn>, V>)RET2VAL);
-            }
-
-            @Override public String toString() {
-                return "putIfAbsentAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
+    @Override public final IgniteInternalFuture<V> getAndPutIfAbsentAsync(final K key, final V val) {
+        return getAndPutAsync(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(final K key, final V val) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        Boolean stored = syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return tx.putAsync(ctx, null, key, val, false, ctx.noVal()).get().success();
-            }
-
-            @Override public String toString() {
-                return "putxIfAbsent [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled && stored)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
-
-        return stored;
+    @Override public final boolean putIfAbsent(final K key, final V val) throws IgniteCheckedException {
+        return put(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx,
-                    readyTopVer,
-                    key,
-                    val,
-                    false,
-                    ctx.noVal()).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "putxIfAbsentAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+    @Override public final IgniteInternalFuture<Boolean> putIfAbsentAsync(final K key, final V val) {
+        return putAsync(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public V getAndReplace(final K key, final V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return syncOp(new SyncOp<V>(true) {
-            @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return (V)tx.putAsync(ctx, null, key, val, true, ctx.hasVal()).get().value();
-            }
-
-            @Override public String toString() {
-                return "replace [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Nullable @Override public final V getAndReplace(final K key, final V val) throws IgniteCheckedException {
+        return getAndPut(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
-            @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx, readyTopVer, key, val, true, ctx.hasVal()).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, V>)RET2VAL);
-            }
-
-            @Override public String toString() {
-                return "replaceAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutAndGetTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
+    @Override public final IgniteInternalFuture<V> getAndReplaceAsync(final K key, final V val) {
+        return getAndPutAsync(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public boolean replace(final K key, final V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return tx.putAsync(ctx, null, key, val, false, ctx.hasVal()).get().success();
-            }
-
-            @Override public String toString() {
-                return "replacex [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Override public final boolean replace(final K key, final V val) throws IgniteCheckedException {
+        return put(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(final K key, final V val) {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx, readyTopVer, key, val, false, ctx.hasVal()).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>) RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "replacexAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Override public final IgniteInternalFuture<Boolean> replaceAsync(final K key, final V val) {
+        return putAsync(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public boolean replace(final K key, final V oldVal, final V newVal) throws IgniteCheckedException {
-        A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
-        if (keyCheck)
-            validateCacheKey(key);
+    @Override public final boolean replace(final K key, final V oldVal, final V newVal) throws IgniteCheckedException {
+        A.notNull(oldVal, "oldVal");
 
-        return syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled())
-                    ctx.deploy().registerClass(oldVal);
-
-                return tx.putAsync(ctx, null, key, newVal, false, ctx.equalsVal(oldVal)).get()
-                    .success();
-            }
-
-            @Override public String toString() {
-                return "replace [key=" + key + ", oldVal=" + oldVal + ", newVal=" + newVal + ']';
-            }
-        });
+        return put(key, newVal, ctx.equalsVal(oldVal));
     }
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<Boolean> replaceAsync(final K key, final V oldVal, final V newVal) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled()) {
-                    try {
-                        ctx.deploy().registerClass(oldVal);
-                    }
-                    catch (IgniteCheckedException e) {
-                        return new GridFinishedFuture<>(e);
-                    }
-                }
+        A.notNull(oldVal, "oldVal");
 
-                return tx.putAsync(ctx, readyTopVer, key, newVal, false, ctx.equalsVal(oldVal)).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "replaceAsync [key=" + key + ", oldVal=" + oldVal + ", newVal=" + newVal + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutAndGetTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+        return putAsync(key, newVal, ctx.equalsVal(oldVal));
     }
 
     /** {@inheritDoc} */
     @Override public void putAll(@Nullable final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        if (F.isEmpty(m))
+            return;
+
         boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         long start = statsEnabled ? System.nanoTime() : 0L;
 
-        if (F.isEmpty(m))
-            return;
-
         if (keyCheck)
             validateCacheKeys(m.keySet());
 
+        putAll0(m);
+
+        if (statsEnabled)
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
+    }
+
+    /**
+     * @param m Map.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void putAll0(final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
         syncOp(new SyncInOp(m.size() == 1) {
             @Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 tx.putAllAsync(ctx, null, m, false).get();
@@ -2889,9 +2727,6 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "putAll [map=" + m + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
@@ -2902,6 +2737,14 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKeys(m.keySet());
 
+        return putAllAsync0(m);
+    }
+
+    /**
+     * @param m Map.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<?> putAllAsync0(final Map<? extends K, ? extends V> m) {
         return asyncOp(new AsyncOp(m.keySet()) {
             @Override public IgniteInternalFuture<?> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAllAsync(ctx,
@@ -2927,11 +2770,25 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKey(key);
 
+        V prevVal = getAndRemove0(key);
+
+        if (statsEnabled)
+            metrics0().addRemoveAndGetTimeNanos(System.nanoTime() - start);
+
+        return prevVal;
+    }
+
+    /**
+     * @param key Key.
+     * @return Previous value.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected V getAndRemove0(final K key) throws IgniteCheckedException {
         final boolean keepBinary = ctx.keepBinary();
 
-        V prevVal = syncOp(new SyncOp<V>(true) {
+        return syncOp(new SyncOp<V>(true) {
             @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                K key0 = keepBinary ? (K)ctx.toCacheKeyObject(key) : key;
+                K key0 = keepBinary ? (K) ctx.toCacheKeyObject(key) : key;
 
                 V ret = tx.removeAllAsync(ctx,
                     null,
@@ -2941,9 +2798,9 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                     /*singleRmv*/false).get().value();
 
                 if (ctx.config().getInterceptor() != null) {
-                    K key = keepBinary ? (K)ctx.unwrapBinaryIfNeeded(key0, true, false) : key0;
+                    K key = keepBinary ? (K) ctx.unwrapBinaryIfNeeded(key0, true, false) : key0;
 
-                    return (V)ctx.config().getInterceptor().onBeforeRemove(new CacheEntryImpl(key, ret)).get2();
+                    return (V) ctx.config().getInterceptor().onBeforeRemove(new CacheEntryImpl(key, ret)).get2();
                 }
 
                 return ret;
@@ -2953,11 +2810,6 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "remove [key=" + key + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addRemoveAndGetTimeNanos(System.nanoTime() - start);
-
-        return prevVal;
     }
 
     /** {@inheritDoc} */
@@ -2971,7 +2823,20 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
+        IgniteInternalFuture<V> fut = getAndRemoveAsync0(key);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
+    }
+
+    /**
+     * @param key Key.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
+        return asyncOp(new AsyncOp<V>() {
             @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 // TODO should we invoke interceptor here?
                 return tx.removeAllAsync(ctx,
@@ -2986,11 +2851,6 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "removeAsync [key=" + key + ']';
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
     }
 
     /** {@inheritDoc} */
@@ -3023,6 +2883,17 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKeys(keys);
 
+        removeAll0(keys);
+
+        if (statsEnabled)
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+    }
+
+    /**
+     * @param keys Keys.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheckedException {
         syncOp(new SyncInOp(keys.size() == 1) {
             @Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 tx.removeAllAsync(ctx,
@@ -3037,24 +2908,34 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "removeAll [keys=" + keys + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<?> removeAllAsync(@Nullable final Collection<? extends K> keys) {
+        if (F.isEmpty(keys))
+            return new GridFinishedFuture<Object>();
+
         final boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
 
-        if (F.isEmpty(keys))
-            return new GridFinishedFuture<Object>();
-
         if (keyCheck)
             validateCacheKeys(keys);
 
-        IgniteInternalFuture<Object> fut = asyncOp(new AsyncOp(keys) {
+        IgniteInternalFuture<Object> fut = removeAllAsync0(keys);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+
+        return fut;
+    }
+
+    /**
+     * @param keys Keys.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<Object> removeAllAsync0(final Collection<? extends K> keys) {
+        return asyncOp(new AsyncOp(keys) {
             @Override public IgniteInternalFuture<?> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.removeAllAsync(ctx,
                     readyTopVer,
@@ -3068,15 +2949,20 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "removeAllAsync [keys=" + keys + ']';
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
-
-        return fut;
     }
 
     /** {@inheritDoc} */
     @Override public boolean remove(final K key) throws IgniteCheckedException {
+        return remove(key, (CacheEntryPredicate)null);
+    }
+
+    /**
+     * @param key Key.
+     * @param filter Filter.
+     * @return {@code True} if entry was removed.
+     * @throws IgniteCheckedException If failed.
+     */
+    public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
         boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         long start = statsEnabled ? System.nanoTime() : 0L;
@@ -3086,13 +2972,27 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKey(key);
 
-        boolean rmv = syncOp(new SyncOp<Boolean>(true) {
+        boolean rmv = remove0(key, filter);
+
+        if (statsEnabled && rmv)
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+
+        return rmv;
+    }
+
+    /**
+     * @param key Key.
+     * @return {@code True} if entry was removed.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected boolean remove0(final K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
+        Boolean res = syncOp(new SyncOp<Boolean>(true) {
             @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return tx.removeAllAsync(ctx,
                     null,
                     Collections.singletonList(key),
                     /*retval*/false,
-                    null,
+                    filter,
                     /*singleRmv*/true).get().success();
             }
 
@@ -3101,10 +3001,9 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
             }
         });
 
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+        assert res != null;
 
-        return rmv;
+        return res;
     }
 
     /** {@inheritDoc} */
@@ -3129,7 +3028,21 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
+        IgniteInternalFuture<Boolean> fut = removeAsync0(key, filter);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
+
+        return fut;
+    }
+
+    /**
+     * @param key Key.
+     * @param filter Filter.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<Boolean> removeAsync0(final K key, @Nullable final CacheEntryPredicate filter) {
+        return asyncOp(new AsyncOp<Boolean>() {
             @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.removeAllAsync(ctx,
                     readyTopVer,
@@ -3144,11 +3057,6 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
                 return "removeAsync [key=" + key + ", filter=" + filter + ']';
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
     }
 
     /** {@inheritDoc} */
@@ -3190,86 +3098,21 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
     }
 
     /** {@inheritDoc} */
-    @Override public boolean remove(final K key, final V val) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
+    @Override public final boolean remove(final K key, final V val) throws IgniteCheckedException {
+        A.notNull(val, "val");
 
-        if (keyCheck)
-            validateCacheKey(key);
-
-        boolean rmv = syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled())
-                    ctx.deploy().registerClass(val);
-
-                return tx.removeAllAsync(ctx,
-                    null,
-                    Collections.singletonList(key),
-                    /*retval*/false,
-                    ctx.equalsVal(val),
-                    /*singleRmv*/false).get().success();
-            }
-
-            @Override public String toString() {
-                return "remove [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
-
-        return rmv;
+        return remove(key, ctx.equalsVal(val));
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
+    @Override public final IgniteInternalFuture<Boolean> removeAsync(final K key, final V val) {
+        A.notNull(key, "val");
 
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled()) {
-                    try {
-                        ctx.deploy().registerClass(val);
-                    }
-                    catch (IgniteCheckedException e) {
-                        return new GridFinishedFuture<>(e);
-                    }
-                }
-
-                return tx.removeAllAsync(ctx,
-                    readyTopVer,
-                    Collections.singletonList(key),
-                    /*retval*/false,
-                    ctx.equalsVal(val),
-                    /*singleRmv*/false).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "removeAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+        return removeAsync(key, ctx.equalsVal(val));
     }
 
     /** {@inheritDoc} */
-    @Override public CacheMetrics clusterMetrics() {
+    @Override public final CacheMetrics clusterMetrics() {
         return clusterMetrics(ctx.grid().cluster().forDataNodes(ctx.name()));
     }
 
@@ -3298,7 +3141,7 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
 
     /** {@inheritDoc} */
     @Override public CacheMetricsMXBean localMxBean() {
-        return localMxBean;
+        return locMxBean;
     }
 
     /** {@inheritDoc} */
@@ -4647,7 +4490,7 @@ protected void saveFuture(final FutureHolder holder, IgniteInternalFuture<?> fut
     /**
      * Releases asynchronous operations permit, if limited.
      */
-    protected void asyncOpRelease() {
+    private void asyncOpRelease() {
         if (asyncOpsSem != null)
             asyncOpsSem.release();
     }
@@ -4813,12 +4656,10 @@ public Set<Cache.Entry<K, V>> entrySet(@Nullable CacheEntryPredicate... filter)
      * @return Cached value.
      * @throws IgniteCheckedException If failed.
      */
-    @Nullable public V get(K key, boolean deserializeBinary, final boolean needVer) throws IgniteCheckedException {
-        checkJta();
-
+    @Nullable public final V get(K key, boolean deserializeBinary, final boolean needVer) throws IgniteCheckedException {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
-        return get(key, taskName, deserializeBinary, needVer);
+        return get0(key, taskName, deserializeBinary, needVer);
     }
 
     /**
@@ -4829,11 +4670,13 @@ public Set<Cache.Entry<K, V>> entrySet(@Nullable CacheEntryPredicate... filter)
      * @return Cached value.
      * @throws IgniteCheckedException If failed.
      */
-    protected V get(
+    protected V get0(
         final K key,
         String taskName,
         boolean deserializeBinary,
         boolean needVer) throws IgniteCheckedException {
+        checkJta();
+
         try {
             return getAsync(key,
                 !ctx.config().isReadFromBackup(),
@@ -4887,7 +4730,7 @@ public final IgniteInternalFuture<V> getAsync(final K key, boolean deserializeBi
      * @return Map of cached values.
      * @throws IgniteCheckedException If read failed.
      */
-    public Map<K, V> getAll(Collection<? extends K> keys, boolean deserializeBinary,
+    protected Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary,
         boolean needVer) throws IgniteCheckedException {
         checkJta();
 
@@ -4942,7 +4785,7 @@ public void forceKeyCheck() {
      * @param key Cache key.
      * @throws IllegalArgumentException If validation fails.
      */
-    protected void validateCacheKey(Object key) {
+    protected final void validateCacheKey(Object key) {
         if (keyCheck) {
             CU.validateCacheKey(key);
 
@@ -4957,7 +4800,7 @@ protected void validateCacheKey(Object key) {
      * @param keys Cache keys.
      * @throws IgniteException If validation fails.
      */
-    protected void validateCacheKeys(Iterable<?> keys) {
+    protected final void validateCacheKeys(Iterable<?> keys) {
         if (keys == null)
             return;
 
@@ -4978,7 +4821,7 @@ protected void validateCacheKeys(Iterable<?> keys) {
      * @param deserializeBinary Deserialize binary flag.
      * @return Public API iterator.
      */
-    protected Iterator<Cache.Entry<K, V>> iterator(final Iterator<? extends GridCacheEntryEx> it,
+    protected final Iterator<Cache.Entry<K, V>> iterator(final Iterator<? extends GridCacheEntryEx> it,
         final boolean deserializeBinary) {
         return new Iterator<Cache.Entry<K, V>>() {
             {
@@ -5296,7 +5139,7 @@ private static PeekModes parsePeekModes(CachePeekMode[] peekModes, boolean prima
      * @param plc Explicitly specified expiry policy for cache operation.
      * @return Expiry policy wrapper.
      */
-    @Nullable public IgniteCacheExpiryPolicy expiryPolicy(@Nullable ExpiryPolicy plc) {
+    @Nullable public final IgniteCacheExpiryPolicy expiryPolicy(@Nullable ExpiryPolicy plc) {
         if (plc == null)
             plc = ctx.expiry();
 
@@ -5421,7 +5264,7 @@ public IgniteInternalFuture<T> op(final IgniteTxLocalAdapter tx, CacheOperationC
          * @param opCtx Operation context.
          * @return Operation future.
          */
-        protected IgniteInternalFuture<T> waitTopologyFuture(IgniteInternalFuture<?> topFut,
+        private IgniteInternalFuture<T> waitTopologyFuture(IgniteInternalFuture<?> topFut,
             final AffinityTopologyVersion topVer,
             final IgniteTxLocalAdapter tx,
             final CacheOperationContext opCtx) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 72e1bb1cc3080..dc6d3dd8271dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -140,7 +140,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
     private CI2<GridNearAtomicAbstractUpdateRequest, GridNearAtomicUpdateResponse> updateReplyClos;
 
     /** Pending */
-    private GridDeferredAckMessageSender deferredUpdateMessageSender;
+    private GridDeferredAckMessageSender deferredUpdateMsgSnd;
 
     /** */
     private GridNearAtomicCache<K, V> near;
@@ -174,6 +174,11 @@ public GridDhtAtomicCache(GridCacheContext<K, V> ctx, GridCacheConcurrentMap map
         msgLog = ctx.shared().atomicMessageLogger();
     }
 
+    /** {@inheritDoc} */
+    @Override protected void checkJta() throws IgniteCheckedException {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isDhtAtomic() {
         return true;
@@ -236,7 +241,7 @@ else if (res.error() != null) {
     @Override public void start() throws IgniteCheckedException {
         super.start();
 
-        deferredUpdateMessageSender = new GridDeferredAckMessageSender(ctx.time(), ctx.closures()) {
+        deferredUpdateMsgSnd = new GridDeferredAckMessageSender(ctx.time(), ctx.closures()) {
             @Override public int getTimeout() {
                 return DEFERRED_UPDATE_RESPONSE_TIMEOUT;
             }
@@ -448,7 +453,7 @@ else if (res.error() != null) {
 
     /** {@inheritDoc} */
     @Override public void stop() {
-        deferredUpdateMessageSender.stop();
+        deferredUpdateMsgSnd.stop();
     }
 
     /**
@@ -464,7 +469,7 @@ public void near(GridNearAtomicCache<K, V> near) {
     }
 
     /** {@inheritDoc} */
-    @Override protected V get(K key, String taskName, boolean deserializeBinary, boolean needVer)
+    @Override protected V get0(K key, String taskName, boolean deserializeBinary, boolean needVer)
         throws IgniteCheckedException {
         ctx.checkSecurity(SecurityPermission.CACHE_READ);
 
@@ -540,6 +545,21 @@ public void near(GridNearAtomicCache<K, V> near) {
         });
     }
 
+    /** {@inheritDoc} */
+    @Override protected Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
+        throws IgniteCheckedException {
+        return getAllAsyncInternal(keys,
+            !ctx.config().isReadFromBackup(),
+            true,
+            null,
+            ctx.kernalContext().job().currentTaskName(),
+            deserializeBinary,
+            false,
+            true,
+            needVer,
+            false).get();
+    }
+
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<Map<K, V>> getAllAsync(
         @Nullable final Collection<? extends K> keys,
@@ -551,6 +571,43 @@ public void near(GridNearAtomicCache<K, V> near) {
         final boolean skipVals,
         final boolean canRemap,
         final boolean needVer
+    ) {
+        return getAllAsyncInternal(keys,
+            forcePrimary,
+            skipTx,
+            subjId,
+            taskName,
+            deserializeBinary,
+            skipVals,
+            canRemap,
+            needVer,
+            true);
+    }
+
+    /**
+     * @param keys Keys.
+     * @param forcePrimary Force primary flag.
+     * @param skipTx Skip tx flag.
+     * @param subjId Subject ID.
+     * @param taskName Task name.
+     * @param deserializeBinary Deserialize binary flag.
+     * @param skipVals Skip values flag.
+     * @param canRemap Can remap flag.
+     * @param needVer Need version flag.
+     * @param asyncOp Async operation flag.
+     * @return Future.
+     */
+    private IgniteInternalFuture<Map<K, V>> getAllAsyncInternal(
+        @Nullable final Collection<? extends K> keys,
+        final boolean forcePrimary,
+        boolean skipTx,
+        @Nullable UUID subjId,
+        final String taskName,
+        final boolean deserializeBinary,
+        final boolean skipVals,
+        final boolean canRemap,
+        final boolean needVer,
+        boolean asyncOp
     ) {
         ctx.checkSecurity(SecurityPermission.CACHE_READ);
 
@@ -562,7 +619,7 @@ public void near(GridNearAtomicCache<K, V> near) {
 
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
-        subjId = ctx.subjectIdPerCall(null, opCtx);
+        subjId = ctx.subjectIdPerCall(subjId, opCtx);
 
         final UUID subjId0 = subjId;
 
@@ -570,44 +627,77 @@ public void near(GridNearAtomicCache<K, V> near) {
 
         final boolean skipStore = opCtx != null && opCtx.skipStore();
 
-        return asyncOp(new CO<IgniteInternalFuture<Map<K, V>>>() {
-            @Override public IgniteInternalFuture<Map<K, V>> apply() {
-                return getAllAsync0(ctx.cacheKeysView(keys),
-                    forcePrimary,
-                    subjId0,
-                    taskName,
-                    deserializeBinary,
-                    expiryPlc,
-                    skipVals,
-                    skipStore,
-                    canRemap,
-                    needVer);
-            }
-        });
+        if (asyncOp) {
+            return asyncOp(new CO<IgniteInternalFuture<Map<K, V>>>() {
+                @Override public IgniteInternalFuture<Map<K, V>> apply() {
+                    return getAllAsync0(ctx.cacheKeysView(keys),
+                        forcePrimary,
+                        subjId0,
+                        taskName,
+                        deserializeBinary,
+                        expiryPlc,
+                        skipVals,
+                        skipStore,
+                        canRemap,
+                        needVer);
+                }
+            });
+        }
+        else {
+            return getAllAsync0(ctx.cacheKeysView(keys),
+                forcePrimary,
+                subjId0,
+                taskName,
+                deserializeBinary,
+                expiryPlc,
+                skipVals,
+                skipStore,
+                canRemap,
+                needVer);
+        }
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndPut(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
-        return getAndPutAsync0(key, val, filter).get();
+    @Override protected V getAndPut0(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
+        return (V)update0(
+            key,
+            val,
+            null,
+            null,
+            true,
+            filter,
+            true,
+            false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public boolean put(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
-        return putAsync(key, val, filter).get();
+    @Override protected boolean put0(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
+        Boolean res = (Boolean)update0(
+            key,
+            val,
+            null,
+            null,
+            false,
+            filter,
+            true,
+            false).get();
+
+        assert res != null;
+
+        return res;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<V> getAndPutAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
-        return updateAsync0(
+        return update0(
             key,
             val,
             null,
             null,
             true,
             filter,
+            true,
             true);
     }
 
@@ -616,13 +706,14 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
         A.notNull(key, "key", val, "val");
 
-        return updateAsync0(
+        return update0(
             key,
             val,
             null,
             null,
             false,
             filter,
+            true,
             true);
     }
 
@@ -630,84 +721,34 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public V tryGetAndPut(K key, V val) throws IgniteCheckedException {
         A.notNull(key, "key", val, "val");
 
-        return (V)updateAsync0(
+        return (V) update0(
             key,
             val,
             null,
             null,
             true,
             null,
+            false,
             false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
-        return getAndPutIfAbsentAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return getAndPutAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
-        return putIfAbsentAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return putAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
-        return getAndReplaceAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return getAndPutAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V val) throws IgniteCheckedException {
-        return replaceAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return putAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
-        return replaceAsync(key, oldVal, newVal).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
-        A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
-        return putAsync(key, newVal, ctx.equalsVal(oldVal));
-    }
-
-    /** {@inheritDoc} */
-    @Override public void putAll(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
-        putAllAsync(m).get();
+    @Override protected void putAll0(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        updateAll0(m,
+            null,
+            null,
+            null,
+            null,
+            false,
+            false,
+            true,
+            UPDATE,
+            false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> putAllAsync(Map<? extends K, ? extends V> m) {
-        return updateAllAsync0(m,
+    @Override public IgniteInternalFuture<?> putAllAsync0(Map<? extends K, ? extends V> m) {
+        return updateAll0(m,
             null,
             null,
             null,
@@ -715,7 +756,8 @@ public void near(GridNearAtomicCache<K, V> near) {
             false,
             false,
             true,
-            UPDATE).chain(RET2NULL);
+            UPDATE,
+            true).chain(RET2NULL);
     }
 
     /** {@inheritDoc} */
@@ -728,7 +770,7 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public IgniteInternalFuture<?> putAllConflictAsync(Map<KeyCacheObject, GridCacheDrInfo> conflictMap) {
         ctx.dr().onReceiveCacheEntriesReceived(conflictMap.size());
 
-        return updateAllAsync0(null,
+        return updateAll0(null,
             null,
             null,
             conflictMap,
@@ -736,57 +778,40 @@ public void near(GridNearAtomicCache<K, V> near) {
             false,
             false,
             true,
-            UPDATE);
+            UPDATE,
+            true);
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndRemove(K key) throws IgniteCheckedException {
-        return getAndRemoveAsync(key).get();
+    @Override public V getAndRemove0(K key) throws IgniteCheckedException {
+        return (V)remove0(key, true, null, false).get();
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<V> getAndRemoveAsync(K key) {
-        A.notNull(key, "key");
-
-        return removeAsync0(key, true, null);
+    @Override public IgniteInternalFuture<V> getAndRemoveAsync0(K key) {
+        return remove0(key, true, null, true);
     }
 
     /** {@inheritDoc} */
-    @Override public void removeAll(Collection<? extends K> keys) throws IgniteCheckedException {
-        removeAllAsync(keys).get();
+    @Override protected void removeAll0(Collection<? extends K> keys) throws IgniteCheckedException {
+        removeAllAsync0(keys, null, false, false, false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> removeAllAsync(Collection<? extends K> keys) {
-        A.notNull(keys, "keys");
-
-        return removeAllAsync0(keys, null, false, false).chain(RET2NULL);
+    @Override public IgniteInternalFuture<Object> removeAllAsync0(Collection<? extends K> keys) {
+        return removeAllAsync0(keys, null, false, false, true).chain(RET2NULL);
     }
 
     /** {@inheritDoc} */
-    @Override public boolean remove(K key) throws IgniteCheckedException {
-        return removeAsync(key, (CacheEntryPredicate)null).get();
+    @Override protected boolean remove0(K key, CacheEntryPredicate filter) throws IgniteCheckedException {
+        return (Boolean)remove0(key, false, filter, false).get();
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key");
-
-        return removeAsync0(key, false, filter);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean remove(K key, V val) throws IgniteCheckedException {
-        return removeAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return removeAsync(key, ctx.equalsVal(val));
+    @Override public IgniteInternalFuture<Boolean> removeAsync0(K key, @Nullable CacheEntryPredicate filter) {
+        return remove0(key, false, filter, true);
     }
 
     /** {@inheritDoc} */
@@ -799,7 +824,7 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public IgniteInternalFuture<?> removeAllConflictAsync(Map<KeyCacheObject, GridCacheVersion> conflictMap) {
         ctx.dr().onReceiveCacheEntriesReceived(conflictMap.size());
 
-        return removeAllAsync0(null, conflictMap, false, false);
+        return removeAllAsync0(null, conflictMap, false, false, true);
     }
 
     /**
@@ -814,7 +839,7 @@ private boolean writeThrough() {
      * @return Future.
      */
     @SuppressWarnings("unchecked")
-    protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>> op) {
+    private <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>> op) {
         IgniteInternalFuture<T> fail = asyncOpAcquire();
 
         if (fail != null)
@@ -872,7 +897,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     /** {@inheritDoc} */
     @Override public <T> EntryProcessorResult<T> invoke(K key, EntryProcessor<K, V, T> entryProcessor, Object... args)
         throws IgniteCheckedException {
-        IgniteInternalFuture<EntryProcessorResult<T>> invokeFut = invokeAsync(key, entryProcessor, args);
+        IgniteInternalFuture<EntryProcessorResult<T>> invokeFut = invoke0(false, key, entryProcessor, args);
 
         EntryProcessorResult<T> res = invokeFut.get();
 
@@ -882,14 +907,28 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     /** {@inheritDoc} */
     @Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(Set<? extends K> keys,
         EntryProcessor<K, V, T> entryProcessor,
-        Object... args)
-        throws IgniteCheckedException {
-        return invokeAllAsync(keys, entryProcessor, args).get();
+        Object... args) throws IgniteCheckedException
+    {
+        return invokeAll0(false, keys, entryProcessor, args).get();
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
     @Override public <T> IgniteInternalFuture<EntryProcessorResult<T>> invokeAsync(K key,
+        EntryProcessor<K, V, T> entryProcessor,
+        Object... args) {
+        return invoke0(true, key, entryProcessor, args);
+    }
+
+    /**
+     * @param async Async operation flag.
+     * @param key Key.
+     * @param entryProcessor Entry processor.
+     * @param args Entry processor arguments.
+     * @return Future.
+     */
+    private <T> IgniteInternalFuture<EntryProcessorResult<T>> invoke0(
+        boolean async,
+        K key,
         EntryProcessor<K, V, T> entryProcessor,
         Object... args) {
         A.notNull(key, "key", entryProcessor, "entryProcessor");
@@ -901,14 +940,15 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut = updateAsync0(
+        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut = update0(
             key,
             null,
             entryProcessor,
             args,
             false,
             null,
-            true);
+            true,
+            async);
 
         return fut.chain(new CX1<IgniteInternalFuture<Map<K, EntryProcessorResult<T>>>, EntryProcessorResult<T>>() {
             @Override public EntryProcessorResult<T> applyx(IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut)
@@ -939,6 +979,21 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public <T> IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> invokeAllAsync(Set<? extends K> keys,
+        final EntryProcessor<K, V, T> entryProcessor,
+        Object... args) {
+        return invokeAll0(true, keys, entryProcessor, args);
+    }
+
+    /**
+     * @param async Async operation flag.
+     * @param keys Keys.
+     * @param entryProcessor Entry processor.
+     * @param args Entry processor arguments.
+     * @return Future.
+     */
+    private <T> IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> invokeAll0(
+        boolean async,
+        Set<? extends K> keys,
         final EntryProcessor<K, V, T> entryProcessor,
         Object... args) {
         A.notNull(keys, "keys", entryProcessor, "entryProcessor");
@@ -956,7 +1011,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> resFut = updateAllAsync0(null,
+        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> resFut = updateAll0(null,
             invokeMap,
             args,
             null,
@@ -964,7 +1019,8 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
             false,
             false,
             true,
-            TRANSFORM);
+            TRANSFORM,
+            async);
 
         return resFut.chain(
             new CX1<IgniteInternalFuture<Map<K, EntryProcessorResult<T>>>, Map<K, EntryProcessorResult<T>>>() {
@@ -982,7 +1038,21 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     @Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(
         Map<? extends K, ? extends EntryProcessor<K, V, T>> map,
         Object... args) throws IgniteCheckedException {
-        return invokeAllAsync(map, args).get();
+        A.notNull(map, "map");
+
+        if (keyCheck)
+            validateCacheKeys(map.keySet());
+
+        return (Map<K, EntryProcessorResult<T>>)updateAll0(null,
+            map,
+            args,
+            null,
+            null,
+            false,
+            false,
+            true,
+            TRANSFORM,
+            false).get();
     }
 
     /** {@inheritDoc} */
@@ -995,7 +1065,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
         if (keyCheck)
             validateCacheKeys(map.keySet());
 
-        return updateAllAsync0(null,
+        return updateAll0(null,
             map,
             args,
             null,
@@ -1003,7 +1073,8 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
             false,
             false,
             true,
-            TRANSFORM);
+            TRANSFORM,
+            true);
     }
 
     /**
@@ -1017,10 +1088,11 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
      * @param retval Return value required flag.
      * @param rawRetval Return {@code GridCacheReturn} instance.
      * @param waitTopFut Whether to wait for topology future.
+     * @param async Async operation flag.
      * @return Completion future.
      */
     @SuppressWarnings("ConstantConditions")
-    private IgniteInternalFuture updateAllAsync0(
+    private IgniteInternalFuture updateAll0(
         @Nullable Map<? extends K, ? extends V> map,
         @Nullable Map<? extends K, ? extends EntryProcessor> invokeMap,
         @Nullable Object[] invokeArgs,
@@ -1029,7 +1101,8 @@ private IgniteInternalFuture updateAllAsync0(
         final boolean retval,
         final boolean rawRetval,
         final boolean waitTopFut,
-        final GridCacheOperation op
+        final GridCacheOperation op,
+        boolean async
     ) {
         assert ctx.updatesAllowed();
 
@@ -1106,13 +1179,20 @@ else if (op == GridCacheOperation.DELETE) {
             opCtx != null && opCtx.noRetries() ? 1 : MAX_RETRIES,
             waitTopFut);
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-                return updateFut;
-            }
-        });
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
+
+            return updateFut;
+        }
     }
 
     /**
@@ -1125,16 +1205,18 @@ else if (op == GridCacheOperation.DELETE) {
      * @param retval Return value flag.
      * @param filter Filter.
      * @param waitTopFut Whether to wait for topology future.
+     * @param async Async operation flag.
      * @return Future.
      */
-    private IgniteInternalFuture updateAsync0(
+    private IgniteInternalFuture update0(
         K key,
         @Nullable V val,
         @Nullable EntryProcessor proc,
         @Nullable Object[] invokeArgs,
         final boolean retval,
         @Nullable final CacheEntryPredicate filter,
-        final boolean waitTopFut
+        final boolean waitTopFut,
+        boolean async
     ) {
         assert val == null || proc == null;
 
@@ -1147,13 +1229,20 @@ private IgniteInternalFuture updateAsync0(
         final GridNearAtomicAbstractUpdateFuture updateFut =
             createSingleUpdateFuture(key, val, proc, invokeArgs, retval, filter, waitTopFut);
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-                return updateFut;
-            }
-        });
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
+
+            return updateFut;
+        }
     }
 
     /**
@@ -1162,33 +1251,38 @@ private IgniteInternalFuture updateAsync0(
      * @param key Key.
      * @param retval Whether to return
      * @param filter Filter.
+     * @param async Async operation flag.
      * @return Future.
      */
-    private IgniteInternalFuture removeAsync0(K key, final boolean retval,
-        @Nullable CacheEntryPredicate filter) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
+    private IgniteInternalFuture remove0(K key, final boolean retval,
+        @Nullable CacheEntryPredicate filter,
+        boolean async) {
         assert ctx.updatesAllowed();
 
-        validateCacheKey(key);
-
         ctx.checkSecurity(SecurityPermission.CACHE_REMOVE);
 
-        final GridNearAtomicAbstractUpdateFuture updateFut =
-            createSingleUpdateFuture(key, null, null, null, retval, filter, true);
+        final GridNearAtomicAbstractUpdateFuture updateFut = createSingleUpdateFuture(key,
+            null,
+            null,
+            null,
+            retval,
+            filter,
+            true);
 
-        if (statsEnabled)
-            updateFut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
 
-                return updateFut;
-            }
-        });
+            return updateFut;
+        }
     }
 
     /**
@@ -1327,14 +1421,11 @@ private IgniteInternalFuture removeAllAsync0(
         @Nullable Collection<? extends K> keys,
         @Nullable Map<KeyCacheObject, GridCacheVersion> conflictMap,
         final boolean retval,
-        boolean rawRetval
+        boolean rawRetval,
+        boolean async
     ) {
         assert ctx.updatesAllowed();
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         assert keys != null || conflictMap != null;
 
         if (keyCheck)
@@ -1381,16 +1472,20 @@ private IgniteInternalFuture removeAllAsync0(
             opCtx != null && opCtx.noRetries() ? 1 : MAX_RETRIES,
             true);
 
-        if (statsEnabled)
-            updateFut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
 
-                return updateFut;
-            }
-        });
+            return updateFut;
+        }
     }
 
     /**
@@ -3252,7 +3347,7 @@ && writeThrough() && !req.skipStore(),
      * @param ver Version to ack.
      */
     private void sendDeferredUpdateResponse(UUID nodeId, GridCacheVersion ver) {
-        deferredUpdateMessageSender.sendDeferredAckMessage(nodeId, ver);
+        deferredUpdateMsgSnd.sendDeferredAckMessage(nodeId, ver);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index ccdc51d2f6139..c9fc983d4850c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -328,19 +328,6 @@ public GridDistributedCacheEntry entryExx(
             needVer);
     }
 
-    /** {@inheritDoc} */
-    @Override protected GridCacheEntryEx entryExSafe(
-        KeyCacheObject key,
-        AffinityTopologyVersion topVer
-    ) {
-        try {
-            return ctx.affinity().localNode(key, topVer) ? entryEx(key) : null;
-        }
-        catch (GridDhtInvalidPartitionException ignored) {
-            return null;
-        }
-    }
-
     /**
      * @param keys Keys to load.
      * @param readThrough Read through flag.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index d1056fdff4284..b843e4e6d3db1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -449,61 +449,11 @@ public void processDhtAtomicUpdateRequest(
         return dht.putAsync0(key, val, filter);
     }
 
-    /** {@inheritDoc} */
-    @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
-        return dht.getAndPutIfAbsent(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
-        return dht.getAndPutIfAbsentAsync(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
-        return dht.putIfAbsent(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
-        return dht.putIfAbsentAsync(key, val);
-    }
-
     /** {@inheritDoc} */
     @Nullable @Override public V tryGetAndPut(K key, V val) throws IgniteCheckedException {
         return dht.tryGetAndPut(key, val);
     }
 
-    /** {@inheritDoc} */
-    @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
-        return dht.getAndReplace(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
-        return dht.getAndReplaceAsync(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V val) throws IgniteCheckedException {
-        return dht.replace(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
-        return dht.replaceAsync(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
-        return dht.replace(key, oldVal, newVal);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
-        return dht.replaceAsync(key, oldVal, newVal);
-    }
-
     /** {@inheritDoc} */
     @Override public void putAll(Map<? extends K, ? extends V> m)
         throws IgniteCheckedException {
@@ -568,6 +518,11 @@ public void processDhtAtomicUpdateRequest(
         return dht.invokeAllAsync(keys, entryProcessor, args);
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean remove(K key, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
+        return dht.remove(key, filter);
+    }
+
     /** {@inheritDoc} */
     @Override public V getAndRemove(K key) throws IgniteCheckedException {
         return dht.getAndRemove(key);
@@ -601,16 +556,6 @@ public void processDhtAtomicUpdateRequest(
         return dht.removeAsync(key, filter);
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean remove(K key, V val) throws IgniteCheckedException {
-        return dht.remove(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
-        return dht.removeAsync(key, val);
-    }
-
     /** {@inheritDoc} */
     @Override public void removeAll() throws IgniteCheckedException {
         dht.removeAll();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index 7da11b6b870be..f86df2f29d684 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -107,6 +107,11 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
         preldr = new GridCachePreloaderAdapter(ctx);
     }
 
+    /** {@inheritDoc} */
+    @Override protected void checkJta() throws IgniteCheckedException {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isLocal() {
         return true;
@@ -119,9 +124,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public V getAndPut(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
+    @Override protected V getAndPut0(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         return (V)updateAllInternal(UPDATE,
@@ -138,16 +141,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean put(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
+    @Override protected boolean put0(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
-        boolean res = (Boolean)updateAllInternal(UPDATE,
+        Boolean res = (Boolean)updateAllInternal(UPDATE,
             Collections.singleton(key),
             Collections.singleton(val),
             null,
@@ -159,8 +156,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
             ctx.readThrough(),
             opCtx != null && opCtx.isKeepBinary());
 
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
+        assert res != null;
 
         return res;
     }
@@ -168,8 +164,6 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<V> getAndPutAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
         return updateAllAsync0(F0.asMap(key, val),
             null,
             null,
@@ -181,8 +175,6 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
         return updateAllAsync0(F0.asMap(key, val),
             null,
             null,
@@ -192,65 +184,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
-        return getAndPut(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
-        return getAndPutAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
-        return put(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
-        return putAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
-        return getAndPut(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
-        return getAndPutAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V val) throws IgniteCheckedException {
-        return put(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
-        return putAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
-        A.notNull(oldVal, "oldVal");
-
-        return put(key, newVal, ctx.equalsVal(oldVal));
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
-        return putAsync(key, newVal, ctx.equalsVal(oldVal));
-    }
-
-    /** {@inheritDoc} */
-    @Override public void putAll(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
+    @Override protected void putAll0(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         updateAllInternal(UPDATE,
@@ -264,13 +198,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
             ctx.writeThrough(),
             ctx.readThrough(),
             opCtx != null && opCtx.isKeepBinary());
-
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> putAllAsync(Map<? extends K, ? extends V> m) {
+    @Override public IgniteInternalFuture<?> putAllAsync0(Map<? extends K, ? extends V> m) {
         return updateAllAsync0(m,
             null,
             null,
@@ -280,8 +211,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public V getAndRemove(K key) throws IgniteCheckedException {
+    @Override protected V getAndRemove0(K key) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         return (V)updateAllInternal(DELETE,
@@ -299,13 +229,13 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<V> getAndRemoveAsync(K key) {
+    @Override public IgniteInternalFuture<V> getAndRemoveAsync0(K key) {
         return removeAllAsync0(Collections.singletonList(key), true, false, null);
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public void removeAll(Collection<? extends K> keys) throws IgniteCheckedException {
+    @Override public void removeAll0(Collection<? extends K> keys) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         updateAllInternal(DELETE,
@@ -322,19 +252,13 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> removeAllAsync(Collection<? extends K> keys) {
+    @Override public IgniteInternalFuture<Object> removeAllAsync0(Collection<? extends K> keys) {
         return removeAllAsync0(keys, false, false, null).chain(RET2NULL);
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public boolean remove(K key) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key");
-
+    @Override public boolean remove0(K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         Boolean rmv = (Boolean)updateAllInternal(DELETE,
@@ -344,49 +268,22 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
             expiryPerCall(),
             false,
             false,
-            null,
+            filter,
             ctx.writeThrough(),
             ctx.readThrough(),
             opCtx != null && opCtx.isKeepBinary());
 
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+        assert rmv != null;
 
         return rmv;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key");
-
+    @Override public IgniteInternalFuture<Boolean> removeAsync0(K key, @Nullable CacheEntryPredicate filter) {
         return removeAllAsync0(Collections.singletonList(key), false, false, filter);
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean remove(K key, V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        CacheOperationContext opCtx = ctx.operationContextPerCall();
-
-        return (Boolean)updateAllInternal(DELETE,
-            Collections.singleton(key),
-            null,
-            null,
-            expiryPerCall(),
-            false,
-            false,
-            ctx.equalsVal(val),
-            ctx.writeThrough(),
-            ctx.readThrough(),
-            opCtx != null && opCtx.isKeepBinary());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
-        return removeAsync(key, ctx.equalsVal(val));
-    }
-
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<?> removeAllAsync() {
         return ctx.closures().callLocalSafe(new Callable<Void>() {
@@ -399,11 +296,13 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-
     @SuppressWarnings("unchecked")
-    @Override @Nullable public V get(K key, boolean deserializeBinary, boolean needVer) throws IgniteCheckedException {
-        String taskName = ctx.kernalContext().job().currentTaskName();
-
+    @Override protected V get0(
+        final K key,
+        String taskName,
+        boolean deserializeBinary,
+        boolean needVer) throws IgniteCheckedException
+    {
         Map<K, V> m = getAllInternal(Collections.singleton(key),
             ctx.isSwapOrOffheapEnabled(),
             ctx.readThrough(),
@@ -419,7 +318,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public final Map<K, V> getAll(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
+    @Override public final Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
         throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
@@ -795,7 +694,7 @@ private IgniteInternalFuture updateAllAsync0(
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture fut = asyncOp(new Callable<Object>() {
+        return asyncOp(new Callable<Object>() {
             @Override public Object call() throws Exception {
                 return updateAllInternal(op,
                     keys,
@@ -810,11 +709,6 @@ private IgniteInternalFuture updateAllAsync0(
                     keepBinary);
             }
         });
-
-        if (ctx.config().isStatisticsEnabled())
-            fut.listen(new UpdatePutTimeStatClosure(metrics0(), System.nanoTime()));
-
-        return fut;
     }
 
     /**
@@ -836,17 +730,13 @@ private IgniteInternalFuture removeAllAsync0(
 
         final boolean readThrough = ctx.readThrough();
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         final ExpiryPolicy expiryPlc = expiryPerCall();
 
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture fut = asyncOp(new Callable<Object>() {
+        return asyncOp(new Callable<Object>() {
             @Override public Object call() throws Exception {
                 return updateAllInternal(DELETE,
                     keys,
@@ -861,11 +751,6 @@ private IgniteInternalFuture removeAllAsync0(
                     keepBinary);
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
-
-        return fut;
     }
 
     /**
@@ -1586,7 +1471,7 @@ private static CachePartialUpdateCheckedException partialUpdateException() {
      * @return Future.
      */
     @SuppressWarnings("unchecked")
-    protected IgniteInternalFuture asyncOp(final Callable<?> op) {
+    private IgniteInternalFuture asyncOp(final Callable<?> op) {
         IgniteInternalFuture fail = asyncOpAcquire();
 
         if (fail != null)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
index 3e3b84e4403bf..648134e498c85 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
@@ -34,7 +34,7 @@ public class CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest extend
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
-        cfg.getTransactionConfiguration().setDefaultTxTimeout(Long.MAX_VALUE);
+        cfg.getTransactionConfiguration().setDefaultTxTimeout(5 * 60_000);
 
         return cfg;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
index ec3b8089cae6e..a235d8f15e3f6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteFileSystem;
@@ -39,14 +41,11 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.transactions.TransactionConcurrency;
-import org.apache.ignite.transactions.TransactionIsolation;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
 /**
  * Test to check for system pool starvation due to {@link IgfsBlocksMessage}.
@@ -125,8 +124,7 @@ public void testStarvation() throws Exception {
             @Override public Void call() throws Exception {
                 GridCacheAdapter dataCache = dataCache(attacker);
 
-                try (IgniteInternalTx tx =
-                         dataCache.txStartEx(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+                try (IgniteInternalTx tx = dataCache.txStartEx(PESSIMISTIC, REPEATABLE_READ)) {
                     dataCache.put(DATA_KEY, 0);
 
                     txStartLatch.countDown();
@@ -185,6 +183,7 @@ private static boolean awaitFuture(final IgniteInternalFuture fut) throws Except
      * Create IGFS file asynchronously.
      *
      * @param path Path.
+     * @param writeStartLatch Write start latch.
      * @return Future.
      */
     private IgniteInternalFuture<Void> createFileAsync(final IgfsPath path, final CountDownLatch writeStartLatch) {

From 0fc6271d8e39125bf5ee341e50a2665a04fc8b1e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Jun 2017 13:42:12 +0300
Subject: [PATCH 305/446] GG-12350: GridDhtAtomicSingleUpdateRequest misses
 topologyVersion() method override.

---
 .../dht/atomic/GridDhtAtomicSingleUpdateRequest.java         | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
index a03d948017f91..0838cb6dd87b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
@@ -360,6 +360,11 @@ public GridDhtAtomicSingleUpdateRequest() {
         return isFlag(KEEP_BINARY_FLAG_MASK);
     }
 
+    /** {@inheritDoc} */
+    @Override public AffinityTopologyVersion topologyVersion() {
+        return topVer;
+    }
+
     /**
      *
      */

From 113a1380da34ea804d68757d39926da97dee09b6 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 13 Jun 2017 08:20:22 +0300
Subject: [PATCH 306/446] GG-12355: Backported IO latency test.

---
 .../org/apache/ignite/internal/GridTopic.java |   5 +-
 .../apache/ignite/internal/IgniteKernal.java  |  16 +-
 .../managers/communication/GridIoManager.java | 699 +++++++++++++++++-
 .../communication/GridIoMessageFactory.java   |   5 +
 .../communication/IgniteIoTestMessage.java    | 594 +++++++++++++++
 .../apache/ignite/mxbean/IgniteMXBean.java    |  44 ++
 6 files changed, 1358 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 248f75b9d2689..dc20be002a6e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -94,7 +94,10 @@ public enum GridTopic {
     TOPIC_QUERY,
 
     /** */
-    TOPIC_TX;
+    TOPIC_TX,
+
+    /** */
+    TOPIC_IO_TEST;
 
     /** Enum values. */
     private static final GridTopic[] VALS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 25f7884c889b9..4237aeebe49ce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -3545,7 +3545,7 @@ private boolean skipDaemon(GridComponent comp) {
     }
 
     /** {@inheritDoc} */
-    public void dumpDebugInfo() {
+    @Override public void dumpDebugInfo() {
         try {
             GridKernalContextImpl ctx = this.ctx;
 
@@ -3615,6 +3615,20 @@ void waitPreviousReconnect() {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void runIoTest(
+        long warmup,
+        long duration,
+        int threads,
+        long maxLatency,
+        int rangesCnt,
+        int payLoadSize,
+        boolean procFromNioThread
+    ) {
+        ctx.io().runIoTest(warmup, duration, threads, maxLatency, rangesCnt, payLoadSize, procFromNioThread,
+            new ArrayList(ctx.cluster().get().forServers().forRemotes().nodes()));
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IgniteKernal.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 3df29cf8fb4e5..4b68e5bdd0da7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -18,23 +18,33 @@
 package org.apache.ignite.internal.managers.communication;
 
 import java.io.Serializable;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Queue;
 import java.util.UUID;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
@@ -44,6 +54,7 @@
 import org.apache.ignite.internal.GridTopic;
 import org.apache.ignite.internal.IgniteComponentType;
 import org.apache.ignite.internal.IgniteDeploymentCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.direct.DirectMessageReader;
 import org.apache.ignite.internal.direct.DirectMessageWriter;
 import org.apache.ignite.internal.managers.GridManagerAdapter;
@@ -55,7 +66,10 @@
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet;
 import org.apache.ignite.internal.util.GridSpinReadWriteLock;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridTuple3;
+import org.apache.ignite.internal.util.lang.IgnitePair;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
@@ -78,11 +92,13 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 import org.jsr166.ConcurrentLinkedDeque8;
+import org.jsr166.LongAdder8;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.GridTopic.TOPIC_COMM_USER;
+import static org.apache.ignite.internal.GridTopic.TOPIC_IO_TEST;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.AFFINITY_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.IDX_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.IGFS_POOL;
@@ -176,6 +192,12 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
     /** Stopping flag. */
     private boolean stopping;
 
+    /** */
+    private final AtomicReference<ConcurrentHashMap<Long, IoTestFuture>> ioTestMap = new AtomicReference<>();
+
+    /** */
+    private final AtomicLong ioTestId = new AtomicLong();
+
     /**
      * @param ctx Grid kernal context.
      */
@@ -297,6 +319,399 @@ public void resetMetrics() {
 
         if (log.isDebugEnabled())
             log.debug(startInfo());
+
+        addMessageListener(TOPIC_IO_TEST, new GridMessageListener() {
+            @Override public void onMessage(UUID nodeId, Object msg) {
+                ClusterNode node = ctx.discovery().node(nodeId);
+
+                if (node == null)
+                    return;
+
+                IgniteIoTestMessage msg0 = (IgniteIoTestMessage)msg;
+
+                msg0.senderNodeId(nodeId);
+
+                if (msg0.request()) {
+                    IgniteIoTestMessage res = new IgniteIoTestMessage(msg0.id(), false, null);
+
+                    res.flags(msg0.flags());
+                    res.onRequestProcessed();
+
+                    res.copyDataFromRequest(msg0);
+
+                    try {
+                        send(node, TOPIC_IO_TEST, res, GridIoPolicy.SYSTEM_POOL);
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to send IO test response [msg=" + msg0 + "]", e);
+                    }
+                }
+                else {
+                    IoTestFuture fut = ioTestMap().get(msg0.id());
+
+                    msg0.onResponseProcessed();
+
+                    if (fut == null)
+                        U.warn(log, "Failed to find IO test future [msg=" + msg0 + ']');
+                    else
+                        fut.onResponse(msg0);
+                }
+            }
+        });
+    }
+
+    /**
+     * @param nodes Nodes.
+     * @param payload Payload.
+     * @param procFromNioThread If {@code true} message is processed from NIO thread.
+     * @return Response future.
+     */
+    public IgniteInternalFuture sendIoTest(List<ClusterNode> nodes, byte[] payload, boolean procFromNioThread) {
+        long id = ioTestId.getAndIncrement();
+
+        IoTestFuture fut = new IoTestFuture(id, nodes.size());
+
+        IgniteIoTestMessage msg = new IgniteIoTestMessage(id, true, payload);
+
+        msg.processFromNioThread(procFromNioThread);
+
+        ioTestMap().put(id, fut);
+
+        for (int i = 0; i < nodes.size(); i++) {
+            ClusterNode node = nodes.get(i);
+
+            try {
+                send(node, TOPIC_IO_TEST, msg, GridIoPolicy.SYSTEM_POOL);
+            }
+            catch (IgniteCheckedException e) {
+                ioTestMap().remove(msg.id());
+
+                return new GridFinishedFuture(e);
+            }
+        }
+
+        return fut;
+    }
+
+    /**
+     * @param node Node.
+     * @param payload Payload.
+     * @param procFromNioThread If {@code true} message is processed from NIO thread.
+     * @return Response future.
+     */
+    public IgniteInternalFuture<List<IgniteIoTestMessage>> sendIoTest(
+        ClusterNode node,
+        byte[] payload,
+        boolean procFromNioThread
+    ) {
+        long id = ioTestId.getAndIncrement();
+
+        IoTestFuture fut = new IoTestFuture(id, 1);
+
+        IgniteIoTestMessage msg = new IgniteIoTestMessage(id, true, payload);
+
+        msg.processFromNioThread(procFromNioThread);
+
+        ioTestMap().put(id, fut);
+
+        try {
+            send(node, TOPIC_IO_TEST, msg, GridIoPolicy.SYSTEM_POOL);
+        }
+        catch (IgniteCheckedException e) {
+            ioTestMap().remove(msg.id());
+
+            return new GridFinishedFuture(e);
+        }
+
+        return fut;
+    }
+
+    /**
+     * @return IO test futures map.
+     */
+    private ConcurrentHashMap<Long, IoTestFuture> ioTestMap() {
+        ConcurrentHashMap<Long, IoTestFuture> map = ioTestMap.get();
+
+        if (map == null) {
+            if (!ioTestMap.compareAndSet(null, map = new ConcurrentHashMap<>()))
+                map = ioTestMap.get();
+        }
+
+        return map;
+    }
+
+    /**
+     * @param warmup Warmup duration in milliseconds.
+     * @param duration Test duration in milliseconds.
+     * @param threads Thread count.
+     * @param latencyLimit Max latency in nanoseconds.
+     * @param rangesCnt Ranges count in resulting histogram.
+     * @param payLoadSize Payload size in bytes.
+     * @param procFromNioThread {@code True} to process requests in NIO threads.
+     * @param nodes Nodes participating in test.
+     */
+    public void runIoTest(
+        final long warmup,
+        final long duration,
+        final int threads,
+        final long latencyLimit,
+        final int rangesCnt,
+        final int payLoadSize,
+        final boolean procFromNioThread,
+        final List<ClusterNode> nodes
+    ) {
+        final ExecutorService svc = Executors.newFixedThreadPool(threads + 1);
+
+        final AtomicBoolean warmupFinished = new AtomicBoolean();
+        final AtomicBoolean done = new AtomicBoolean();
+        final CyclicBarrier bar = new CyclicBarrier(threads + 1);
+        final LongAdder8 cnt = new LongAdder8();
+        final long sleepDuration = 5000;
+        final byte[] payLoad = new byte[payLoadSize];
+        final Map<UUID, IoTestThreadLocalNodeResults>[] res = new Map[threads];
+
+        boolean failed = true;
+
+        try {
+            svc.execute(new Runnable() {
+                @Override public void run() {
+                    boolean failed = true;
+
+                    try {
+                        bar.await();
+
+                        long start = System.currentTimeMillis();
+
+                        if (log.isInfoEnabled())
+                            log.info("IO test started " +
+                                "[warmup=" + warmup +
+                                ", duration=" + duration +
+                                ", threads=" + threads +
+                                ", latencyLimit=" + latencyLimit +
+                                ", rangesCnt=" + rangesCnt +
+                                ", payLoadSize=" + payLoadSize +
+                                ", procFromNioThreads=" + procFromNioThread + ']'
+                            );
+
+                        for (;;) {
+                            if (!warmupFinished.get() && System.currentTimeMillis() - start > warmup) {
+                                if (log.isInfoEnabled())
+                                    log.info("IO test warmup finished.");
+
+                                warmupFinished.set(true);
+
+                                start = System.currentTimeMillis();
+                            }
+
+                            if (warmupFinished.get() && System.currentTimeMillis() - start > duration) {
+                                if (log.isInfoEnabled())
+                                    log.info("IO test finished, will wait for all threads to finish.");
+
+                                done.set(true);
+
+                                bar.await();
+
+                                failed = false;
+
+                                break;
+                            }
+
+                            if (log.isInfoEnabled())
+                                log.info("IO test [opsCnt/sec=" + (cnt.sumThenReset() * 1000 / sleepDuration) +
+                                    ", warmup=" + !warmupFinished.get() +
+                                    ", elapsed=" + (System.currentTimeMillis() - start) + ']');
+
+                            Thread.sleep(sleepDuration);
+                        }
+
+                        // At this point all threads have finished the test and
+                        // stored data to the resulting array of maps.
+                        // Need to iterate it over and sum values for all threads.
+                        printIoTestResults(res);
+                    }
+                    catch (InterruptedException | BrokenBarrierException e) {
+                        U.error(log, "IO test failed.", e);
+                    }
+                    finally {
+                        if (failed)
+                            bar.reset();
+
+                        svc.shutdown();
+                    }
+                }
+            });
+
+            for (int i = 0; i < threads; i++) {
+                final int i0 = i;
+
+                res[i] = U.newHashMap(nodes.size());
+
+                svc.execute(new Runnable() {
+                    @Override public void run() {
+                        boolean failed = true;
+                        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+                        int size = nodes.size();
+                        Map<UUID, IoTestThreadLocalNodeResults> res0 = res[i0];
+
+                        try {
+                            boolean warmupFinished0 = false;
+
+                            bar.await();
+
+                            for (;;) {
+                                if (done.get())
+                                    break;
+
+                                if (!warmupFinished0)
+                                    warmupFinished0 = warmupFinished.get();
+
+                                ClusterNode node = nodes.get(rnd.nextInt(size));
+
+                                List<IgniteIoTestMessage> msgs = sendIoTest(node, payLoad, procFromNioThread).get();
+
+                                cnt.increment();
+
+                                for (IgniteIoTestMessage msg : msgs) {
+                                    UUID nodeId = msg.senderNodeId();
+
+                                    assert nodeId != null;
+
+                                    IoTestThreadLocalNodeResults nodeRes = res0.get(nodeId);
+
+                                    if (nodeRes == null)
+                                        res0.put(nodeId,
+                                            nodeRes = new IoTestThreadLocalNodeResults(rangesCnt, latencyLimit));
+
+                                    nodeRes.onResult(msg);
+                                }
+                            }
+
+                            bar.await();
+
+                            failed = false;
+                        }
+                        catch (Exception e) {
+                            U.error(log, "IO test worker thread failed.", e);
+                        }
+                        finally {
+                            if (failed)
+                                bar.reset();
+                        }
+                    }
+                });
+            }
+
+            failed = false;
+        }
+        finally {
+            if (failed)
+                U.shutdownNow(GridIoManager.class, svc, log);
+        }
+    }
+
+    /**
+     * @param rawRes Resulting map.
+     */
+    private void printIoTestResults(
+        Map<UUID, IoTestThreadLocalNodeResults>[] rawRes
+    ) {
+        Map<UUID, IoTestNodeResults> res = new HashMap<>();
+
+        for (Map<UUID, IoTestThreadLocalNodeResults> r : rawRes) {
+            for (Entry<UUID, IoTestThreadLocalNodeResults> e : r.entrySet()) {
+                IoTestNodeResults r0 = res.get(e.getKey());
+
+                if (r0 == null)
+                    res.put(e.getKey(), r0 = new IoTestNodeResults());
+
+                r0.add(e.getValue());
+            }
+        }
+
+        SimpleDateFormat dateFmt = new SimpleDateFormat("HH:mm:ss,SSS");
+
+        StringBuilder b = new StringBuilder(U.nl())
+            .append("IO test results (round-trip count per each latency bin).")
+            .append(U.nl());
+
+        for (Entry<UUID, IoTestNodeResults> e : res.entrySet()) {
+            ClusterNode node = ctx.discovery().node(e.getKey());
+
+            long binLatencyMcs = e.getValue().binLatencyMcs();
+
+            b.append("Node ID: ").append(e.getKey()).append(" (addrs=")
+                .append(node != null ? node.addresses().toString() : "n/a")
+                .append(", binLatency=").append(binLatencyMcs).append("mcs")
+                .append(')').append(U.nl());
+
+            b.append("Latency bin, mcs | Count exclusive | Percentage exclusive | " +
+                "Count inclusive | Percentage inclusive ").append(U.nl());
+
+            long[] nodeRes = e.getValue().resLatency;
+
+            long sum = 0;
+
+            for (int i = 0; i < nodeRes.length; i++)
+                sum += nodeRes[i];
+
+            long curSum = 0;
+
+            for (int i = 0; i < nodeRes.length; i++) {
+                curSum += nodeRes[i];
+
+                if (i < nodeRes.length - 1)
+                    b.append(String.format("<%11d mcs | %15d | %19.6f%% | %15d | %19.6f%%\n",
+                        (i + 1) * binLatencyMcs,
+                        nodeRes[i], (100.0 * nodeRes[i]) / sum,
+                        curSum, (100.0 * curSum) / sum));
+                else
+                    b.append(String.format(">%11d mcs | %15d | %19.6f%% | %15d | %19.6f%%\n",
+                        i * binLatencyMcs,
+                        nodeRes[i], (100.0 * nodeRes[i]) / sum,
+                        curSum, (100.0 * curSum) / sum));
+            }
+
+            b.append(U.nl()).append("Total latency (ns): ").append(U.nl())
+                .append(String.format("%15d", e.getValue().totalLatency)).append(U.nl());
+
+            b.append(U.nl()).append("Max latencies (ns):").append(U.nl());
+            format(b, e.getValue().maxLatency, dateFmt);
+
+            b.append(U.nl()).append("Max request send queue times (ns):").append(U.nl());
+            format(b, e.getValue().maxReqSendQueueTime, dateFmt);
+
+            b.append(U.nl()).append("Max request receive queue times (ns):").append(U.nl());
+            format(b, e.getValue().maxReqRcvQueueTime, dateFmt);
+
+            b.append(U.nl()).append("Max response send queue times (ns):").append(U.nl());
+            format(b, e.getValue().maxResSendQueueTime, dateFmt);
+
+            b.append(U.nl()).append("Max response receive queue times (ns):").append(U.nl());
+            format(b, e.getValue().maxResRcvQueueTime, dateFmt);
+
+            b.append(U.nl()).append("Max request wire times (millis):").append(U.nl());
+            format(b, e.getValue().maxReqWireTimeMillis, dateFmt);
+
+            b.append(U.nl()).append("Max response wire times (millis):").append(U.nl());
+            format(b, e.getValue().maxResWireTimeMillis, dateFmt);
+
+            b.append(U.nl());
+        }
+
+        if (log.isInfoEnabled())
+            log.info(b.toString());
+    }
+
+    /**
+     * @param b Builder.
+     * @param pairs Pairs to format.
+     * @param dateFmt Formatter.
+     */
+    private void format(StringBuilder b, Collection<IgnitePair<Long>> pairs, SimpleDateFormat dateFmt) {
+        for (IgnitePair<Long> p : pairs) {
+            b.append(String.format("%15d", p.get1())).append(" ")
+                .append(dateFmt.format(new Date(p.get2()))).append(U.nl());
+        }
     }
 
     /** {@inheritDoc} */
@@ -678,8 +1093,22 @@ private void processRegularMessage(
                     msgC.run();
                 }
             }
+
+            @Override public String toString() {
+                return "Message closure [msg=" + msg + ']';
+            }
         };
 
+        if (msg.topicOrdinal() == TOPIC_IO_TEST.ordinal()) {
+            IgniteIoTestMessage msg0 = (IgniteIoTestMessage)msg.message();
+
+            if (msg0.processFromNioThread()) {
+                c.run();
+
+                return;
+            }
+        }
+
         try {
             pools.poolForPolicy(plc).execute(c);
         }
@@ -1796,8 +2225,7 @@ public boolean removeMessageListener(Object topic, @Nullable GridMessageListener
         if (rmv && log.isDebugEnabled())
             log.debug("Removed message listener [topic=" + topic + ", lsnr=" + lsnr + ']');
 
-        if (lsnr instanceof ArrayListener)
-        {
+        if (lsnr instanceof ArrayListener) {
             for (GridMessageListener childLsnr : ((ArrayListener)lsnr).arr)
                 closeListener(childLsnr);
         }
@@ -2460,4 +2888,269 @@ public UUID nodeId() {
             return S.toString(DelayedMessage.class, this, super.toString());
         }
     }
+
+    /**
+     *
+     */
+    private class IoTestFuture extends GridFutureAdapter<List<IgniteIoTestMessage>> {
+        /** */
+        private final long id;
+
+        /** */
+        private final int cntr;
+
+        /** */
+        private final List<IgniteIoTestMessage> ress;
+
+        /**
+         * @param id ID.
+         * @param cntr Counter.
+         */
+        IoTestFuture(long id, int cntr) {
+            assert cntr > 0 : cntr;
+
+            this.id = id;
+            this.cntr = cntr;
+
+            ress = new ArrayList<>(cntr);
+        }
+
+        /**
+         *
+         */
+        void onResponse(IgniteIoTestMessage res) {
+            boolean complete;
+
+            synchronized (this) {
+                ress.add(res);
+
+                complete = cntr == ress.size();
+            }
+
+            if (complete)
+                onDone(ress);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean onDone(List<IgniteIoTestMessage> res, @Nullable Throwable err) {
+            if (super.onDone(res, err)) {
+                ioTestMap().remove(id);
+
+                return true;
+            }
+
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(IoTestFuture.class, this);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class IoTestThreadLocalNodeResults {
+        /** */
+        private final long[] resLatency;
+
+        /** */
+        private final int rangesCnt;
+
+        /** */
+        private long totalLatency;
+
+        /** */
+        private long maxLatency;
+
+        /** */
+        private long maxLatencyTs;
+
+        /** */
+        private long maxReqSendQueueTime;
+
+        /** */
+        private long maxReqSendQueueTimeTs;
+
+        /** */
+        private long maxReqRcvQueueTime;
+
+        /** */
+        private long maxReqRcvQueueTimeTs;
+
+        /** */
+        private long maxResSendQueueTime;
+
+        /** */
+        private long maxResSendQueueTimeTs;
+
+        /** */
+        private long maxResRcvQueueTime;
+
+        /** */
+        private long maxResRcvQueueTimeTs;
+
+        /** */
+        private long maxReqWireTimeMillis;
+
+        /** */
+        private long maxReqWireTimeTs;
+
+        /** */
+        private long maxResWireTimeMillis;
+
+        /** */
+        private long maxResWireTimeTs;
+
+        /** */
+        private final long latencyLimit;
+
+        /**
+         * @param rangesCnt Ranges count.
+         * @param latencyLimit
+         */
+        public IoTestThreadLocalNodeResults(int rangesCnt, long latencyLimit) {
+            this.rangesCnt = rangesCnt;
+            this.latencyLimit = latencyLimit;
+
+            resLatency = new long[rangesCnt + 1];
+        }
+
+        /**
+         * @param msg
+         */
+        public void onResult(IgniteIoTestMessage msg) {
+            long now = System.currentTimeMillis();
+
+            long latency = msg.responseProcessedTs() - msg.requestCreateTs();
+
+            int idx = latency >= latencyLimit ?
+                rangesCnt /* Timed out. */ :
+                (int)Math.floor((1.0 * latency) / ((1.0 * latencyLimit) / rangesCnt));
+
+            resLatency[idx]++;
+
+            totalLatency += latency;
+
+            if (maxLatency < latency) {
+                maxLatency = latency;
+                maxLatencyTs = now;
+            }
+
+            long reqSndQueueTime = msg.requestSendTs() - msg.requestCreateTs();
+
+            if (maxReqSendQueueTime < reqSndQueueTime) {
+                maxReqSendQueueTime = reqSndQueueTime;
+                maxReqSendQueueTimeTs = now;
+            }
+
+            long reqRcvQueueTime = msg.requestProcessTs() - msg.requestReceiveTs();
+
+            if (maxReqRcvQueueTime < reqRcvQueueTime) {
+                maxReqRcvQueueTime = reqRcvQueueTime;
+                maxReqRcvQueueTimeTs = now;
+            }
+
+            long resSndQueueTime = msg.responseSendTs() - msg.requestProcessTs();
+
+            if (maxResSendQueueTime < resSndQueueTime) {
+                maxResSendQueueTime = resSndQueueTime;
+                maxResSendQueueTimeTs = now;
+            }
+
+            long resRcvQueueTime = msg.responseProcessedTs() - msg.responseReceiveTs();
+
+            if (maxResRcvQueueTime < resRcvQueueTime) {
+                maxResRcvQueueTime = resRcvQueueTime;
+                maxResRcvQueueTimeTs = now;
+            }
+
+            long reqWireTimeMillis = msg.requestReceivedTsMillis() - msg.requestSendTsMillis();
+
+            if (maxReqWireTimeMillis < reqWireTimeMillis) {
+                maxReqWireTimeMillis = reqWireTimeMillis;
+                maxReqWireTimeTs = now;
+            }
+
+            long resWireTimeMillis = msg.responseReceivedTsMillis() - msg.requestSendTsMillis();
+
+            if (maxResWireTimeMillis < resWireTimeMillis) {
+                maxResWireTimeMillis = resWireTimeMillis;
+                maxResWireTimeTs = now;
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private static class IoTestNodeResults {
+        /** */
+        private long latencyLimit;
+
+        /** */
+        private long[] resLatency;
+
+        /** */
+        private long totalLatency;
+
+        /** */
+        private Collection<IgnitePair<Long>> maxLatency = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxReqSendQueueTime = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxReqRcvQueueTime = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxResSendQueueTime = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxResRcvQueueTime = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxReqWireTimeMillis = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxResWireTimeMillis = new ArrayList<>();
+
+        /**
+         * @param res Node results to add.
+         */
+        public void add(IoTestThreadLocalNodeResults res) {
+            if (resLatency == null) {
+                resLatency = res.resLatency.clone();
+                latencyLimit = res.latencyLimit;
+            }
+            else {
+                assert latencyLimit == res.latencyLimit;
+                assert resLatency.length == res.resLatency.length;
+
+                for (int i = 0; i < resLatency.length; i++)
+                    resLatency[i] += res.resLatency[i];
+            }
+
+            totalLatency += res.totalLatency;
+
+            maxLatency.add(F.pair(res.maxLatency, res.maxLatencyTs));
+            maxReqSendQueueTime.add(F.pair(res.maxReqSendQueueTime, res.maxReqSendQueueTimeTs));
+            maxReqRcvQueueTime.add(F.pair(res.maxReqRcvQueueTime, res.maxReqRcvQueueTimeTs));
+            maxResSendQueueTime.add(F.pair(res.maxResSendQueueTime, res.maxResSendQueueTimeTs));
+            maxResRcvQueueTime.add(F.pair(res.maxResRcvQueueTime, res.maxResRcvQueueTimeTs));
+            maxReqWireTimeMillis.add(F.pair(res.maxReqWireTimeMillis, res.maxReqWireTimeTs));
+            maxResWireTimeMillis.add(F.pair(res.maxResWireTimeMillis, res.maxResWireTimeTs));
+        }
+
+        /**
+         * @return Bin latency in microseconds.
+         */
+        public long binLatencyMcs() {
+            if (resLatency == null)
+                throw new IllegalStateException();
+
+            return latencyLimit / (1000 * (resLatency.length - 1));
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index f36191cbafcd9..743d5f74d2ec9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -165,6 +165,11 @@ public GridIoMessageFactory(MessageFactory[] ext) {
         Message msg = null;
 
         switch (type) {
+            case -43:
+                msg = new IgniteIoTestMessage();
+
+                break;
+
             case -27:
                 msg = new GridDhtTxOnePhaseCommitAckRequest();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
new file mode 100644
index 0000000000000..6145439d87b4f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ *
+ */
+public class IgniteIoTestMessage implements Message {
+    /** */
+    private static byte FLAG_PROC_FROM_NIO = 1;
+
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private long id;
+
+    /** */
+    private byte flags;
+
+    /** */
+    private boolean req;
+
+    /** */
+    private byte payload[];
+
+    /** */
+    private long reqCreateTs;
+
+    /** */
+    private long reqSndTs;
+
+    /** */
+    private long reqSndTsMillis;
+
+    /** */
+    private long reqRcvTs;
+
+    /** */
+    private long reqRcvTsMillis;
+
+    /** */
+    private long reqProcTs;
+
+    /** */
+    private long resSndTs;
+
+    /** */
+    private long resSndTsMillis;
+
+    /** */
+    private long resRcvTs;
+
+    /** */
+    private long resRcvTsMillis;
+
+    /** */
+    private long resProcTs;
+
+    /** */
+    @GridDirectTransient
+    private UUID sndNodeId;
+
+    /**
+     *
+     */
+    public IgniteIoTestMessage() {
+        // No-op.
+    }
+
+    /**
+     * @param id Message ID.
+     * @param req Request flag.
+     * @param payload Payload.
+     */
+    public IgniteIoTestMessage(long id, boolean req, byte[] payload) {
+        this.id = id;
+        this.req = req;
+        this.payload = payload;
+
+        reqCreateTs = System.nanoTime();
+    }
+
+    /**
+     * @return {@code True} if message should be processed from NIO thread
+     * (otherwise message is submitted to system pool).
+     */
+    public boolean processFromNioThread() {
+        return isFlag(FLAG_PROC_FROM_NIO);
+    }
+
+    /**
+     * @param procFromNioThread {@code True} if message should be processed from NIO thread.
+     */
+    public void processFromNioThread(boolean procFromNioThread) {
+        setFlag(procFromNioThread, FLAG_PROC_FROM_NIO);
+    }
+
+    /**
+     * @param flags Flags.
+     */
+    public void flags(byte flags) {
+        this.flags = flags;
+    }
+
+    /**
+     * @return Flags.
+     */
+    public byte flags() {
+        return flags;
+    }
+
+    /**
+     * Sets flag mask.
+     *
+     * @param flag Set or clear.
+     * @param mask Mask.
+     */
+    private void setFlag(boolean flag, int mask) {
+        flags = flag ? (byte)(flags | mask) : (byte)(flags & ~mask);
+    }
+
+    /**
+     * Reads flag mask.
+     *
+     * @param mask Mask to read.
+     * @return Flag value.
+     */
+    private boolean isFlag(int mask) {
+        return (flags & mask) != 0;
+    }
+
+    /**
+     * @return {@code true} if this is request.
+     */
+    public boolean request() {
+        return req;
+    }
+
+    /**
+     * @return ID.
+     */
+    public long id() {
+        return id;
+    }
+
+    /**
+     * @return Request create timestamp.
+     */
+    public long requestCreateTs() {
+        return reqCreateTs;
+    }
+
+    /**
+     * @return Request send timestamp.
+     */
+    public long requestSendTs() {
+        return reqSndTs;
+    }
+
+    /**
+     * @return Request receive timestamp.
+     */
+    public long requestReceiveTs() {
+        return reqRcvTs;
+    }
+
+    /**
+     * @return Request process started timestamp.
+     */
+    public long requestProcessTs() {
+        return reqProcTs;
+    }
+
+    /**
+     * @return Response send timestamp.
+     */
+    public long responseSendTs() {
+        return resSndTs;
+    }
+
+    /**
+     * @return Response receive timestamp.
+     */
+    public long responseReceiveTs() {
+        return resRcvTs;
+    }
+
+    /**
+     * @return Response process timestamp.
+     */
+    public long responseProcessTs() {
+        return resProcTs;
+    }
+
+    /**
+     * @return Request send timestamp (millis).
+     */
+    public long requestSendTsMillis() {
+        return reqSndTsMillis;
+    }
+
+    /**
+     * @return Request received timestamp (millis).
+     */
+    public long requestReceivedTsMillis() {
+        return reqRcvTsMillis;
+    }
+
+    /**
+     * @return Response send timestamp (millis).
+     */
+    public long responseSendTsMillis() {
+        return resSndTsMillis;
+    }
+
+    /**
+     * @return Response received timestamp (millis).
+     */
+    public long responseReceivedTsMillis() {
+        return resRcvTsMillis;
+    }
+
+    /**
+     * This method is called to initialize tracing variables.
+     * TODO: introduce direct message lifecycle API?
+     */
+    public void onAfterRead() {
+        if (req && reqRcvTs == 0) {
+            reqRcvTs = System.nanoTime();
+
+            reqRcvTsMillis = System.currentTimeMillis();
+        }
+
+        if (!req && resRcvTs == 0) {
+            resRcvTs = System.nanoTime();
+
+            resRcvTsMillis = System.currentTimeMillis();
+        }
+    }
+
+    /**
+     * This method is called to initialize tracing variables.
+     * TODO: introduce direct message lifecycle API?
+     */
+    public void onBeforeWrite() {
+        if (req && reqSndTs == 0) {
+            reqSndTs = System.nanoTime();
+
+            reqSndTsMillis = System.currentTimeMillis();
+        }
+
+        if (!req && resSndTs == 0) {
+            resSndTs = System.nanoTime();
+
+            resSndTsMillis = System.currentTimeMillis();
+        }
+    }
+
+    /**
+     *
+     */
+    public void copyDataFromRequest(IgniteIoTestMessage req) {
+        reqCreateTs = req.reqCreateTs;
+
+        reqSndTs = req.reqSndTs;
+        reqSndTsMillis = req.reqSndTsMillis;
+
+        reqRcvTs = req.reqRcvTs;
+        reqRcvTsMillis = req.reqRcvTsMillis;
+    }
+
+    /**
+     *
+     */
+    public void onRequestProcessed() {
+        reqProcTs = System.nanoTime();
+    }
+
+    /**
+     *
+     */
+    public void onResponseProcessed() {
+        resProcTs = System.nanoTime();
+    }
+
+    /**
+     * @return Response processed timestamp.
+     */
+    public long responseProcessedTs() {
+        return resProcTs;
+    }
+
+    /**
+     * @return Sender node ID.
+     */
+    public UUID senderNodeId() {
+        return sndNodeId;
+    }
+
+    /**
+     * @param sndNodeId Sender node ID.
+     */
+    public void senderNodeId(UUID sndNodeId) {
+        this.sndNodeId = sndNodeId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        onBeforeWrite();
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeByte("flags", flags))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeLong("id", id))
+                    return false;
+
+                writer.incrementState();
+
+            case 2:
+                if (!writer.writeByteArray("payload", payload))
+                    return false;
+
+                writer.incrementState();
+
+            case 3:
+                if (!writer.writeBoolean("req", req))
+                    return false;
+
+                writer.incrementState();
+
+            case 4:
+                if (!writer.writeLong("reqCreateTs", reqCreateTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 5:
+                if (!writer.writeLong("reqProcTs", reqProcTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 6:
+                if (!writer.writeLong("reqRcvTs", reqRcvTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 7:
+                if (!writer.writeLong("reqRcvTsMillis", reqRcvTsMillis))
+                    return false;
+
+                writer.incrementState();
+
+            case 8:
+                if (!writer.writeLong("reqSndTs", reqSndTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 9:
+                if (!writer.writeLong("reqSndTsMillis", reqSndTsMillis))
+                    return false;
+
+                writer.incrementState();
+
+            case 10:
+                if (!writer.writeLong("resProcTs", resProcTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 11:
+                if (!writer.writeLong("resRcvTs", resRcvTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 12:
+                if (!writer.writeLong("resRcvTsMillis", resRcvTsMillis))
+                    return false;
+
+                writer.incrementState();
+
+            case 13:
+                if (!writer.writeLong("resSndTs", resSndTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 14:
+                if (!writer.writeLong("resSndTsMillis", resSndTsMillis))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                flags = reader.readByte("flags");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                id = reader.readLong("id");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 2:
+                payload = reader.readByteArray("payload");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 3:
+                req = reader.readBoolean("req");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 4:
+                reqCreateTs = reader.readLong("reqCreateTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 5:
+                reqProcTs = reader.readLong("reqProcTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 6:
+                reqRcvTs = reader.readLong("reqRcvTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 7:
+                reqRcvTsMillis = reader.readLong("reqRcvTsMillis");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 8:
+                reqSndTs = reader.readLong("reqSndTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 9:
+                reqSndTsMillis = reader.readLong("reqSndTsMillis");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 10:
+                resProcTs = reader.readLong("resProcTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 11:
+                resRcvTs = reader.readLong("resRcvTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 12:
+                resRcvTsMillis = reader.readLong("resRcvTsMillis");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 13:
+                resSndTs = reader.readLong("resSndTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 14:
+                resSndTsMillis = reader.readLong("resSndTsMillis");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        onAfterRead();
+
+        return reader.afterMessageRead(IgniteIoTestMessage.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte directType() {
+        return -43;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 15;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IgniteIoTestMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
index 0754c27ee05bd..0a73d5ed61a0f 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
@@ -372,4 +372,48 @@ public interface IgniteMXBean {
      */
     @MXBeanDescription("Dumps debug information for the current node.")
     public void dumpDebugInfo();
+
+    /**
+     * Runs IO latency test against all remote server nodes in cluster.
+     *
+     * @param warmup Warmup duration in milliseconds.
+     * @param duration Test duration in milliseconds.
+     * @param threads Thread count.
+     * @param maxLatency Max latency in nanoseconds.
+     * @param rangesCnt Ranges count in resulting histogram.
+     * @param payLoadSize Payload size in bytes.
+     * @param procFromNioThread {@code True} to process requests in NIO threads.
+     */
+    @MXBeanDescription("Runs IO latency test against all remote server nodes in cluster.")
+    @MXBeanParametersNames(
+        {
+            "warmup",
+            "duration",
+            "threads",
+            "maxLatency",
+            "rangesCnt",
+            "payLoadSize",
+            "procFromNioThread"
+        }
+    )
+    @MXBeanParametersDescriptions(
+        {
+            "Warmup duration (millis).",
+            "Test duration (millis).",
+            "Threads count.",
+            "Maximum latency expected (nanos).",
+            "Ranges count for histogram.",
+            "Payload size (bytes).",
+            "Process requests in NIO-threads flag."
+        }
+    )
+    void runIoTest(
+        long warmup,
+        long duration,
+        int threads,
+        long maxLatency,
+        int rangesCnt,
+        int payLoadSize,
+        boolean procFromNioThread
+    );
 }

From 540ca449f1bd2386d3ba0722afb21dd3a504d044 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 13 Jun 2017 20:55:38 +0300
Subject: [PATCH 307/446] GG-12355: Added discovery ring latency test + made it
 available from MBean (cherry-picked from master).

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 20 +++++
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 57 +++++++++++++-
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |  6 ++
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |  5 ++
 .../discovery/tcp/TcpDiscoverySpiMBean.java   | 22 ++++++
 .../TcpDiscoveryRingLatencyCheckMessage.java  | 77 +++++++++++++++++++
 6 files changed, 186 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryRingLatencyCheckMessage.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index b075dc1fb5289..71550447fa04d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -97,6 +97,7 @@
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeLeftMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingRequest;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingResponse;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryRingLatencyCheckMessage;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -834,6 +835,16 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
             U.closeQuiet(sockStream.socket());
     }
 
+    /** {@inheritDoc} */
+    @Override public void checkRingLatency(int maxHops) {
+        TcpDiscoveryRingLatencyCheckMessage msg = new TcpDiscoveryRingLatencyCheckMessage(getLocalNodeId(), maxHops);
+
+        if (log.isInfoEnabled())
+            log.info("Latency check initiated: " + msg.id());
+
+        sockWriter.sendMessage(msg);
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteSpiThread workerThread() {
         return msgWorker;
@@ -1228,6 +1239,12 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                         msg,
                         sockTimeout);
 
+                    IgniteUuid latencyCheckId = msg instanceof TcpDiscoveryRingLatencyCheckMessage ?
+                        msg.id() : null;
+
+                    if (latencyCheckId != null && log.isInfoEnabled())
+                        log.info("Latency check message has been written to socket: " + latencyCheckId);
+
                     msg = null;
 
                     if (ack) {
@@ -1255,6 +1272,9 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
 
                             throw new IOException("Failed to get acknowledge for message: " + unacked);
                         }
+
+                        if (latencyCheckId != null && log.isInfoEnabled())
+                            log.info("Latency check message has been acked: " + latencyCheckId);
                     }
                 }
                 catch (InterruptedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 58b362ffd0f15..6bdd80cbe6ff6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -129,6 +129,7 @@
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingRequest;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingResponse;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryRedirectToClient;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryRingLatencyCheckMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryStatusCheckMessage;
 import org.apache.ignite.thread.IgniteThreadPoolExecutor;
 import org.jetbrains.annotations.Nullable;
@@ -1583,6 +1584,16 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void checkRingLatency(int maxHops) {
+        TcpDiscoveryRingLatencyCheckMessage msg = new TcpDiscoveryRingLatencyCheckMessage(getLocalNodeId(), maxHops);
+
+        if (log.isInfoEnabled())
+            log.info("Latency check initiated: " + msg.id());
+
+        msgWorker.addMessage(msg);
+    }
+
     /** {@inheritDoc} */
     @Override void simulateNodeFailure() {
         U.warn(log, "Simulating node failure: " + getLocalNodeId());
@@ -2647,6 +2658,9 @@ else if (msg instanceof TcpDiscoveryCustomEventMessage)
             else if (msg instanceof TcpDiscoveryClientPingRequest)
                 processClientPingRequest((TcpDiscoveryClientPingRequest)msg);
 
+            else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage)
+                processRingLatencyCheckMessage((TcpDiscoveryRingLatencyCheckMessage)msg);
+
             else
                 assert false : "Unknown message type: " + msg.getClass().getSimpleName();
 
@@ -3054,12 +3068,20 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof
                                     }
                                 }
 
+                                boolean latencyCheck = msg instanceof TcpDiscoveryRingLatencyCheckMessage;
+
+                                if (latencyCheck && log.isInfoEnabled())
+                                    log.info("Latency check message has been written to socket: " + msg.id());
+
                                 spi.writeToSocket(sock, out, msg, timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()));
 
                                 long tstamp0 = U.currentTimeMillis();
 
                                 int res = spi.readReceipt(sock, timeoutHelper.nextTimeoutChunk(ackTimeout0));
 
+                                if (latencyCheck && log.isInfoEnabled())
+                                    log.info("Latency check message has been acked: " + msg.id());
+
                                 spi.stats.onMessageSent(msg, tstamp0 - tstamp, U.currentTimeMillis() - tstamp0);
 
                                 onMessageExchanged();
@@ -4477,6 +4499,33 @@ private void processNodeAddFinishedMessage(TcpDiscoveryNodeAddFinishedMessage ms
             checkPendingCustomMessages();
         }
 
+        /**
+         * Processes latency check message.
+         *
+         * @param msg Latency check message.
+         */
+        private void processRingLatencyCheckMessage(TcpDiscoveryRingLatencyCheckMessage msg) {
+            assert msg != null;
+
+            if (msg.maxHopsReached()) {
+                if (log.isInfoEnabled())
+                    log.info("Latency check has been discarded (max hops reached) [id=" + msg.id() +
+                        ", maxHops=" + msg.maxHops() + ']');
+
+                return;
+            }
+
+            if (log.isInfoEnabled())
+                log.info("Latency check processing: " + msg.id());
+
+            if (sendMessageToRemotes(msg))
+                sendMessageAcrossRing(msg);
+            else {
+                if (log.isInfoEnabled())
+                    log.info("Latency check has been discarded (no remote nodes): " + msg.id());
+            }
+        }
+
         /**
          * Processes node left message.
          *
@@ -6056,7 +6105,7 @@ else if (msg instanceof TcpDiscoveryLoopbackProblemMessage) {
 
                             continue;
                         }
-                        if (msg instanceof TcpDiscoveryPingResponse) {
+                        else if (msg instanceof TcpDiscoveryPingResponse) {
                             assert msg.client() : msg;
 
                             ClientMessageWorker clientWorker = clientMsgWorkers.get(msg.creatorNodeId());
@@ -6066,6 +6115,12 @@ else if (msg instanceof TcpDiscoveryLoopbackProblemMessage) {
 
                             continue;
                         }
+                        else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage) {
+                            if (log.isInfoEnabled())
+                                log.info("Latency check message has been read: " + msg.id());
+
+                            ((TcpDiscoveryRingLatencyCheckMessage)msg).onRead();
+                        }
 
                         TcpDiscoveryClientHeartbeatMessage heartbeatMsg = null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index 84c2ff28afef0..cb85dc1c44b86 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryRingLatencyCheckMessage;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -281,6 +282,11 @@ protected static String threadStatus(Thread t) {
      */
     public abstract void brakeConnection();
 
+    /**
+     * @param maxHops Maximum hops for {@link TcpDiscoveryRingLatencyCheckMessage}.
+     */
+    public abstract void checkRingLatency(int maxHops);
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index a2a47feb31486..cefbcd675a303 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1160,6 +1160,11 @@ LinkedHashSet<InetSocketAddress> getNodeAddresses(TcpDiscoveryNode node, boolean
         return stats.coordinatorSinceTimestamp();
     }
 
+    /** {@inheritDoc} */
+    @Override public void checkRingLatency(int maxHops) {
+        impl.checkRingLatency(maxHops);
+    }
+
     /** {@inheritDoc} */
     @Override protected void onContextInitialized0(IgniteSpiContext spiCtx) throws IgniteSpiException {
         super.onContextInitialized0(spiCtx);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
index 1427929a4cb88..e96088dd99d37 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
@@ -20,6 +20,8 @@
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.mxbean.MXBeanDescription;
+import org.apache.ignite.mxbean.MXBeanParametersDescriptions;
+import org.apache.ignite.mxbean.MXBeanParametersNames;
 import org.apache.ignite.spi.IgniteSpiManagementMBean;
 import org.jetbrains.annotations.Nullable;
 
@@ -281,4 +283,24 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
      */
     @MXBeanDescription("Client mode.")
     public boolean isClientMode() throws IllegalStateException;
+
+    /**
+     * Diagnosis method for determining ring message latency.
+     * On this method call special message will be sent across the ring
+     * and stats about the message will appear in the logs of each node.
+     *
+     * @param maxHops Maximum hops for the message (3 * TOTAL_NODE_CNT is recommended).
+     */
+    @MXBeanDescription("Check ring latency.")
+    @MXBeanParametersNames(
+        {
+            "maxHops"
+        }
+    )
+    @MXBeanParametersDescriptions(
+        {
+            "Maximum hops for the message (3 * TOTAL_NODE_CNT is recommended)."
+        }
+    )
+    public void checkRingLatency(int maxHops);
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryRingLatencyCheckMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryRingLatencyCheckMessage.java
new file mode 100644
index 0000000000000..d8c11452e641e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryRingLatencyCheckMessage.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.messages;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class TcpDiscoveryRingLatencyCheckMessage extends TcpDiscoveryAbstractMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private int maxHops;
+
+    /** */
+    private int curHop;
+
+    /**
+     * @param creatorNodeId Creator node ID.
+     * @param maxHops Max hops for this message.
+     */
+    public TcpDiscoveryRingLatencyCheckMessage(
+        UUID creatorNodeId,
+        int maxHops
+    ) {
+        super(creatorNodeId);
+
+        assert maxHops > 0;
+
+        this.maxHops = maxHops;
+    }
+
+    /**
+     *
+     */
+    public void onRead() {
+        curHop++;
+    }
+
+    /**
+     * @return Max hops.
+     */
+    public int maxHops() {
+        return maxHops;
+    }
+
+    /**
+     * @return {@code True} if max hops has been reached.
+     */
+    public boolean maxHopsReached() {
+        return curHop == maxHops;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(TcpDiscoveryRingLatencyCheckMessage.class, this, "super", super.toString());
+    }
+}

From 8cd9e829380f4c91cc9bb126169863286d1cb323 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Jun 2017 15:40:14 +0300
Subject: [PATCH 308/446] GG-12353: Added local binary context flag.

Backport of IGNITE-5223 with fixes.
---
 .../apache/ignite/IgniteSystemProperties.java |  10 +-
 .../internal/binary/BinaryReaderExImpl.java   |   2 +-
 .../BinaryMarshallerLocalMetadataCache.java   | 126 ++++++++++++++++++
 3 files changed, 128 insertions(+), 10 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index f35e9e620070a..775aae03a1ef4 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -320,14 +320,6 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_H2_DEBUG_CONSOLE = "IGNITE_H2_DEBUG_CONSOLE";
 
-    /**
-     * This property allows to specify user defined port which H2 indexing SPI will use
-     * to start H2 debug console on. If this property is not set or set to 0, H2 debug
-     * console will use system-provided dynamic port.
-     * This property is only relevant when {@link #IGNITE_H2_DEBUG_CONSOLE} property is set.
-     */
-    public static final String IGNITE_H2_DEBUG_CONSOLE_PORT = "IGNITE_H2_DEBUG_CONSOLE_PORT";
-
     /**
      * If this property is set to {@code true} then shared memory space native debug will be enabled.
      */
@@ -511,7 +503,7 @@ public final class IgniteSystemProperties {
     /**
      * Whether Ignite can access unaligned memory addresses.
      * <p>
-     * Defaults to {@code false}, meaning that unaligned access will be performed only on x86 architecture.
+     * Defaults to {@code} false, meaning that unaligned access will be performed only on x86 architecture.
      */
     public static final String IGNITE_UNALIGNED_MEMORY_ACCESS = "IGNITE_UNALIGNED_MEMORY_ACCESS";
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
index ad2e736fa5f34..7f71b460e5223 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
@@ -1723,7 +1723,7 @@ public BinarySchema getOrCreateSchema() {
             if (fieldIdLen != BinaryUtils.FIELD_ID_LEN) {
                 BinaryTypeImpl type = (BinaryTypeImpl)ctx.metadata(typeId);
 
-                if (type == null || type.metadata() == null || type.metadata().schemas().isEmpty()) {
+                if (type == null || type.metadata() == null || type.metadata().schemas() == null || type.metadata().schemas().isEmpty()) {
                     if (IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE, false)) {
                         BinaryClassDescriptor desc = ctx.descriptorForTypeId(true, typeId, getClass().getClassLoader(), false);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java
new file mode 100644
index 0000000000000..8a2d38b61a48b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.testframework.junits.IgniteTestResources;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE property.
+ */
+public class BinaryMarshallerLocalMetadataCache extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName,
+        IgniteTestResources rsrcs) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName, rsrcs);
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE, "true");
+
+        startGrid(0);
+        startGrid(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        try {
+            stopAllGrids();
+        }
+        finally {
+            System.clearProperty(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLocalMetadata() throws Exception {
+        final BinaryObject obj = grid(0).binary().toBinary(new OptimizedContainer(new Optimized()));
+
+        ClusterGroup remotes = grid(0).cluster().forRemotes();
+
+        OptimizedContainer res = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
+            @Override public OptimizedContainer call() throws Exception {
+
+                return obj.deserialize();
+            }
+        });
+
+        OptimizedContainer res2 = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
+            @Override public OptimizedContainer call() throws Exception {
+
+                return obj.deserialize();
+            }
+        });
+
+        System.out.println(res);
+        System.out.println(res2);
+    }
+
+    /**
+     *
+     */
+    private static class OptimizedContainer {
+        /** */
+        private Optimized optim;
+
+        /**
+         * @param optim Val.
+         */
+        public OptimizedContainer(Optimized optim) {
+            this.optim = optim;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Optimized implements Externalizable {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private String fld;
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(ObjectOutput out) throws IOException {
+            U.writeUTFStringNullable(out, fld);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+            fld = U.readUTFStringNullable(in);
+        }
+    }
+}

From f8224d13cf9a6432ba65e0016370ba51bbb544e9 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 15 Jun 2017 22:49:45 +0300
Subject: [PATCH 309/446] GG-12299: Make sure concurrent type registrations do
 not trigger multiple cache updates.

---
 .../internal/MarshallerContextAdapter.java    | 134 ++++++++++++++----
 .../BinaryMarshallerLocalMetadataCache.java   |  63 +++++---
 2 files changed, 149 insertions(+), 48 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java
index ad3439325d446..56007adb34544 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java
@@ -32,6 +32,8 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap2;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.IgniteOutClosureX;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.MarshallerContext;
 import org.apache.ignite.plugin.PluginProvider;
@@ -49,7 +51,7 @@ public abstract class MarshallerContextAdapter implements MarshallerContext {
     private static final String JDK_CLS_NAMES_FILE = "META-INF/classnames-jdk.properties";
 
     /** */
-    private final ConcurrentMap<Integer, String> map = new ConcurrentHashMap8<>();
+    private final ConcurrentMap<Integer, Object> map = new ConcurrentHashMap8<>();
 
     /** */
     private final Set<String> registeredSystemTypes = new HashSet<>();
@@ -134,55 +136,129 @@ private void processResource(URL url) throws IOException {
 
                 int typeId = clsName.hashCode();
 
-                String oldClsName;
+                Object oldClsNameOrFuture = map.put(typeId, clsName);
 
-                if ((oldClsName = map.put(typeId, clsName)) != null) {
-                    if (!oldClsName.equals(clsName))
-                        throw new IgniteException("Duplicate type ID [id=" + typeId + ", clsName=" + clsName +
-                        ", oldClsName=" + oldClsName + ']');
-                }
+                try {
+                    String oldClsName = unwrap(oldClsNameOrFuture);
+
+                    if (oldClsName != null) {
+                        if (!oldClsName.equals(clsName))
+                            throw new IgniteException("Duplicate type ID [id=" + typeId + ", clsName=" + clsName +
+                            ", oldClsName=" + oldClsName + ']');
+                    }
 
-                registeredSystemTypes.add(clsName);
+                    registeredSystemTypes.add(clsName);
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IllegalStateException("Failed to process type ID [typeId=" + typeId +
+                        ", clsName" + clsName + ']', e);
+                }
             }
         }
     }
 
     /** {@inheritDoc} */
-    @Override public boolean registerClass(int id, Class cls) throws IgniteCheckedException {
-        boolean registered = true;
-
-        String clsName = map.get(id);
+    @Override public boolean registerClass(final int id, final Class cls) throws IgniteCheckedException {
+        Object clsNameOrFuture = map.get(id);
+
+        String clsName = clsNameOrFuture != null ?
+            unwrap(clsNameOrFuture) :
+            computeIfAbsent(id, new IgniteOutClosureX<String>() {
+                @Override public String applyx() throws IgniteCheckedException {
+                    return registerClassName(id, cls.getName()) ? cls.getName() : null;
+                }
+            });
 
-        if (clsName == null) {
-            registered = registerClassName(id, cls.getName());
+        // The only way we can have clsName eq null here is a failing concurrent thread.
+        if (clsName == null)
+            return false;
 
-            if (registered)
-                map.putIfAbsent(id, cls.getName());
-        }
-        else if (!clsName.equals(cls.getName()))
+        if (!clsName.equals(cls.getName()))
             throw new IgniteCheckedException("Duplicate ID [id=" + id + ", oldCls=" + clsName +
                 ", newCls=" + cls.getName());
 
-        return registered;
+        return true;
     }
 
     /** {@inheritDoc} */
-    @Override public Class getClass(int id, ClassLoader ldr) throws ClassNotFoundException, IgniteCheckedException {
-        String clsName = map.get(id);
+    @Override public Class getClass(final int id, ClassLoader ldr) throws ClassNotFoundException, IgniteCheckedException {
+        Object clsNameOrFuture = map.get(id);
+
+        String clsName = clsNameOrFuture != null ?
+            unwrap(clsNameOrFuture) :
+            computeIfAbsent(id, new IgniteOutClosureX<String>() {
+                @Override public String applyx() throws IgniteCheckedException {
+                    return className(id);
+                }
+            });
 
-        if (clsName == null) {
-            clsName = className(id);
+        if (clsName == null)
+            throw new ClassNotFoundException("Unknown type ID: " + id);
 
-            if (clsName == null)
-                throw new ClassNotFoundException("Unknown type ID: " + id);
+        return U.forName(clsName, ldr);
+    }
+
+    /**
+     * Computes the map value for the given ID. Will make sure that if there are two threads are calling
+     *      {@code computeIfAbsent}, only one of them will invoke the closure. If the closure threw an exeption,
+     *      all threads attempting to compute the value will throw this exception.
+     *
+     * @param id Type ID.
+     * @param clo Close to compute.
+     * @return Computed value.
+     * @throws IgniteCheckedException If closure threw an exception.
+     */
+    private String computeIfAbsent(int id, IgniteOutClosureX<String> clo) throws IgniteCheckedException {
+        Object clsNameOrFuture = map.get(id);
 
-            String old = map.putIfAbsent(id, clsName);
+        if (clsNameOrFuture == null) {
+            GridFutureAdapter<String> fut = new GridFutureAdapter<>();
 
-            if (old != null)
-                clsName = old;
+            Object old = map.putIfAbsent(id, fut);
+
+            if (old == null) {
+                String clsName = null;
+
+                try {
+                    try {
+                        clsName = clo.applyx();
+
+                        fut.onDone(clsName);
+
+                        clsNameOrFuture = clsName;
+                    }
+                    catch (Throwable e) {
+                        fut.onDone(e);
+
+                        throw e;
+                    }
+                }
+                finally {
+                    if (clsName != null)
+                        map.replace(id, fut, clsName);
+                    else
+                        map.remove(id, fut);
+                }
+            }
+            else
+                clsNameOrFuture = old;
         }
 
-        return U.forName(clsName, ldr);
+        // Unwrap the existing object.
+        return unwrap(clsNameOrFuture);
+    }
+
+    /**
+     * Unwraps an object into the String. Expects the object be {@code null}, a String or a GridFutureAdapter.
+     *
+     * @param clsNameOrFuture Class name or future to unwrap.
+     * @return Unwrapped value.
+     * @throws IgniteCheckedException If future completed with an exception.
+     */
+    private String unwrap(Object clsNameOrFuture) throws IgniteCheckedException {
+        return clsNameOrFuture == null ? null :
+            clsNameOrFuture instanceof String ? (String)clsNameOrFuture :
+                ((GridFutureAdapter<String>)clsNameOrFuture).get();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java
index 8a2d38b61a48b..571091e004567 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java
@@ -21,12 +21,19 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.util.Collections;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.binary.BinaryObject;
-import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.MarshallerContextAdapter;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.marshaller.MarshallerContext;
+import org.apache.ignite.plugin.PluginProvider;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.IgniteTestResources;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -39,7 +46,9 @@ public class BinaryMarshallerLocalMetadataCache extends GridCommonAbstractTest {
         IgniteTestResources rsrcs) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName, rsrcs);
 
-        cfg.setMarshaller(new BinaryMarshaller());
+        cfg.setMarshaller(new BinaryMarshallerWrapper());
+
+        cfg.setCacheConfiguration(new CacheConfiguration().setName("part").setBackups(1));
 
         return cfg;
     }
@@ -66,26 +75,20 @@ public class BinaryMarshallerLocalMetadataCache extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testLocalMetadata() throws Exception {
-        final BinaryObject obj = grid(0).binary().toBinary(new OptimizedContainer(new Optimized()));
-
-        ClusterGroup remotes = grid(0).cluster().forRemotes();
+        final CyclicBarrier bar = new CyclicBarrier(64);
 
-        OptimizedContainer res = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
-            @Override public OptimizedContainer call() throws Exception {
-
-                return obj.deserialize();
-            }
-        });
+        awaitPartitionMapExchange();
 
-        OptimizedContainer res2 = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
-            @Override public OptimizedContainer call() throws Exception {
+        GridTestUtils.runMultiThreaded(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                bar.await();
 
-                return obj.deserialize();
+                return grid(0).binary().toBinary(new OptimizedContainer(new Optimized()));
             }
-        });
+        }, 64, "async-runner");
 
-        System.out.println(res);
-        System.out.println(res2);
+        // We expect 3 here because Externalizable classes are registered twice with different type IDs.
+        assertEquals(3, ((BinaryMarshallerWrapper)grid(0).configuration().getMarshaller()).registerClassCalled.get());
     }
 
     /**
@@ -123,4 +126,26 @@ private static class Optimized implements Externalizable {
             fld = U.readUTFStringNullable(in);
         }
     }
+
+    private static class BinaryMarshallerWrapper extends BinaryMarshaller {
+        private MarshallerContext ctx0 = new MarshallerContextAdapter(Collections.<PluginProvider>emptyList()) {
+            @Override protected boolean registerClassName(int id, String clsName) {
+                U.dumpStack(id + " " + clsName);
+                registerClassCalled.incrementAndGet();
+
+                return true;
+            }
+
+            @Override protected String className(int id) throws IgniteCheckedException {
+                return null;
+            }
+        };
+
+        private AtomicInteger registerClassCalled = new AtomicInteger();
+
+        /** {@inheritDoc} */
+        @Override public MarshallerContext getContext() {
+            return ctx0;
+        }
+    }
 }

From 4ffc3acfa1bc43bea8c79bfd1864787c15cfc4de Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 20 Jun 2017 07:59:09 +0300
Subject: [PATCH 310/446] IGNITE-5528 - IS_EVICT_DISABLED flag is not cleared
 on cache store exception.

---
 .../processors/cache/GridCacheAdapter.java    |  62 ++++---
 .../processors/cache/GridCacheEntryEx.java    |   3 +-
 .../processors/cache/GridCacheMapEntry.java   |   2 +-
 ...tionEvictionDuringReadThroughSelfTest.java | 160 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |   3 +
 5 files changed, 205 insertions(+), 25 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 47c9998239777..220c192f21c46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2061,35 +2061,18 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                     }
                                 });
 
-                                if (loaded.size() != loadKeys.size()) {
-                                    boolean needTouch =
-                                        tx0 == null || (!tx0.implicit() && tx0.isolation() == READ_COMMITTED);
-
-                                    for (Map.Entry<KeyCacheObject, EntryGetResult> e : loadKeys.entrySet()) {
-                                        if (loaded.contains(e.getKey()))
-                                            continue;
-
-                                        if (needTouch || e.getValue().reserved()) {
-                                            GridCacheEntryEx entry = peekEx(e.getKey());
-
-                                            if (entry != null) {
-                                                if (e.getValue().reserved())
-                                                    entry.clearReserveForLoad(e.getValue().version());
-
-                                                if (needTouch)
-                                                    ctx.evicts().touch(entry, topVer);
-                                            }
-                                        }
-                                    }
-                                }
+                                clearReservationsIfNeeded(topVer, loadKeys, loaded, tx0);
 
                                 return map;
                             }
                         }), true),
                         new C2<Map<K, V>, Exception, IgniteInternalFuture<Map<K, V>>>() {
                             @Override public IgniteInternalFuture<Map<K, V>> apply(Map<K, V> map, Exception e) {
-                                if (e != null)
+                                if (e != null) {
+                                    clearReservationsIfNeeded(topVer, loadKeys, loaded, tx0);
+
                                     return new GridFinishedFuture<>(e);
+                                }
 
                                 if (tx0 == null || (!tx0.implicit() && tx0.isolation() == READ_COMMITTED)) {
                                     Collection<KeyCacheObject> notFound = new HashSet<>(loadKeys.keySet());
@@ -2146,6 +2129,41 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
         }
     }
 
+    /**
+     * @param topVer Affinity topology version for which load was performed.
+     * @param loadKeys Keys to load.
+     * @param loaded Actually loaded keys.
+     * @param tx0 Transaction within which the load was run, if any.
+     */
+    private void clearReservationsIfNeeded(
+        AffinityTopologyVersion topVer,
+        Map<KeyCacheObject, EntryGetResult> loadKeys,
+        Collection<KeyCacheObject> loaded,
+        IgniteTxLocalAdapter tx0
+    ) {
+        if (loaded.size() != loadKeys.size()) {
+            boolean needTouch =
+                tx0 == null || (!tx0.implicit() && tx0.isolation() == READ_COMMITTED);
+
+            for (Map.Entry<KeyCacheObject, EntryGetResult> e : loadKeys.entrySet()) {
+                if (loaded.contains(e.getKey()))
+                    continue;
+
+                if (needTouch || e.getValue().reserved()) {
+                    GridCacheEntryEx entry = peekEx(e.getKey());
+
+                    if (entry != null) {
+                        if (e.getValue().reserved())
+                            entry.clearReserveForLoad(e.getValue().version());
+
+                        if (needTouch)
+                            ctx.evicts().touch(entry, topVer);
+                    }
+                }
+            }
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public final V getAndPut(K key, V val) throws IgniteCheckedException {
         return getAndPut(key, val, null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index ccd22850d9eb7..2574ee2623d0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -363,9 +363,8 @@ public EntryGetResult innerGetAndReserveForLoad(boolean readSwap,
 
     /**
      * @param ver Expected entry version.
-     * @throws IgniteCheckedException If failed.
      */
-    public void clearReserveForLoad(GridCacheVersion ver) throws IgniteCheckedException;
+    public void clearReserveForLoad(GridCacheVersion ver);
 
     /**
      * Reloads entry from underlying storage.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index ea01bca4bd50b..1b2d63bc624f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -3621,7 +3621,7 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
     }
 
     /** {@inheritDoc} */
-    @Override public synchronized void clearReserveForLoad(GridCacheVersion ver) throws IgniteCheckedException {
+    @Override public synchronized void clearReserveForLoad(GridCacheVersion ver) {
         if (obsoleteVersionExtras() != null)
             return;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java
new file mode 100644
index 0000000000000..d5351f7e37776
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.LinkedHashSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.expiry.CreatedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class GridCachePartitionEvictionDuringReadThroughSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration<Integer, Integer> ccfg =
+            new CacheConfiguration<Integer, Integer>()
+                .setName("config")
+                .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+                .setBackups(0) // No need for backup, just load from the store if needed
+                .setCacheStoreFactory(new CacheStoreFactory())
+                .setEvictionPolicy(new LruEvictionPolicy(100))
+                .setNearConfiguration(new NearCacheConfiguration<Integer, Integer>()
+                .setNearEvictionPolicy(new LruEvictionPolicy<Integer, Integer>()));
+
+        ccfg.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.MINUTES, 1)))
+            .setReadThrough(true)
+            .setWriteThrough(false);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testPartitionRent() throws Exception {
+        startGrid(0);
+
+        final AtomicBoolean done = new AtomicBoolean();
+
+        IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Integer>() {
+            @Override
+            public Integer call() throws Exception {
+                LinkedHashSet<Integer> set = new LinkedHashSet<>();
+
+                set.add(1);
+                set.add(2);
+                set.add(3);
+                set.add(4);
+                set.add(5);
+
+                while (!done.get()) {
+                    try {
+                        grid(0).<Integer, Integer>cache("config").getAll(set);
+                    }
+                    catch (Throwable ignore) {
+                        // No-op.
+                    }
+                }
+
+                return null;
+            }
+        }, 4, "loader");
+
+        IgniteInternalFuture<Void> startFut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                for (int i = 1; i < 5; i++) {
+                    startGrid(i);
+
+                    awaitPartitionMapExchange();
+                }
+
+                return null;
+            }
+        });
+
+        startFut.get();
+
+        done.set(true);
+
+        fut.get();
+    }
+
+    /**
+     *
+     */
+    private static class CacheStoreFactory implements Factory<CacheStore<Integer, Integer>> {
+        /** {@inheritDoc} */
+        @Override public CacheStore<Integer, Integer> create() {
+            return new HangingCacheStore();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class HangingCacheStore extends CacheStoreAdapter<Integer, Integer> {
+        /** */
+        private CountDownLatch releaseLatch = new CountDownLatch(1);
+
+        /** {@inheritDoc} */
+        @Override public Integer load(Integer key) throws CacheLoaderException {
+            if (key == 3)
+                throw new CacheLoaderException();
+
+            return key;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Integer, ? extends Integer> entry) throws CacheWriterException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 7582f5cc3ee88..ab0007b2f1459 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentFairAffinityTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest;
+import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest;
 import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest;
@@ -59,6 +60,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(IgniteCacheReadThroughEvictionsVariationsSuite.suite());
         suite.addTestSuite(IgniteCacheTxIteratorSelfTest.class);
 
+        suite.addTestSuite(GridCachePartitionEvictionDuringReadThroughSelfTest.class);
+
         return suite;
     }
 }

From 9036ad239d68eff663bc73a81baab2826b054d9a Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Jun 2017 18:25:31 +0300
Subject: [PATCH 311/446] Added MBean for system cache executors.

---
 .../apache/ignite/internal/IgniteKernal.java  | 38 +++++++++++++++----
 1 file changed, 30 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 4237aeebe49ce..2aa05f1b9ab11 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -61,10 +61,10 @@
 import org.apache.ignite.IgniteEvents;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteFileSystem;
+import org.apache.ignite.IgniteLock;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.IgniteQueue;
-import org.apache.ignite.IgniteLock;
 import org.apache.ignite.IgniteScheduler;
 import org.apache.ignite.IgniteSemaphore;
 import org.apache.ignite.IgniteServices;
@@ -115,7 +115,6 @@
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
 import org.apache.ignite.internal.processors.hadoop.Hadoop;
-import org.apache.ignite.internal.processors.hadoop.HadoopClassLoader;
 import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.job.GridJobProcessor;
 import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor;
@@ -175,6 +174,7 @@
 import org.apache.ignite.thread.IgniteStripedThreadPoolExecutor;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONFIG_URL;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DAEMON;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_NO_ASCII;
@@ -183,7 +183,6 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_STARVATION_CHECK_INTERVAL;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SUCCESS_FILE;
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.IgniteSystemProperties.getBoolean;
 import static org.apache.ignite.IgniteSystemProperties.snapshot;
 import static org.apache.ignite.internal.GridKernalState.DISCONNECTED;
@@ -200,7 +199,6 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CLIENT_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CONSISTENCY_CHECK_SKIPPED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DAEMON;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_GRID_NAME;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IPS;
@@ -209,11 +207,12 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JVM_ARGS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JVM_PID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LANG_RUNTIME;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM;
@@ -292,6 +291,14 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     @GridToStringExclude
     private ObjectName p2PExecSvcMBean;
 
+    /** */
+    @GridToStringExclude
+    private ObjectName utilityExecSvcMBean;
+
+    /** */
+    @GridToStringExclude
+    private ObjectName marshallerExecSvcMBean;
+
     /** */
     @GridToStringExclude
     private ObjectName restExecSvcMBean;
@@ -969,7 +976,14 @@ public void start(final IgniteConfiguration cfg,
             // Register MBeans.
             registerKernalMBean();
             registerLocalNodeMBean();
-            registerExecutorMBeans(execSvc, sysExecSvc, p2pExecSvc, mgmtExecSvc, restExecSvc);
+            registerExecutorMBeans(
+                execSvc,
+                sysExecSvc,
+                p2pExecSvc,
+                mgmtExecSvc,
+                restExecSvc,
+                utilityCachePool,
+                marshCachePool);
 
             // Lifecycle bean notifications.
             notifyLifecycleBeans(AFTER_NODE_START);
@@ -1523,15 +1537,21 @@ private void registerLocalNodeMBean() throws IgniteCheckedException {
     }
 
     /** @throws IgniteCheckedException If registration failed. */
-    private void registerExecutorMBeans(ExecutorService execSvc,
+    private void registerExecutorMBeans(
+        ExecutorService execSvc,
         ExecutorService sysExecSvc,
         ExecutorService p2pExecSvc,
         ExecutorService mgmtExecSvc,
-        ExecutorService restExecSvc) throws IgniteCheckedException {
+        ExecutorService restExecSvc,
+        ExecutorService utilityExecSvc,
+        ExecutorService marshallerExecSvc
+    ) throws IgniteCheckedException {
         pubExecSvcMBean = registerExecutorMBean(execSvc, "GridExecutionExecutor");
         sysExecSvcMBean = registerExecutorMBean(sysExecSvc, "GridSystemExecutor");
         mgmtExecSvcMBean = registerExecutorMBean(mgmtExecSvc, "GridManagementExecutor");
         p2PExecSvcMBean = registerExecutorMBean(p2pExecSvc, "GridClassLoadingExecutor");
+        utilityExecSvcMBean = registerExecutorMBean(utilityExecSvc ,"GridUtilityCacheExecutor");
+        marshallerExecSvcMBean = registerExecutorMBean(marshallerExecSvc, "GridMarshallerCacheExecutor");
 
         ConnectorConfiguration clientCfg = cfg.getConnectorConfiguration();
 
@@ -2027,6 +2047,8 @@ else if (state == STARTING)
                     unregisterMBean(p2PExecSvcMBean) &
                     unregisterMBean(kernalMBean) &
                     unregisterMBean(locNodeMBean) &
+                    unregisterMBean(utilityExecSvcMBean) &
+                    unregisterMBean(marshallerExecSvcMBean) &
                     unregisterMBean(restExecSvcMBean)
             ))
                 errOnStop = false;

From ed34a5dc681ea8f284f4d25c5575ad46569cc600 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Jun 2017 18:33:55 +0300
Subject: [PATCH 312/446] Partial fix of IGNITE-5562.

---
 .../spi/discovery/tcp/internal/TcpDiscoveryStatistics.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
index c79064491bb43..a69dbd9aa89aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
@@ -316,7 +316,9 @@ public synchronized void onMessageProcessingFinished(TcpDiscoveryAbstractMessage
      */
     public synchronized void onMessageSent(TcpDiscoveryAbstractMessage msg, long time, long ackTime) {
         assert msg != null;
-        assert time >= 0 : time;
+
+        if (time < 0)
+            time = 0;
 
         if (crdSinceTs.get() > 0 &&
             (msg instanceof TcpDiscoveryCustomEventMessage) ||

From d427021f329292fb69d348ba949ad1f8f1e9089e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Jun 2017 19:30:27 +0300
Subject: [PATCH 313/446] IGNITE-5552: ServiceProcessor recalculates all
 service assignments even if there is a pending topology change.

---
 .../service/GridServiceProcessor.java         | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 25a8edbfc7c17..9c746401b1479 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -169,7 +169,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
     private IgniteInternalCache<Object, Object> cache;
 
     /** Topology listener. */
-    private GridLocalEventListener topLsnr = new TopologyListener();
+    private final GridLocalEventListener topLsnr = new TopologyListener();
 
     static {
         Set<IgniteProductVersion> versions = new TreeSet<>(new Comparator<IgniteProductVersion>() {
@@ -1574,6 +1574,9 @@ private void onDeployment(final GridServiceDeployment dep, final AffinityTopolog
      * Topology listener.
      */
     private class TopologyListener implements GridLocalEventListener {
+        /** */
+        private volatile AffinityTopologyVersion currTopVer = null;
+
         /** {@inheritDoc} */
         @Override public void onEvent(Event evt) {
             if (!busyLock.enterBusy())
@@ -1601,6 +1604,8 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                 else
                     topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
 
+                currTopVer = topVer;
+
                 depExe.submit(new DepRunnable() {
                     @Override public void run0() {
                         ClusterNode oldest = ctx.discovery().oldestAliveCacheServerNode(topVer);
@@ -1618,6 +1623,19 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                                 boolean firstTime = true;
 
                                 while (it.hasNext()) {
+                                    // If topology changed again, let next event handle it.
+                                    AffinityTopologyVersion currTopVer0 = currTopVer;
+
+                                    if (currTopVer0 != topVer) {
+                                        if (log.isInfoEnabled())
+                                            log.info("Service processor detected a topology change during " +
+                                                "assignments calculation (will abort current iteration and " +
+                                                "re-calculate on the newer version): " +
+                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer + ']');
+
+                                        return;
+                                    }
+
                                     Cache.Entry<Object, Object> e = it.next();
 
                                     if (!(e.getKey() instanceof GridServiceDeploymentKey))

From f1b9cdc0716a1b23f54d68ce0fe19eb85107567d Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Wed, 14 Jun 2017 21:37:54 +0300
Subject: [PATCH 314/446] GG-12354: Partial fix of IGNITE-5473: Introduce
 troubleshooting logger.

---
 .../dht/GridDhtPartitionTopologyImpl.java     |  2 +-
 ...NearAtomicAbstractSingleUpdateRequest.java |  8 +++
 .../GridNearAtomicAbstractUpdateFuture.java   |  8 +++
 ...idNearAtomicSingleUpdateInvokeRequest.java |  6 ++
 .../atomic/GridNearAtomicUpdateFuture.java    |  1 +
 .../atomic/GridNearAtomicUpdateResponse.java  |  2 +-
 .../service/GridServiceProcessor.java         | 49 ++++++++++---
 .../service/ServiceContextImpl.java           |  2 +
 .../util/tostring/GridToStringBuilder.java    |  2 +-
 .../tcp/TcpCommunicationSpi.java              | 57 +++++++--------
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 72 +++++++++++++------
 .../spi/discovery/tcp/TcpDiscoverySpi.java    | 16 +++--
 12 files changed, 157 insertions(+), 68 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 1b4dcc9293dee..1ad0ff0538fa5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -210,7 +210,7 @@ private boolean waitForRent() throws IgniteCheckedException {
 
                             break;
                         }
-                        catch (IgniteFutureTimeoutCheckedException e) {
+                        catch (IgniteFutureTimeoutCheckedException ignore) {
                             if (dumpCnt++ < GridDhtPartitionsExchangeFuture.DUMP_PENDING_OBJECTS_THRESHOLD) {
                                 U.warn(log, "Failed to wait for partition eviction [" +
                                     "topVer=" + topVer +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractSingleUpdateRequest.java
index 61deeee1e1042..1e87e251a6680 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractSingleUpdateRequest.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
 import org.apache.ignite.plugin.extensions.communication.MessageWriter;
 import org.jetbrains.annotations.NotNull;
@@ -559,4 +560,11 @@ private boolean isFlag(int mask) {
     @Override public byte fieldsCount() {
         return 11;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(GridNearAtomicAbstractSingleUpdateRequest.class, this,
+            "nodeId", nodeId, "futVer", futVer, "topVer", topVer,
+            "parent", super.toString());
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
index c92e0f5ec5b42..936c8a3bd3c87 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
@@ -342,4 +343,11 @@ protected final GridCacheVersion addAtomicFuture(AffinityTopologyVersion topVer)
 
         return futVer;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(GridNearAtomicAbstractUpdateFuture.class, this,
+            "topLocked", topLocked, "remapCnt", remapCnt, "resCnt", resCnt, "err", err,
+            "parent", super.toString());
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
index df9e38441e79f..bf6bc17f16732 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
@@ -303,4 +304,9 @@ public GridNearAtomicSingleUpdateInvokeRequest() {
     @Override public byte directType() {
         return 126;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(GridNearAtomicSingleUpdateRequest.class, this, super.toString());
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index 950e5bdd2b9b1..a252d9ae30c2b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -90,6 +90,7 @@ public class GridNearAtomicUpdateFuture extends GridNearAtomicAbstractUpdateFutu
     private Map<UUID, GridNearAtomicFullUpdateRequest> mappings;
 
     /** Keys to remap. */
+    @GridToStringInclude
     private Collection<KeyCacheObject> remapKeys;
 
     /** Not null is operation is mapped to single node. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
index 2e38733e8b900..e48b7265f51c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
@@ -642,6 +642,6 @@ public synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwabl
 
     /** {@inheritDoc} */
     @Override public String toString() {
-        return S.toString(GridNearAtomicUpdateResponse.class, this, "parent");
+        return S.toString(GridNearAtomicUpdateResponse.class, this, super.toString());
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 9c746401b1479..4ecf674a48420 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -974,12 +974,24 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
                         int cnt = maxPerNodeCnt == 0 ? totalCnt == 0 ? 1 : totalCnt : maxPerNodeCnt;
 
                         cnts.put(n.id(), cnt);
+
+                        if (log.isInfoEnabled())
+                            log.info("Assigned service to primary node [svc=" + dep.configuration().getName() +
+                                ", topVer=" + topVer + ", node=" + n.id() + ']');
                     }
                 }
                 else {
                     if (!nodes.isEmpty()) {
                         int size = nodes.size();
 
+                        if (log.isInfoEnabled())
+                            log.info("Calculating assignments for service " +
+                                "[svc=" + dep.configuration().getName() +
+                                ", topVer=" + topVer +
+                                ", nodes=" + U.nodeIds(nodes) +
+                                ", oldAssignment=" + (oldAssigns == null ? "NA" : oldAssigns.assigns()) +
+                                ", totalCnt=" + totalCnt + ", maxPerNodeCnt=" + maxPerNodeCnt + ']');
+
                         int perNodeCnt = totalCnt != 0 ? totalCnt / size : maxPerNodeCnt;
                         int remainder = totalCnt != 0 ? totalCnt % size : 0;
 
@@ -1055,6 +1067,10 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
 
                 assigns.assigns(cnts);
 
+                if (log.isInfoEnabled())
+                    log.info("Calculated new assignments for service [svc=" + dep.configuration().getName() +
+                        ", assignment=" + assigns + ']');
+
                 cache.put(key, assigns);
 
                 tx.commit();
@@ -1095,6 +1111,10 @@ private void redeploy(GridServiceAssignments assigns) {
         Collection<ServiceContextImpl> toInit = new ArrayList<>();
 
         synchronized (ctxs) {
+            if (log.isInfoEnabled())
+                log.info("Updating service deployment [locNodeId=" + ctx.localNodeId() +
+                    ", svc=" + assigns.name() + ", assigns=" + assigns + ']');
+
             if (ctxs.size() > assignCnt) {
                 int cancelCnt = ctxs.size() - assignCnt;
 
@@ -1483,6 +1503,10 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
         }
         // Handle undeployment.
         else {
+            if (log.isInfoEnabled())
+                log.info("Received undeploy notification, will cancel all instances of the service " +
+                    "[svc=" + e.getKey().name() + "]");
+
             String name = e.getKey().name();
 
             undeploy(name);
@@ -1524,13 +1548,15 @@ private void onDeployment(final GridServiceDeployment dep, final AffinityTopolog
         try {
             AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-            // If topology version changed, reassignment will happen from topology event.
-            if (newTopVer.equals(topVer))
-                reassign(dep, topVer);
-        }
-        catch (IgniteCheckedException e) {
-            if (!(e instanceof ClusterTopologyCheckedException))
-                log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
+                // If topology version changed, reassignment will happen from topology event.
+                if (newTopVer.equals(topVer)){
+                    if (log.isInfoEnabled())
+                        log.info("Will calculate new service deployment assignment (new service has been deployed) [" +
+                            "svc=" + dep.configuration().getName() + ", topVer=" + topVer + ']');reassign(dep, topVer);}
+            }
+            catch (IgniteCheckedException e) {
+                if (!(e instanceof ClusterTopologyCheckedException))
+                    log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
 
             AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
@@ -1817,9 +1843,12 @@ private void undeploy(String name) {
             ctxs = locSvcs.remove(name);
         }
 
-        if (ctxs != null) {
-            synchronized (ctxs) {
-                cancel(ctxs, ctxs.size());
+            if (ctxs != null) {
+                synchronized (ctxs) {
+                    if (log.isInfoEnabled())
+                        log.info("Undeploying services [svc=" + e.getKey().name() + ", assigns=" + assigns +
+                            ", ctxs=" + ctxs + ']');cancel(ctxs, ctxs.size());
+
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceContextImpl.java
index 66061d5c6f13e..490b5986faff9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceContextImpl.java
@@ -23,6 +23,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceContext;
@@ -48,6 +49,7 @@ public class ServiceContextImpl implements ServiceContext {
     private final String cacheName;
 
     /** Affinity key. */
+    @GridToStringInclude
     private final Object affKey;
 
     /** Executor service. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
index 6807b3f8ad0ed..e4c8c66c05580 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
@@ -87,7 +87,7 @@ public class GridToStringBuilder {
     private static final ReadWriteLock rwLock = new ReentrantReadWriteLock();
 
     /** Maximum number of collection (map) entries to print. */
-    public static final int MAX_COL_SIZE = 100;
+    public static final int MAX_COL_SIZE = 200;
 
     /** {@link IgniteSystemProperties#IGNITE_TO_STRING_INCLUDE_SENSITIVE} */
     public static final boolean INCLUDE_SENSITIVE =
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 6dba6b2b5fd0c..80cb77633b844 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -422,8 +422,8 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                 final ClusterNode rmtNode = getSpiContext().node(sndId);
 
                 if (rmtNode == null) {
-                    if (log.isDebugEnabled())
-                        log.debug("Close incoming connection, unknown node: " + sndId);
+                    U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId +
+                        ", ses=" + ses + ']');
 
                     ses.close();
 
@@ -441,10 +441,10 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
 
                 if (oldClient != null) {
                     if (oldClient instanceof GridTcpNioCommunicationClient) {
-                        if (log.isDebugEnabled())
-                            log.debug("Received incoming connection when already connected " +
-                                    "to this node, rejecting [locNode=" + locNode.id() +
-                                    ", rmtNode=" + sndId + ']');
+                        if (log.isInfoEnabled())
+                            log.info("Received incoming connection when already connected " +
+                                "to this node, rejecting [locNode=" + locNode.id() +
+                                ", rmtNode=" + sndId + ']');
 
                         ses.send(new RecoveryLastReceivedMessage(-1));
 
@@ -472,10 +472,10 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
 
                     if (oldClient != null) {
                         if (oldClient instanceof GridTcpNioCommunicationClient) {
-                            if (log.isDebugEnabled())
-                                log.debug("Received incoming connection when already connected " +
-                                        "to this node, rejecting [locNode=" + locNode.id() +
-                                        ", rmtNode=" + sndId + ']');
+                            if (log.isInfoEnabled())
+                                log.info("Received incoming connection when already connected " +
+                                    "to this node, rejecting [locNode=" + locNode.id() +
+                                    ", rmtNode=" + sndId + ']');
 
                             ses.send(new RecoveryLastReceivedMessage(-1));
 
@@ -495,7 +495,7 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
 
                     if (log.isDebugEnabled())
                         log.debug("Received incoming connection from remote node " +
-                                "[rmtNode=" + rmtNode.id() + ", reserved=" + reserved + ']');
+                            "[rmtNode=" + rmtNode.id() + ", reserved=" + reserved + ']');
 
                     if (reserved) {
                         try {
@@ -511,11 +511,11 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                 }
                 else {
                     if (oldFut instanceof ConnectFuture && locNode.order() < rmtNode.order()) {
-                        if (log.isDebugEnabled()) {
-                            log.debug("Received incoming connection from remote node while " +
-                                    "connecting to this node, rejecting [locNode=" + locNode.id() +
-                                    ", locNodeOrder=" + locNode.order() + ", rmtNode=" + rmtNode.id() +
-                                    ", rmtNodeOrder=" + rmtNode.order() + ']');
+                        if (log.isInfoEnabled()) {
+                            log.info("Received incoming connection from remote node while " +
+                                "connecting to this node, rejecting [locNode=" + locNode.id() +
+                                ", locNodeOrder=" + locNode.order() + ", rmtNode=" + rmtNode.id() +
+                                ", rmtNodeOrder=" + rmtNode.order() + ']');
                         }
 
                         ses.send(new RecoveryLastReceivedMessage(-1));
@@ -2502,11 +2502,11 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                                 ", addr=" + addr + ", err=" + e + ']');
 
                     if (attempt == reconCnt || connTimeout0 > maxConnTimeout) {
-                        if (log.isDebugEnabled())
-                            log.debug("Handshake timedout (will stop attempts to perform the handshake) " +
-                                "[timeout=" + connTimeout0 + ", maxConnTimeout=" + maxConnTimeout +
-                                ", attempt=" + attempt + ", reconCnt=" + reconCnt +
-                                ", err=" + e.getMessage() + ", addr=" + addr + ']');
+                        U.warn(log, "Handshake timedout (will stop attempts to perform the handshake) " +
+                            "[node=" + node.id() + ", timeout=" + connTimeout0 +
+                            ", maxConnTimeout=" + maxConnTimeout +
+                            ", attempt=" + attempt + ", reconCnt=" + reconCnt +
+                            ", err=" + e.getMessage() + ", addr=" + addr + ']');
 
                         if (errs == null)
                             errs = new IgniteCheckedException("Failed to connect to node (is node still alive?). " +
@@ -2540,6 +2540,10 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
 
                     boolean failureDetThrReached = timeoutHelper.checkFailureTimeoutReached(e);
 
+                    U.error(log, "Failed to establish connection to a remote node [node=" + node +
+                        ", addr=" + addr + ", connectAttempts=" + connectAttempts +
+                        ", failureDetThrReached" + failureDetThrReached + ']', e);
+
                     if (failureDetThrReached)
                         LT.warn(log, "Connect timed out (consider increasing 'failureDetectionTimeout' " +
                             "configuration property) [addr=" + addr + ", failureDetectionTimeout=" +
@@ -2582,13 +2586,10 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     "[addrs=" + addrs + ']');
 
             if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
-                X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
-                    IgniteSpiOperationTimeoutException.class)) {
-                LT.warn(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
-                    "cluster [" +
-                    "rmtNode=" + node +
-                    ", err=" + errs +
-                    ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
+                    X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
+                        IgniteSpiOperationTimeoutException.class)) {
+                    U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
+                        "cluster [" + "rmtNode=" + node + ']', errs);
 
                 getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
                     "rmtNode=" + node +
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 6bdd80cbe6ff6..2f7e9b470aae6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -514,9 +514,6 @@ else if (log.isInfoEnabled()) {
     @Override public boolean pingNode(UUID nodeId) {
         assert nodeId != null;
 
-        if (log.isDebugEnabled())
-            log.debug("Pinging node: " + nodeId + "]");
-
         if (nodeId == getLocalNodeId())
             return true;
 
@@ -528,8 +525,18 @@ else if (log.isInfoEnabled()) {
         if (!nodeAlive(nodeId))
             return false;
 
+        long start = U.currentTimeMillis();
+
+        if (log.isInfoEnabled())
+            log.info("Pinging node: " + nodeId);
+
         boolean res = pingNode(node);
 
+        long end = System.currentTimeMillis();
+
+        if (log.isInfoEnabled())
+            log.info("Finished node ping [nodeId=" + nodeId + ", res=" + res + ", time=" + (end - start) + "ms]");
+
         if (!res && !node.isClient() && nodeAlive(nodeId)) {
             LT.warn(log, "Failed to ping node (status check will be initiated): " + nodeId);
 
@@ -1686,9 +1693,9 @@ TcpDiscoveryNodesRing ring() {
 
         assert log.isInfoEnabled();
 
-        synchronized (mux) {
-            StringBuilder b = new StringBuilder(U.nl());
+        StringBuilder b = new StringBuilder(U.nl());
 
+        synchronized (mux) {
             b.append(">>>").append(U.nl());
             b.append(">>>").append("Dumping discovery SPI debug info.").append(U.nl());
             b.append(">>>").append(U.nl());
@@ -1735,9 +1742,9 @@ TcpDiscoveryNodesRing ring() {
             b.append(U.nl());
 
             b.append("Stats: ").append(spi.stats).append(U.nl());
-
-            U.quietAndInfo(log, b.toString());
         }
+
+        U.quietAndInfo(log, b.toString());
     }
 
     /**
@@ -3171,12 +3178,8 @@ else if (e instanceof SocketTimeoutException ||
 
                             // If node existed on connection initialization we should check
                             // whether it has not gone yet.
-                            if (nextNodeExists)
-                                U.warn(log, "Failed to send message to next node [msg=" + msg + ", next=" + next +
-                                    ", errMsg=" + (err != null ? err.getMessage() : "N/A") + ']');
-                            else if (log.isDebugEnabled())
-                                log.debug("Failed to send message to next node [msg=" + msg + ", next=" + next +
-                                    ", errMsg=" + (err != null ? err.getMessage() : "N/A") + ']');
+                            U.warn(log, "Failed to send message to next node [msg=" + msg + ", next=" + next +
+                                ", errMsg=" + (err != null ? err.getMessage() : "N/A") + ']');
                         }
                     }
 
@@ -3358,6 +3361,8 @@ private void processJoinRequestMessage(final TcpDiscoveryJoinRequestMessage msg)
                 if (existingNode != null) {
                     if (!node.socketAddresses().equals(existingNode.socketAddresses())) {
                         if (!pingNode(existingNode)) {
+                            U.warn(log, "Sending node failed message for existing node: " + node);
+
                             addMessage(new TcpDiscoveryNodeFailedMessage(locNodeId,
                                 existingNode.id(), existingNode.internalOrder()));
 
@@ -5421,8 +5426,7 @@ private void checkFailedNodesList() {
 
             if (msgs != null) {
                 for (TcpDiscoveryNodeFailedMessage msg : msgs) {
-                    if (log.isDebugEnabled())
-                        log.debug("Add node failed message for node from failed nodes list: " + msg);
+                    U.warn(log, "Added node failed message for node from failed nodes list: " + msg);
 
                     addMessage(msg);
                 }
@@ -5641,8 +5645,9 @@ private class TcpServer extends IgniteSpiThread {
 
                     long tstamp = U.currentTimeMillis();
 
-                    if (log.isDebugEnabled())
-                        log.debug("Accepted incoming connection from addr: " + sock.getInetAddress());
+                    if (log.isInfoEnabled())
+                        log.info("TCP discovery accepted incoming connection " +
+                            "[rmtAddr=" + sock.getInetAddress() + ", rmtPort=" + sock.getPort() + ']');
 
                     SocketReader reader = new SocketReader(sock);
 
@@ -5650,6 +5655,10 @@ private class TcpServer extends IgniteSpiThread {
                         readers.add(reader);
                     }
 
+                    if (log.isInfoEnabled())
+                        log.info("TCP discovery spawning a new thread for connection " +
+                            "[rmtAddr=" + sock.getInetAddress() + ", rmtPort=" + sock.getPort() + ']');
+
                     reader.start();
 
                     spi.stats.onServerSocketInitialized(U.currentTimeMillis() - tstamp);
@@ -5712,6 +5721,12 @@ private class SocketReader extends IgniteSpiThread {
 
             ClientMessageWorker clientMsgWrk = null;
 
+            SocketAddress rmtAddr = sock.getRemoteSocketAddress();
+
+            if (log.isInfoEnabled())
+                log.info("Started serving remote node connection [rmtAddr=" + rmtAddr +
+                    ", rmtPort=" + sock.getPort() + ']');
+
             try {
                 InputStream in;
 
@@ -5742,11 +5757,11 @@ private class SocketReader extends IgniteSpiThread {
                         else {
                             if (log.isDebugEnabled())
                                 log.debug("Failed to read magic header (too few bytes received) " +
-                                    "[rmtAddr=" + sock.getRemoteSocketAddress() +
+                                    "[rmtAddr=" + rmtAddr +
                                     ", locAddr=" + sock.getLocalSocketAddress() + ']');
 
                             LT.warn(log, "Failed to read magic header (too few bytes received) [rmtAddr=" +
-                                sock.getRemoteSocketAddress() + ", locAddr=" + sock.getLocalSocketAddress() + ']');
+                                rmtAddr + ", locAddr=" + sock.getLocalSocketAddress() + ']');
 
                             return;
                         }
@@ -5758,7 +5773,7 @@ private class SocketReader extends IgniteSpiThread {
                                 "this Ignite port?" +
                                 (!spi.isSslEnabled() ? " missed SSL configuration?" : "" ) +
                                 ") " +
-                                "[rmtAddr=" + sock.getRemoteSocketAddress() +
+                                "[rmtAddr=" + rmtAddr +
                                 ", locAddr=" + sock.getLocalSocketAddress() + ']');
 
                         LT.warn(log, "Unknown connection detected (is some other software connecting to " +
@@ -5779,6 +5794,11 @@ private class SocketReader extends IgniteSpiThread {
                         if (!spi.isNodeStopping0()) {
                             TcpDiscoveryPingRequest req = (TcpDiscoveryPingRequest)msg;
 
+                            if (log.isInfoEnabled())
+                                log.info("Received ping request from the remote node " +
+                                    "[rmtNodeId=" + msg.creatorNodeId() +
+                                    ", rmtAddr=" + rmtAddr + ", rmtPort=" + sock.getPort() + "]");
+
                             TcpDiscoveryPingResponse res = new TcpDiscoveryPingResponse(locNodeId);
 
                             IgniteSpiOperationTimeoutHelper timeoutHelper =
@@ -5792,6 +5812,12 @@ private class SocketReader extends IgniteSpiThread {
                             }
 
                             spi.writeToSocket(sock, res, timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()));
+
+                            sock.shutdownOutput();
+
+                            if (log.isInfoEnabled())
+                                log.info("Finished writing ping response " + "[rmtNodeId=" + msg.creatorNodeId() +
+                                    ", rmtAddr=" + rmtAddr + ", rmtPort=" + sock.getPort() + "]");
                         }
                         else if (log.isDebugEnabled())
                             log.debug("Ignore ping request, node is stopping.");
@@ -5917,7 +5943,7 @@ else if ((X.hasCause(e, ObjectStreamException.class) || !sock.isClosed())
                     else if (e.hasCause(ClassNotFoundException.class))
                         LT.warn(log, "Failed to read message due to ClassNotFoundException " +
                             "(make sure same versions of all classes are available on all nodes) " +
-                            "[rmtAddr=" + sock.getRemoteSocketAddress() +
+                            "[rmtAddr=" + rmtAddr +
                             ", err=" + X.cause(e, ClassNotFoundException.class).getMessage() + ']');
 
                         // Always report marshalling problems.
@@ -6208,6 +6234,10 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage) {
                 }
 
                 U.closeQuiet(sock);
+
+                if (log.isInfoEnabled())
+                    log.info("Finished serving remote node connection [rmtAddr=" + rmtAddr +
+                        ", rmtPort=" + sock.getPort());
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index cefbcd675a303..3cc4ee53242b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1482,7 +1482,8 @@ protected <T> T readMessage(Socket sock, @Nullable InputStream in, long timeout)
         catch (IOException | IgniteCheckedException e) {
             if (X.hasCause(e, SocketTimeoutException.class))
                 LT.warn(log, "Timed out waiting for message to be read (most probably, the reason is " +
-                    "in long GC pauses on remote node) [curTimeout=" + timeout + ']');
+                    "long GC pauses on remote node) [curTimeout=" + timeout +
+                    ", rmtAddr=" + sock.getRemoteSocketAddress() + ", rmtPort=" + sock.getPort() + ']');
 
             throw e;
         }
@@ -1523,8 +1524,9 @@ protected int readReceipt(Socket sock, long timeout) throws IOException {
         catch (SocketTimeoutException e) {
             LT.warn(log, "Timed out waiting for message delivery receipt (most probably, the reason is " +
                 "in long GC pauses on remote node; consider tuning GC and increasing 'ackTimeout' " +
-                "configuration property). Will retry to send message with increased timeout. " +
-                "Current timeout: " + timeout + '.');
+                "configuration property). Will retry to send message with increased timeout " +
+                "[currentTimeout=" + timeout + ", rmtAddr=" + sock.getRemoteSocketAddress() +
+                ", rmtPort=" + sock.getPort() + ']');
 
             stats.onAckTimeout();
 
@@ -2066,9 +2068,11 @@ boolean cancel() {
 
                 LT.warn(log, "Socket write has timed out (consider increasing " +
                     (failureDetectionTimeoutEnabled() ?
-                    "'IgniteConfiguration.failureDetectionTimeout' configuration property) [" +
-                    "failureDetectionTimeout=" + failureDetectionTimeout() + ']' :
-                    "'sockTimeout' configuration property) [sockTimeout=" + sockTimeout + ']'));
+                        "'IgniteConfiguration.failureDetectionTimeout' configuration property) [" +
+                        "failureDetectionTimeout=" + failureDetectionTimeout() :
+                        "'sockTimeout' configuration property) [sockTimeout=" + sockTimeout) +
+                    ", rmtAddr=" + sock.getRemoteSocketAddress() + ", rmtPort=" + sock.getPort() +
+                    ", sockTimeout=" + sockTimeout + ']');
 
                 stats.onSocketTimeout();
             }

From beb2409cfe2045789443d47de735d879961d371e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 23 Jun 2017 12:26:06 +0300
Subject: [PATCH 315/446] GG-12352: Forcible node drop makes cluster instable
 in some cases. Disable forcible node drop by default.

---
 .../apache/ignite/IgniteSystemProperties.java |  6 +++
 .../service/GridServiceProcessor.java         |  2 +-
 .../spi/IgniteSpiOperationTimeoutHelper.java  |  6 ++-
 .../tcp/TcpCommunicationSpi.java              | 46 +++++++++++++------
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  6 ++-
 .../tcp/TcpCommunicationSpiDropNodesTest.java | 16 +++++++
 .../TcpCommunicationSpiFaultyClientTest.java  | 15 ++++++
 7 files changed, 79 insertions(+), 18 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 775aae03a1ef4..f473c51e74a93 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -533,6 +533,12 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_SECURITY_COMPATIBILITY_MODE = "IGNITE_SECURITY_COMPATIBILITY_MODE";
 
+    /**
+     * If this property is set, a node will forcible fail a remote node when it fails to establish a communication
+     * connection.
+     */
+    public static final String IGNITE_ENABLE_FORCIBLE_NODE_KILL = "IGNITE_ENABLE_FORCIBLE_NODE_KILL";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 4ecf674a48420..75c79a3a31fdc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1604,7 +1604,7 @@ private class TopologyListener implements GridLocalEventListener {
         private volatile AffinityTopologyVersion currTopVer = null;
 
         /** {@inheritDoc} */
-        @Override public void onEvent(Event evt) {
+        @Override public void onEvent(final Event evt) {
             if (!busyLock.enterBusy())
                 return;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
index e17b0dd1b196f..1d9fa94f44475 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
@@ -96,7 +96,9 @@ public boolean checkFailureTimeoutReached(Exception e) {
         if (!failureDetectionTimeoutEnabled)
             return false;
 
-        return e instanceof IgniteSpiOperationTimeoutException || e instanceof SocketTimeoutException ||
-            X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketException.class);
+        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class))
+            return true;
+
+        return X.hasCause(e, SocketException.class) && (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 80cb77633b844..98183d8afbd7a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -23,6 +23,7 @@
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -53,6 +54,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.AddressResolver;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -328,6 +330,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     /** */
     private ConnectGateway connectGate;
 
+    /** */
+    private boolean enableForcibleNodeKill = IgniteSystemProperties
+        .getBoolean(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
     /** Server listener. */
     private final GridNioServerListener<Message> srvLsnr =
         new GridNioServerListenerAdapter<Message>() {
@@ -2073,8 +2079,13 @@ private GridCommunicationClient reserveClient(ClusterNode node) throws IgniteChe
                                     }
                                 }
                             }
-                            else
+                            else {
                                 U.sleep(200);
+
+                                if (getSpiContext().node(node.id()) == null)
+                                    throw new ClusterTopologyCheckedException("Failed to send message " +
+                                        "(node left topology): " + node);
+                            }
                         }
 
                         fut.onDone(client0);
@@ -2470,6 +2481,7 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
+                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +
@@ -2561,8 +2573,10 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     errs.addSuppressed(new IgniteCheckedException("Failed to connect to address: " + addr, e));
 
                     // Reconnect for the second time, if connection is not established.
-                    if (!failureDetThrReached && connectAttempts < 2 &&
-                        (e instanceof ConnectException || X.hasCause(e, ConnectException.class))) {
+                    if (!failureDetThrReached && connectAttempts < 5 &&
+                        (X.hasCause(e, ConnectException.class) || X.hasCause(e, SocketTimeoutException.class))) {
+                        U.sleep(200);
+
                         connectAttempts++;
 
                         continue;
@@ -2585,19 +2599,22 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     "operating system firewall is disabled on local and remote hosts) " +
                     "[addrs=" + addrs + ']');
 
-            if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
+            if (enableForcibleNodeKill) {
+                if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
                     X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
                         IgniteSpiOperationTimeoutException.class)) {
                     U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
                         "cluster [" + "rmtNode=" + node + ']', errs);
 
-                getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
-                    "rmtNode=" + node +
-                    ", errs=" + errs +
-                    ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
+                    getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
+                        "rmtNode=" + node +
+                        ", errs=" + errs +
+                        ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
+                }
             }
 
-            throw errs;
+            if (X.hasCause(errs, ConnectException.class))
+                throw errs;
         }
 
         return client;
@@ -2829,8 +2846,9 @@ else if (log.isDebugEnabled())
 
             // Ignoring whatever happened after timeout - reporting only timeout event.
             if (!cancelled)
-                throw new HandshakeTimeoutException("Failed to perform handshake due to timeout (consider increasing " +
-                    "'connectionTimeout' configuration property).");
+                throw new HandshakeTimeoutException(
+                    new IgniteSpiOperationTimeoutException("Failed to perform handshake due to timeout " +
+                        "(consider increasing 'connectionTimeout' configuration property)."));
         }
 
         return rcvCnt;
@@ -2980,10 +2998,10 @@ private static class HandshakeTimeoutException extends IgniteCheckedException {
         private static final long serialVersionUID = 0L;
 
         /**
-         * @param msg Message.
+         * @param cause Exception cause
          */
-        HandshakeTimeoutException(String msg) {
-            super(msg);
+        HandshakeTimeoutException(IgniteSpiOperationTimeoutException cause) {
+            super(cause);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 2f7e9b470aae6..856370c2a0ee3 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1146,6 +1146,8 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
             boolean openSock = false;
 
+            boolean wasHandshake = false;
+
             Socket sock = null;
 
             try {
@@ -1163,6 +1165,8 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(
                     ackTimeout0));
 
+                wasHandshake = true;
+
                 if (msg instanceof TcpDiscoveryJoinRequestMessage) {
                     boolean ignore = false;
 
@@ -1264,7 +1268,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 if (!spi.failureDetectionTimeoutEnabled() && ++reconCnt == spi.getReconnectCount())
                     break;
 
-                if (!openSock) {
+                if (!openSock || !wasHandshake) {
                     // Reconnect for the second time, if connection is not established.
                     if (connectAttempts < 2) {
                         connectAttempts++;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
index d29231e449b0b..7b438d6795a8b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
@@ -25,6 +25,7 @@
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.TimeUnit;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
@@ -41,6 +42,7 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.config.GridTestProperties;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
@@ -82,6 +84,20 @@ public class TcpCommunicationSpiDropNodesTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
index 6e99487bd46ae..33625e4b9edb4 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
@@ -95,6 +96,20 @@ public class TcpCommunicationSpiFaultyClientTest extends GridCommonAbstractTest
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();

From 802f18fc250cbae8959192c78bb28dc525ed3cf7 Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Fri, 23 Jun 2017 00:24:57 +0300
Subject: [PATCH 316/446] Fix compilation

---
 .../internal/processors/service/GridServiceProcessor.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 75c79a3a31fdc..009bf47c57101 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1846,7 +1846,7 @@ private void undeploy(String name) {
             if (ctxs != null) {
                 synchronized (ctxs) {
                     if (log.isInfoEnabled())
-                        log.info("Undeploying services [svc=" + e.getKey().name() + ", assigns=" + assigns +
+                        log.info("Undeploying services [svc=" + name +
                             ", ctxs=" + ctxs + ']');cancel(ctxs, ctxs.size());
 
             }

From bfec212b1ece0e9e791de6dfb642324834fa77ca Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Fri, 23 Jun 2017 00:24:57 +0300
Subject: [PATCH 317/446] Partially reverted GG-12352.

---
 .../spi/IgniteSpiOperationTimeoutHelper.java  |  4 +--
 .../tcp/TcpCommunicationSpi.java              |  1 -
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  6 +----
 .../apache/ignite/spi/GridTcpForwarder.java   | 26 +++++++++++++++++++
 4 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
index 1d9fa94f44475..33896361e2c21 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
@@ -96,9 +96,9 @@ public boolean checkFailureTimeoutReached(Exception e) {
         if (!failureDetectionTimeoutEnabled)
             return false;
 
-        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class))
+        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class, SocketException.class))
             return true;
 
-        return X.hasCause(e, SocketException.class) && (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
+        return (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 98183d8afbd7a..a87734e5c73e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2481,7 +2481,6 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
-                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 856370c2a0ee3..2f7e9b470aae6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1146,8 +1146,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
             boolean openSock = false;
 
-            boolean wasHandshake = false;
-
             Socket sock = null;
 
             try {
@@ -1165,8 +1163,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(
                     ackTimeout0));
 
-                wasHandshake = true;
-
                 if (msg instanceof TcpDiscoveryJoinRequestMessage) {
                     boolean ignore = false;
 
@@ -1268,7 +1264,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 if (!spi.failureDetectionTimeoutEnabled() && ++reconCnt == spi.getReconnectCount())
                     break;
 
-                if (!openSock || !wasHandshake) {
+                if (!openSock) {
                     // Reconnect for the second time, if connection is not established.
                     if (connectAttempts < 2) {
                         connectAttempts++;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
index d08321e18e8b5..68d97c1a3932b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
@@ -23,6 +23,7 @@
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -85,6 +86,10 @@ public GridTcpForwarder(
                                 outputCon.getInputStream(), inputCon.getOutputStream()
                             );
 
+                            //Force closing sibling if one of thread failed.
+                            forwardThread1.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread2));
+                            forwardThread2.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread1));
+
                             forwardThread1.start();
                             forwardThread2.start();
 
@@ -127,6 +132,25 @@ public GridTcpForwarder(
         U.join(mainThread, log);
     }
 
+    /**
+     *
+     */
+    private static class ForwarderExceptionHandler implements Thread.UncaughtExceptionHandler {
+        /** */
+        private Thread siblingThread;
+
+        /** */
+        public ForwarderExceptionHandler(Thread siblingThread) {
+
+            this.siblingThread = siblingThread;
+        }
+
+        /** */
+        @Override public void uncaughtException(Thread t, Throwable e) {
+            siblingThread.interrupt();
+        }
+    }
+
     /**
      * Thread reads data from input stream and write to output stream.
      */
@@ -166,6 +190,8 @@ private ForwardThread(String name, InputStream inputStream, OutputStream outputS
             }
             catch (IOException e) {
                 log.error("IOException while forwarding data [threadName=" + getName() + "]", e);
+
+                throw new IgniteException(e);
             }
         }
     }

From 1abc14fdc4e39d8245c3e50fb2cf3d183df08021 Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Fri, 23 Jun 2017 00:24:57 +0300
Subject: [PATCH 318/446] Partially reverted GG-12352.

---
 .../service/GridServiceProcessor.java         |  2 +-
 .../spi/IgniteSpiOperationTimeoutHelper.java  |  4 +--
 .../tcp/TcpCommunicationSpi.java              |  2 --
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  6 +----
 .../apache/ignite/spi/GridTcpForwarder.java   | 26 +++++++++++++++++++
 .../tcp/TcpCommunicationSpiDropNodesTest.java |  1 -
 .../TcpCommunicationSpiFaultyClientTest.java  |  5 ----
 7 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 009bf47c57101..20bcff7c780d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1657,7 +1657,7 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                                             log.info("Service processor detected a topology change during " +
                                                 "assignments calculation (will abort current iteration and " +
                                                 "re-calculate on the newer version): " +
-                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer + ']');
+                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer0 + ']');
 
                                         return;
                                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
index 1d9fa94f44475..33896361e2c21 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
@@ -96,9 +96,9 @@ public boolean checkFailureTimeoutReached(Exception e) {
         if (!failureDetectionTimeoutEnabled)
             return false;
 
-        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class))
+        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class, SocketException.class))
             return true;
 
-        return X.hasCause(e, SocketException.class) && (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
+        return (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 98183d8afbd7a..1bfa56accf2e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -23,7 +23,6 @@
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -2481,7 +2480,6 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
-                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 856370c2a0ee3..2f7e9b470aae6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1146,8 +1146,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
             boolean openSock = false;
 
-            boolean wasHandshake = false;
-
             Socket sock = null;
 
             try {
@@ -1165,8 +1163,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(
                     ackTimeout0));
 
-                wasHandshake = true;
-
                 if (msg instanceof TcpDiscoveryJoinRequestMessage) {
                     boolean ignore = false;
 
@@ -1268,7 +1264,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 if (!spi.failureDetectionTimeoutEnabled() && ++reconCnt == spi.getReconnectCount())
                     break;
 
-                if (!openSock || !wasHandshake) {
+                if (!openSock) {
                     // Reconnect for the second time, if connection is not established.
                     if (connectAttempts < 2) {
                         connectAttempts++;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
index d08321e18e8b5..68d97c1a3932b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
@@ -23,6 +23,7 @@
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -85,6 +86,10 @@ public GridTcpForwarder(
                                 outputCon.getInputStream(), inputCon.getOutputStream()
                             );
 
+                            //Force closing sibling if one of thread failed.
+                            forwardThread1.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread2));
+                            forwardThread2.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread1));
+
                             forwardThread1.start();
                             forwardThread2.start();
 
@@ -127,6 +132,25 @@ public GridTcpForwarder(
         U.join(mainThread, log);
     }
 
+    /**
+     *
+     */
+    private static class ForwarderExceptionHandler implements Thread.UncaughtExceptionHandler {
+        /** */
+        private Thread siblingThread;
+
+        /** */
+        public ForwarderExceptionHandler(Thread siblingThread) {
+
+            this.siblingThread = siblingThread;
+        }
+
+        /** */
+        @Override public void uncaughtException(Thread t, Throwable e) {
+            siblingThread.interrupt();
+        }
+    }
+
     /**
      * Thread reads data from input stream and write to output stream.
      */
@@ -166,6 +190,8 @@ private ForwardThread(String name, InputStream inputStream, OutputStream outputS
             }
             catch (IOException e) {
                 log.error("IOException while forwarding data [threadName=" + getName() + "]", e);
+
+                throw new IgniteException(e);
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
index 7b438d6795a8b..58a0dc170f9d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
@@ -42,7 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.config.GridTestProperties;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
index 33625e4b9edb4..dcfa91d6431a9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
@@ -18,16 +18,13 @@
 package org.apache.ignite.spi.communication.tcp;
 
 import java.io.IOException;
-import java.io.OutputStream;
 import java.net.InetAddress;
 import java.net.ServerSocket;
-import java.net.Socket;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
@@ -45,8 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 

From 897f4c00c4945eda3f9f4a41d064ded2f6f27ccc Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 27 Jun 2017 12:55:11 +0300
Subject: [PATCH 319/446] gg-12133 Deadlock for metadata update from
 GridCacheMapEntry.innerUpdate

---
 .../checkpoint/GridCheckpointManager.java     |   2 +-
 .../managers/communication/GridIoManager.java |  10 +-
 .../communication/GridMessageListener.java    |   3 +-
 .../GridDeploymentCommunication.java          |   4 +-
 .../eventstorage/GridEventStorageManager.java |   4 +-
 .../processors/cache/GridCacheIoManager.java  |  37 ++--
 .../cache/transactions/IgniteTxManager.java   |   2 +-
 .../clock/GridClockSyncProcessor.java         |   2 +-
 .../continuous/GridContinuousProcessor.java   |   4 +-
 .../datastreamer/DataStreamProcessor.java     |   2 +-
 .../datastreamer/DataStreamerImpl.java        |   2 +-
 .../processors/igfs/IgfsDataManager.java      |   2 +-
 .../igfs/IgfsFragmentizerManager.java         |   4 +-
 .../processors/job/GridJobProcessor.java      |   8 +-
 .../handlers/task/GridTaskCommandHandler.java |   4 +-
 .../processors/task/GridTaskProcessor.java    |   6 +-
 .../jobstealing/JobStealingCollisionSpi.java  |   2 +-
 .../TestRecordingCommunicationSpi.java        |  29 +++
 ...CommunicationManagerListenersSelfTest.java |   2 +-
 .../GridCommunicationSendMessageSelfTest.java |   2 +-
 .../GridCachePartitionedGetSelfTest.java      |   2 +-
 .../IgniteBinaryMetadataUpdateFromInvoke.java | 187 ++++++++++++++++++
 .../communication/GridIoManagerBenchmark.java |   4 +-
 .../GridIoManagerBenchmark0.java              |  12 +-
 .../GridCacheMessageSelfTest.java             |   2 +-
 .../testframework/GridSpiTestContext.java     |   4 +-
 .../IgniteCacheRestartTestSuite2.java         |   2 +
 .../query/h2/opt/GridH2IndexBase.java         |   2 +-
 .../h2/twostep/GridMapQueryExecutor.java      |   2 +-
 .../h2/twostep/GridReduceQueryExecutor.java   |   2 +-
 30 files changed, 289 insertions(+), 61 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
index 9124cafa79a6b..40c2c636ace93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
@@ -449,7 +449,7 @@ private class CheckpointRequestListener implements GridMessageListener {
          * @param msg Received message.
          */
         @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridCheckpointRequest req = (GridCheckpointRequest)msg;
 
             if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 4b68e5bdd0da7..a4e32980114e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -321,7 +321,7 @@ public void resetMetrics() {
             log.debug(startInfo());
 
         addMessageListener(TOPIC_IO_TEST, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 ClusterNode node = ctx.discovery().node(nodeId);
 
                 if (node == null)
@@ -1508,7 +1508,7 @@ private void invokeListener(Byte plc, GridMessageListener lsnr, UUID nodeId, Obj
             CUR_PLC.set(plc);
 
         try {
-            lsnr.onMessage(nodeId, msg);
+            lsnr.onMessage(nodeId, msg, plc);
         }
         finally {
             if (change)
@@ -2333,14 +2333,14 @@ private static class ArrayListener implements GridMessageListener {
          * @param nodeId Node ID.
          * @param msg Message.
          */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridMessageListener[] arr0 = arr;
 
             if (arr0 == null)
                 return;
 
             for (GridMessageListener l : arr0)
-                l.onMessage(nodeId, msg);
+                l.onMessage(nodeId, msg, plc);
         }
 
         /**
@@ -2440,7 +2440,7 @@ private class GridUserMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
         @SuppressWarnings({"SynchronizationOnLocalVariableOrMethodParameter", "ConstantConditions",
             "OverlyStrongTypeCast"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (!(msg instanceof GridIoUserMessage)) {
                 U.error(log, "Received unknown message (potentially fatal problem): " + msg);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
index 39935917c3e65..c7de57c959c18 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
@@ -30,6 +30,7 @@ public interface GridMessageListener extends EventListener {
      * @param nodeId ID of node that sent the message. Note that may have already
      *      left topology by the time this message is received.
      * @param msg Message received.
+     * @param plc Message policy (pool).
      */
-    public void onMessage(UUID nodeId, Object msg);
+    public void onMessage(UUID nodeId, Object msg, byte plc);
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
index a571ae445438f..661bf68b61a42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
@@ -82,7 +82,7 @@ class GridDeploymentCommunication {
         this.log = log.getLogger(getClass());
 
         peerLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 processDeploymentRequest(nodeId, msg);
             }
         };
@@ -416,7 +416,7 @@ GridDeploymentResponse sendResourceRequest(final String rsrcName, IgniteUuid cls
         };
 
         GridMessageListener resLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert nodeId != null;
                 assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
index 607bb9688aaf1..b77ec27496c50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
@@ -877,7 +877,7 @@ private <T extends Event> List<T> query(IgnitePredicate<T> p, Collection<? exten
 
         GridMessageListener resLsnr = new GridMessageListener() {
             @SuppressWarnings("deprecation")
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert nodeId != null;
                 assert msg != null;
 
@@ -1054,7 +1054,7 @@ private boolean[] copy(boolean[] arr) {
      */
     private class RequestListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 924ce7917b4eb..39087b628beb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -127,7 +128,7 @@ public class GridCacheIoManager extends GridCacheSharedManagerAdapter {
 
     /** Message listener. */
     private GridMessageListener lsnr = new GridMessageListener() {
-        @Override public void onMessage(final UUID nodeId, final Object msg) {
+        @Override public void onMessage(final UUID nodeId, final Object msg, final byte plc) {
             if (log.isDebugEnabled())
                 log.debug("Received unordered cache communication message [nodeId=" + nodeId +
                     ", locId=" + cctx.localNodeId() + ", msg=" + msg + ']');
@@ -206,6 +207,9 @@ else if (cacheDesc.receivedFromStartVersion() != null)
                         log.debug(msg0.toString());
                     }
 
+                    if (plc == GridIoPolicy.UTILITY_CACHE_POOL)
+                        rmtAffVer = new AffinityTopologyVersion(cctx.localNode().order());
+
                     fut = cctx.exchange().affinityReadyFuture(rmtAffVer);
                 }
             }
@@ -213,22 +217,27 @@ else if (cacheDesc.receivedFromStartVersion() != null)
             if (fut != null && !fut.isDone()) {
                 fut.listen(new CI1<IgniteInternalFuture<?>>() {
                     @Override public void apply(IgniteInternalFuture<?> t) {
-                        cctx.kernalContext().closure().runLocalSafe(new Runnable() {
-                            @Override public void run() {
-                                IgniteLogger log = cacheMsg.messageLogger(cctx);
+                        try {
+                            cctx.kernalContext().pools().poolForPolicy(plc).execute(new Runnable() {
+                                @Override public void run() {
+                                    IgniteLogger log = cacheMsg.messageLogger(cctx);
 
-                                if (log.isDebugEnabled()) {
-                                    StringBuilder msg0 = new StringBuilder("Process cache message after wait for " +
-                                        "affinity topology version [");
+                                    if (log.isDebugEnabled()) {
+                                        StringBuilder msg0 = new StringBuilder("Process cache message after wait for " +
+                                            "affinity topology version [");
 
-                                    appendMessageInfo(cacheMsg, nodeId, msg0).append(']');
+                                        appendMessageInfo(cacheMsg, nodeId, msg0).append(']');
 
-                                    log.debug(msg0.toString());
-                                }
+                                        log.debug(msg0.toString());
+                                    }
 
-                                handleMessage(nodeId, cacheMsg);
-                            }
-                        });
+                                    handleMessage(nodeId, cacheMsg);
+                                }
+                            });
+                        }
+                        catch (IgniteCheckedException e) {
+                            U.error(cacheMsg.messageLogger(cctx), "Failed to get pool for policy: " + plc, e);
+                        }
                     }
                 });
 
@@ -1336,7 +1345,7 @@ private class OrderedMessageListener implements GridMessageListener {
 
         /** {@inheritDoc} */
         @SuppressWarnings({"CatchGenericClass", "unchecked"})
-        @Override public void onMessage(final UUID nodeId, Object msg) {
+        @Override public void onMessage(final UUID nodeId, Object msg, byte plc) {
             if (log.isDebugEnabled())
                 log.debug("Received cache ordered message [nodeId=" + nodeId + ", msg=" + msg + ']');
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 2c02f967d00be..2f21614352e8c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -2472,7 +2472,7 @@ private CommitListener(IgniteInternalTx tx) {
     private class DeadlockDetectionListener implements GridMessageListener {
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridCacheMessage cacheMsg = (GridCacheMessage)msg;
 
             unmarshall(nodeId, cacheMsg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
index 07643164368ce..3586956da99dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
@@ -96,7 +96,7 @@ public GridClockSyncProcessor(GridKernalContext ctx) {
         srv.start(ctx);
 
         ctx.io().addMessageListener(TOPIC_TIME_SYNC, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert msg instanceof GridClockDeltaSnapshotMessage;
 
                 GridClockDeltaSnapshotMessage msg0 = (GridClockDeltaSnapshotMessage)msg;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 9fd9b6d44b89e..f0429bc3a3c91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -288,7 +288,7 @@ public GridContinuousProcessor(GridKernalContext ctx) {
             });
 
         ctx.io().addMessageListener(TOPIC_CONTINUOUS, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object obj) {
+            @Override public void onMessage(UUID nodeId, Object obj, byte plc) {
                 GridContinuousMessage msg = (GridContinuousMessage)obj;
 
                 if (msg.data() == null && msg.dataBytes() != null) {
@@ -721,7 +721,7 @@ public IgniteInternalFuture<UUID> startRoutine(GridContinuousHandler hnd,
     private void registerMessageListener(GridContinuousHandler hnd) {
         if (hnd.orderedTopic() != null) {
             ctx.io().addMessageListener(hnd.orderedTopic(), new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object obj) {
+                @Override public void onMessage(UUID nodeId, Object obj, byte plc) {
                     GridContinuousMessage msg = (GridContinuousMessage)obj;
 
                     // Only notification can be ordered.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index fee4dd6616b28..6f35a52d5b673 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -80,7 +80,7 @@ public DataStreamProcessor(GridKernalContext ctx) {
 
         if (!ctx.clientNode()) {
             ctx.io().addMessageListener(TOPIC_DATASTREAM, new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     assert msg instanceof DataStreamerRequest;
 
                     processRequest(nodeId, (DataStreamerRequest)msg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index bb9ffdd5795a2..515314eb08728 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -321,7 +321,7 @@ public DataStreamerImpl(
         topic = TOPIC_DATASTREAM.topic(IgniteUuid.fromUuid(ctx.localNodeId()));
 
         ctx.io().addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert msg instanceof DataStreamerResponse;
 
                 DataStreamerResponse res = (DataStreamerResponse)msg;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index e534800efd7a6..59e1b724764e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -153,7 +153,7 @@ void awaitInit() {
         topic = F.isEmpty(igfsName) ? TOPIC_IGFS : TOPIC_IGFS.topic(igfsName);
 
         igfsCtx.kernalContext().io().addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (msg instanceof IgfsBlocksMessage)
                     processBlocksMessage(nodeId, (IgfsBlocksMessage)msg);
                 else if (msg instanceof IgfsAckMessage)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
index 2e82f33024b57..f76b877f38443 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
@@ -453,7 +453,7 @@ protected FragmentizerCoordinator() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof IgfsFragmentizerResponse) {
                 IgfsFragmentizerResponse res = (IgfsFragmentizerResponse)msg;
 
@@ -673,7 +673,7 @@ protected FragmentizerWorker() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof IgfsFragmentizerRequest ||
                 msg instanceof IgfsSyncMessage) {
                 if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index ea9cbd7dbd272..eb3300ce3412c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -444,7 +444,7 @@ public Collection<ComputeJobSibling> requestJobSiblings(
         final Condition cond = lock.newCondition();
 
         GridMessageListener msgLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 String err = null;
                 GridJobSiblingsResponse res = null;
 
@@ -1842,7 +1842,7 @@ private class JobHoldListener implements GridJobHoldListener {
      */
     private class JobSessionListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
@@ -1858,7 +1858,7 @@ private class JobSessionListener implements GridMessageListener {
      */
     private class JobCancelListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
@@ -1876,7 +1876,7 @@ private class JobCancelListener implements GridMessageListener {
      */
     private class JobExecutionListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
index 947435cb58d83..88f8d4de9241e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
@@ -109,7 +109,7 @@ public GridTaskCommandHandler(final GridKernalContext ctx) {
         super(ctx);
 
         ctx.io().addMessageListener(TOPIC_REST, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!(msg instanceof GridTaskResultRequest)) {
                     U.warn(log, "Received unexpected message instead of task result request: " + msg);
 
@@ -425,7 +425,7 @@ private IgniteBiTuple<String, GridTaskResultResponse> requestTaskResult(final UU
         final Condition cond = lock.newCondition();
 
         GridMessageListener msgLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 String err = null;
                 GridTaskResultResponse res = null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 12213581b3286..a807ad9efe5dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -1225,7 +1225,7 @@ private JobMessageListener(boolean jobResOnly) {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof GridJobExecuteResponse)
                 processJobExecuteResponse(nodeId, (GridJobExecuteResponse)msg);
             else if (jobResOnly)
@@ -1269,7 +1269,7 @@ private class TaskDiscoveryListener implements GridLocalEventListener {
      */
     private class JobSiblingsMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (!(msg instanceof GridJobSiblingsRequest)) {
                 U.warn(log, "Received unexpected message instead of siblings request: " + msg);
 
@@ -1341,7 +1341,7 @@ private class JobSiblingsMessageListener implements GridMessageListener {
      */
     private class TaskCancelMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert msg != null;
 
             if (!(msg instanceof GridTaskCancelRequest)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
index f778bfcc73ae8..fca249893980a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
@@ -547,7 +547,7 @@ public void setStealingAttributes(Map<String, ? extends Serializable> stealAttrs
 
         spiCtx.addMessageListener(
             msgLsnr = new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     MessageInfo info = rcvMsgMap.get(nodeId);
 
                     if (info == null) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
index 2aed459fedc59..17ca1a7ffc7c4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
@@ -88,6 +88,8 @@ public class TestRecordingCommunicationSpi extends TcpCommunicationSpi {
 
                     blockedMsgs.add(new T2<>(node, ioMsg));
 
+                    this.notifyAll();
+
                     return;
                 }
             }
@@ -136,6 +138,33 @@ public boolean hasBlockedMessages() {
         }
     }
 
+    /**
+     * @param cls Message class.
+     * @param nodeName Node name.
+     * @throws InterruptedException If interrupted.
+     */
+    public void waitForBlocked(Class<?> cls, String nodeName) throws InterruptedException {
+        synchronized (this) {
+            while (!hasMessage(cls, nodeName))
+                wait();
+        }
+    }
+
+    /**
+     * @param cls Message class.
+     * @param nodeName Node name.
+     * @return {@code True} if has blocked message.
+     */
+    private boolean hasMessage(Class<?> cls, String nodeName) {
+        for (T2<ClusterNode, GridIoMessage> msg : blockedMsgs) {
+            if (msg.get2().message().getClass() == cls &&
+                nodeName.equals(msg.get1().attribute(ATTR_GRID_NAME)))
+                return true;
+        }
+
+        return false;
+    }
+
     /**
      * @param blockP Message block predicate.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
index 7613543335ce5..9289f866c8fb5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
@@ -159,7 +159,7 @@ private static class MessageListeningTask extends ComputeTaskSplitAdapter<Object
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             // No-op.
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
index 8503b48f5ec48..08ee3475e9f9a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
@@ -121,7 +121,7 @@ private void doSend() throws Exception {
         final CountDownLatch latch = new CountDownLatch(SAMPLE_CNT);
 
         mgr1.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 latch.countDown();
             }
         });
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
index 2cb4a005676b7..47d6092f2b2c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
@@ -226,7 +226,7 @@ private void prepare() throws Exception {
                 ((IgniteKernal)g).context().io().addMessageListener(
                     TOPIC_CACHE,
                     new GridMessageListener() {
-                        @Override public void onMessage(UUID nodeId, Object msg) {
+                        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                             info("Received message from node [nodeId=" + nodeId + ", msg=" + msg + ']');
 
                             if (msg instanceof GridNearSingleGetRequest) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java
new file mode 100644
index 0000000000000..f919ca736115c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class IgniteBinaryMetadataUpdateFromInvoke extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setMarshaller(null);
+
+        cfg.setClientMode(client);
+
+        CacheConfiguration ccfg = new CacheConfiguration("cache");
+
+        ccfg.setAtomicityMode(ATOMIC);
+        ccfg.setBackups(1);
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        cfg.setConsistentId(gridName);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMetadataUpdateFromAtomicInvoke() throws Exception {
+        for (int i = 0; i < 2; i++) {
+            client = false;
+
+            log.info("Iteration: " + i);
+
+            final int SRVS = 4;
+
+            startGrids(SRVS);
+
+            awaitPartitionMapExchange();
+
+            final CyclicBarrier b = new CyclicBarrier(3);
+
+            final AtomicBoolean stop = new AtomicBoolean();
+
+            final int META_PRIMARY_NODE = 0;
+            final int META_UPDATE_FROM_NODE = 1;
+
+            BinaryMarshaller marsh = (BinaryMarshaller)ignite(META_PRIMARY_NODE).configuration().getMarshaller();
+
+            marsh.getContext().registerClass(
+                marsh.binaryMarshaller().context().typeId(TestEnum1.class.getName()), TestEnum1.class);
+
+            TestRecordingCommunicationSpi testSpi =
+                (TestRecordingCommunicationSpi)ignite(META_UPDATE_FROM_NODE).configuration().getCommunicationSpi();
+
+            testSpi.blockMessages(GridNearTxPrepareRequest.class, getTestGridName(META_PRIMARY_NODE));
+
+            IgniteInternalFuture<?> sFut1 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    IgniteCache<Object, Object> cache = ignite(META_UPDATE_FROM_NODE).cache("cache");
+
+                    List<Integer> keys = primaryKeys(cache, 1);
+
+                    b.await();
+
+                    cache.invoke(keys.get(0), new TestEntryProcessor());
+
+                    return null;
+                }
+            }, "async-node-0");
+
+            IgniteInternalFuture<?> sFut2 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    IgniteCache<Object, Object> cache = ignite(META_PRIMARY_NODE).cache("cache");
+
+                    List<Integer> keys = primaryKeys(cache, 1);
+
+                    b.await();
+
+                    Thread.sleep(2000);
+
+                    cache.invoke(keys.get(0), new TestEntryProcessor());
+
+                    return null;
+                }
+            }, "async-node-1");
+
+            IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    client = true;
+
+                    b.await();
+
+                    for (int i = 0; i < 1; i++)
+                        startGrid(SRVS + i);
+
+                    stop.set(true);
+
+                    return null;
+                }
+            });
+
+            testSpi.waitForBlocked(GridNearTxPrepareRequest.class, getTestGridName(META_PRIMARY_NODE));
+
+            U.sleep(5000);
+
+            testSpi.stopBlock();
+
+            sFut1.get();
+            sFut2.get();
+            fut2.get();
+
+            stopAllGrids();
+        }
+    }
+
+    /**
+     *
+     */
+    static class TestEntryProcessor implements CacheEntryProcessor<Object, Object, Object> {
+        @Override public Object process(MutableEntry<Object, Object> e, Object... args) {
+            e.setValue(TestEnum1.ENUM);
+
+            return null;
+        }
+    }
+
+    /**
+     *
+     */
+    enum TestEnum1 {
+        /** */
+        ENUM
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
index 723495cfa89c2..293e5789ddf3c 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
@@ -240,7 +240,7 @@ private static void receiveMessages(final IgniteKernal g) {
         GridMessageListener lsnr = new GridMessageListener() {
             private ClusterNode node;
 
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (node == null)
                     node = g.context().discovery().node(nodeId);
 
@@ -336,7 +336,7 @@ private ClusterNode awaitOther(final GridDiscoveryManager disc) throws Interrupt
      */
     private static class SenderMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             msgCntr.increment();
 
             if (testLatency)
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
index f2c62559397b4..83efd7de42098 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
@@ -130,7 +130,7 @@ public void testThroughput() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -141,7 +141,7 @@ public void testThroughput() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 msgCntr.increment();
 
                 sem.release();
@@ -224,7 +224,7 @@ public void testLatency() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -235,7 +235,7 @@ public void testLatency() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 map.get(((GridTestMessage)msg).id()).countDown();
             }
         });
@@ -324,7 +324,7 @@ public void testVariableLoad() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -335,7 +335,7 @@ public void testVariableLoad() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 msgCntr.increment();
 
                 sem.release();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
index 9c975422a2601..2d04db2fedab9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
@@ -139,7 +139,7 @@ private void doSend() throws Exception {
         final CountDownLatch latch = new CountDownLatch(SAMPLE_CNT);
 
         mgr1.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 try {
                     latch.countDown();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index 1c8acbc3960ce..0c04039753beb 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -328,7 +328,7 @@ public Serializable removeSentMessage(ClusterNode node) {
     @SuppressWarnings("deprecation")
     public void triggerMessage(ClusterNode node, Object msg) {
         for (GridMessageListener lsnr : msgLsnrs)
-            lsnr.onMessage(node.id(), msg);
+            lsnr.onMessage(node.id(), msg, (byte)0);
     }
 
     /** {@inheritDoc} */
@@ -667,7 +667,7 @@ private class GridLocalMessageListener implements GridMessageListener {
         @SuppressWarnings({
             "SynchronizationOnLocalVariableOrMethodParameter", "ConstantConditions",
             "OverlyStrongTypeCast"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridIoUserMessage ioMsg = (GridIoUserMessage)msg;
 
             ClusterNode node = locNode;
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
index 05137868443c8..fb38c5535b35b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.GridCachePutAllFailoverSelfTest;
+import org.apache.ignite.internal.processors.cache.IgniteBinaryMetadataUpdateFromInvoke;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicPutAllFailoverSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCachePutAllRestartTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteBinaryMetadataUpdateNodeRestartTest;
@@ -45,6 +46,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCachePutAllFailoverSelfTest.class);
 
         suite.addTestSuite(IgniteBinaryMetadataUpdateNodeRestartTest.class);
+        suite.addTestSuite(IgniteBinaryMetadataUpdateFromInvoke.class);
 
         suite.addTestSuite(IgniteCacheGetRestartTest.class);
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index c29239f143719..22b94c7ac796b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -138,7 +138,7 @@ protected final void initDistributedJoinMessaging(GridH2Table tbl) {
             msgTopic = new IgniteBiTuple<>(GridTopic.TOPIC_QUERY, tbl.identifier() + '.' + getName());
 
             msgLsnr = new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     GridSpinBusyLock l = desc.indexing().busyLock();
 
                     if (!l.enterBusy())
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index ac1a6a60c0200..0605287ded597 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -167,7 +167,7 @@ public void start(final GridKernalContext ctx, IgniteH2Indexing h2) throws Ignit
         }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
 
         ctx.io().addMessageListener(GridTopic.TOPIC_QUERY, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!busyLock.enterBusy())
                     return;
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 3f886ee686d66..71c0e1245b625 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -190,7 +190,7 @@ public void start(final GridKernalContext ctx, final IgniteH2Indexing h2) throws
         log = ctx.log(GridReduceQueryExecutor.class);
 
         ctx.io().addMessageListener(GridTopic.TOPIC_QUERY, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!busyLock.enterBusy())
                     return;
 

From 90e96db22dbb7a341bfe1a8130b6dc16d5d8ae81 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Tue, 27 Jun 2017 14:43:06 +0300
Subject: [PATCH 320/446] Compilation is fixed.

---
 .../java/org/apache/ignite/IgniteSystemProperties.java    | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index f473c51e74a93..e9bbf5a10df4c 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -320,6 +320,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_H2_DEBUG_CONSOLE = "IGNITE_H2_DEBUG_CONSOLE";
 
+    /**
+     * This property allows to specify user defined port which H2 indexing SPI will use
+     * to start H2 debug console on. If this property is not set or set to 0, H2 debug
+     * console will use system-provided dynamic port.
+     * This property is only relevant when {@link #IGNITE_H2_DEBUG_CONSOLE} property is set.
+     */
+    public static final String IGNITE_H2_DEBUG_CONSOLE_PORT = "IGNITE_H2_DEBUG_CONSOLE_PORT";
+
     /**
      * If this property is set to {@code true} then shared memory space native debug will be enabled.
      */

From 3f33a902ed0d0a3e27be548209fe8e7933da57a9 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 26 Jun 2017 12:27:58 +0300
Subject: [PATCH 321/446] Minor fix for GG-12197 "Ignore events for discarded
 update in CLOCK mode".

---
 .../processors/cache/GridCacheMapEntry.java       | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 6ae2a7cc3a77a..bfa20d9bed139 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -2280,13 +2280,14 @@ else if (ttl != CU.TTL_ZERO)
                             if (updateCntr != null)
                                 updateCntr0 = updateCntr;
 
-                            cctx.continuousQueries().skipUpdateEvent(
-                                lsnrs,
-                                key,
-                                partition(),
-                                updateCntr0,
-                                primary,
-                                topVer);
+                            if (lsnrs != null)
+                                cctx.continuousQueries().skipUpdateEvent(
+                                    lsnrs,
+                                    key,
+                                    partition(),
+                                    updateCntr0,
+                                    primary,
+                                    topVer);
                         }
 
                         return new GridCacheUpdateAtomicResult(false,

From 3668b91817c1da7958cc3d7c4dddf890a2237772 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 27 Jun 2017 10:34:42 +0300
Subject: [PATCH 322/446] Partially reverted GG-12352.

---
 .../service/GridServiceProcessor.java         |  2 +-
 .../spi/IgniteSpiOperationTimeoutHelper.java  |  4 +--
 .../tcp/TcpCommunicationSpi.java              |  2 --
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 16 ++----------
 .../IgniteClientReconnectAbstractTest.java    |  5 ++++
 ...teBinaryMetadataUpdateNodeRestartTest.java | 10 +++++++
 ...gniteCacheNearRestartRollbackSelfTest.java | 15 +++++++++++
 ...SynchronizationModesMultithreadedTest.java |  5 ++++
 .../apache/ignite/spi/GridTcpForwarder.java   | 26 +++++++++++++++++++
 .../tcp/TcpCommunicationSpiDropNodesTest.java |  1 -
 .../TcpCommunicationSpiFaultyClientTest.java  |  5 ----
 .../discovery/tcp/TcpDiscoverySelfTest.java   |  1 +
 12 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 4c9f8e800465a..5c7a299b2bd4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1659,7 +1659,7 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                                             log.info("Service processor detected a topology change during " +
                                                 "assignments calculation (will abort current iteration and " +
                                                 "re-calculate on the newer version): " +
-                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer + ']');
+                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer0 + ']');
 
                                         return;
                                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
index 1d9fa94f44475..33896361e2c21 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
@@ -96,9 +96,9 @@ public boolean checkFailureTimeoutReached(Exception e) {
         if (!failureDetectionTimeoutEnabled)
             return false;
 
-        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class))
+        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class, SocketException.class))
             return true;
 
-        return X.hasCause(e, SocketException.class) && (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
+        return (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 269a98525cd42..10d2141ffdf15 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -23,7 +23,6 @@
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -2918,7 +2917,6 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
-                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 80a0f18359cdf..410a351586768 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1147,8 +1147,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
             boolean openSock = false;
 
-            boolean wasHandshake = false;
-
             Socket sock = null;
 
             try {
@@ -1166,8 +1164,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(
                     ackTimeout0));
 
-                wasHandshake = true;
-
                 if (msg instanceof TcpDiscoveryJoinRequestMessage) {
                     boolean ignore = false;
 
@@ -1247,7 +1243,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
                 errs.add(e);
 
-                if (X.hasCause(e, SSLException.class)) {
+                if (X.hasCause(e, SSLException.class, StreamCorruptedException.class)) {
                     if (--sslConnectAttempts == 0)
                         throw new IgniteException("Unable to establish secure connection. " +
                             "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
@@ -1255,21 +1251,13 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                     continue;
                 }
 
-                if (X.hasCause(e, StreamCorruptedException.class)) {
-                    if (--sslConnectAttempts == 0)
-                        throw new IgniteException("Unable to establish plain connection. " +
-                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
-
-                    continue;
-                }
-
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
                 if (!spi.failureDetectionTimeoutEnabled() && ++reconCnt == spi.getReconnectCount())
                     break;
 
-                if (!openSock || !wasHandshake) {
+                if (!openSock) {
                     // Reconnect for the second time, if connection is not established.
                     if (connectAttempts < 2) {
                         connectAttempts++;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
index 4d49366e1ad23..a793760f44fbf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
@@ -69,6 +69,9 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
     /** */
     private static final long RECONNECT_TIMEOUT = 10_000;
 
+    /** Reconnect should occurs before failure detection time is out. */
+    public static final long FAILURE_DETECTION_TIMEOUT = RECONNECT_TIMEOUT +2000;
+
     /** */
     protected boolean clientMode;
 
@@ -76,6 +79,8 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
+        cfg.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT);
+
         TestTcpDiscoverySpi disco = new TestTcpDiscoverySpi();
 
         disco.setIpFinder(ipFinder);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
index 814fb08eacba5..3d552847fd71d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheEntryProcessor;
 import org.apache.ignite.cluster.ClusterTopologyException;
@@ -88,10 +89,19 @@ public class IgniteBinaryMetadataUpdateNodeRestartTest extends GridCommonAbstrac
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         super.afterTestsStopped();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index a48497d8f39c4..3f242b5726867 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -31,6 +31,7 @@
 import javax.cache.processor.MutableEntry;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -92,6 +93,20 @@ public class IgniteCacheNearRestartRollbackSelfTest extends GridCommonAbstractTe
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+    }
+
     /**
      * @param gridName Grid name.
      * @return Cache configuration.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
index 08396daa5efeb..c214a772d422c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cache.store.CacheStoreAdapter;
@@ -104,6 +105,8 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+
         startGrids(SRVS);
 
         clientMode = true;
@@ -119,6 +122,8 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         super.afterTestsStopped();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
index d08321e18e8b5..68d97c1a3932b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
@@ -23,6 +23,7 @@
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -85,6 +86,10 @@ public GridTcpForwarder(
                                 outputCon.getInputStream(), inputCon.getOutputStream()
                             );
 
+                            //Force closing sibling if one of thread failed.
+                            forwardThread1.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread2));
+                            forwardThread2.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread1));
+
                             forwardThread1.start();
                             forwardThread2.start();
 
@@ -127,6 +132,25 @@ public GridTcpForwarder(
         U.join(mainThread, log);
     }
 
+    /**
+     *
+     */
+    private static class ForwarderExceptionHandler implements Thread.UncaughtExceptionHandler {
+        /** */
+        private Thread siblingThread;
+
+        /** */
+        public ForwarderExceptionHandler(Thread siblingThread) {
+
+            this.siblingThread = siblingThread;
+        }
+
+        /** */
+        @Override public void uncaughtException(Thread t, Throwable e) {
+            siblingThread.interrupt();
+        }
+    }
+
     /**
      * Thread reads data from input stream and write to output stream.
      */
@@ -166,6 +190,8 @@ private ForwardThread(String name, InputStream inputStream, OutputStream outputS
             }
             catch (IOException e) {
                 log.error("IOException while forwarding data [threadName=" + getName() + "]", e);
+
+                throw new IgniteException(e);
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
index 71a758024b6a8..3315c17046af5 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
@@ -42,7 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.config.GridTestProperties;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
index c18a89f04e041..b0e543df495c1 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
@@ -18,16 +18,13 @@
 package org.apache.ignite.spi.communication.tcp;
 
 import java.io.IOException;
-import java.io.OutputStream;
 import java.net.InetAddress;
 import java.net.ServerSocket;
-import java.net.Socket;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
@@ -45,8 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
index 043208c955b44..cd1776ca2bc1c 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
@@ -219,6 +219,7 @@ else if (gridName.contains("testNodeShutdownOnRingMessageWorkerFailureFailedNode
         return cfg;
     }
 
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         discoMap = null;

From 172f41c489c0ca5b7613163cd325b0c01d5b28b1 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 27 Jun 2017 12:55:11 +0300
Subject: [PATCH 323/446] gg-12133 Deadlock for metadata update from
 GridCacheMapEntry.innerUpdate

(cherry picked from commit 897f4c0)
---
 .../checkpoint/GridCheckpointManager.java     |   2 +-
 .../managers/communication/GridIoManager.java |  10 +-
 .../communication/GridMessageListener.java    |   3 +-
 .../GridDeploymentCommunication.java          |   4 +-
 .../eventstorage/GridEventStorageManager.java |   4 +-
 .../processors/cache/GridCacheIoManager.java  |  37 ++--
 .../cache/transactions/IgniteTxManager.java   |   2 +-
 .../clock/GridClockSyncProcessor.java         |   2 +-
 .../continuous/GridContinuousProcessor.java   |   4 +-
 .../datastreamer/DataStreamProcessor.java     |   2 +-
 .../datastreamer/DataStreamerImpl.java        |   2 +-
 .../processors/igfs/IgfsDataManager.java      |   2 +-
 .../igfs/IgfsFragmentizerManager.java         |   4 +-
 .../processors/job/GridJobProcessor.java      |   8 +-
 .../handlers/task/GridTaskCommandHandler.java |   4 +-
 .../processors/task/GridTaskProcessor.java    |   6 +-
 .../jobstealing/JobStealingCollisionSpi.java  |   2 +-
 .../TestRecordingCommunicationSpi.java        |  29 +++
 ...CommunicationManagerListenersSelfTest.java |   2 +-
 .../GridCommunicationSendMessageSelfTest.java |   2 +-
 .../GridCachePartitionedGetSelfTest.java      |   2 +-
 .../IgniteBinaryMetadataUpdateFromInvoke.java | 187 ++++++++++++++++++
 ...adlockDetectionMessageMarshallingTest.java |   2 +-
 .../communication/GridIoManagerBenchmark.java |   4 +-
 .../GridIoManagerBenchmark0.java              |  12 +-
 .../GridCacheMessageSelfTest.java             |   2 +-
 .../testframework/GridSpiTestContext.java     |   4 +-
 .../IgniteCacheRestartTestSuite2.java         |   2 +
 .../query/h2/opt/GridH2IndexBase.java         |   2 +-
 .../h2/twostep/GridMapQueryExecutor.java      |   2 +-
 .../h2/twostep/GridReduceQueryExecutor.java   |   2 +-
 31 files changed, 290 insertions(+), 62 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
index 9124cafa79a6b..40c2c636ace93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
@@ -449,7 +449,7 @@ private class CheckpointRequestListener implements GridMessageListener {
          * @param msg Received message.
          */
         @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridCheckpointRequest req = (GridCheckpointRequest)msg;
 
             if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 06dcb8664445c..2cb4c3eebe1a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -322,7 +322,7 @@ public void resetMetrics() {
             log.debug(startInfo());
 
         addMessageListener(TOPIC_IO_TEST, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 ClusterNode node = ctx.discovery().node(nodeId);
 
                 if (node == null)
@@ -1509,7 +1509,7 @@ private void invokeListener(Byte plc, GridMessageListener lsnr, UUID nodeId, Obj
             CUR_PLC.set(plc);
 
         try {
-            lsnr.onMessage(nodeId, msg);
+            lsnr.onMessage(nodeId, msg, plc);
         }
         finally {
             if (change)
@@ -2334,14 +2334,14 @@ private static class ArrayListener implements GridMessageListener {
          * @param nodeId Node ID.
          * @param msg Message.
          */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridMessageListener[] arr0 = arr;
 
             if (arr0 == null)
                 return;
 
             for (GridMessageListener l : arr0)
-                l.onMessage(nodeId, msg);
+                l.onMessage(nodeId, msg, plc);
         }
 
         /**
@@ -2441,7 +2441,7 @@ private class GridUserMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
         @SuppressWarnings({"SynchronizationOnLocalVariableOrMethodParameter", "ConstantConditions",
             "OverlyStrongTypeCast"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (!(msg instanceof GridIoUserMessage)) {
                 U.error(log, "Received unknown message (potentially fatal problem): " + msg);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
index 39935917c3e65..c7de57c959c18 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
@@ -30,6 +30,7 @@ public interface GridMessageListener extends EventListener {
      * @param nodeId ID of node that sent the message. Note that may have already
      *      left topology by the time this message is received.
      * @param msg Message received.
+     * @param plc Message policy (pool).
      */
-    public void onMessage(UUID nodeId, Object msg);
+    public void onMessage(UUID nodeId, Object msg, byte plc);
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
index a571ae445438f..661bf68b61a42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
@@ -82,7 +82,7 @@ class GridDeploymentCommunication {
         this.log = log.getLogger(getClass());
 
         peerLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 processDeploymentRequest(nodeId, msg);
             }
         };
@@ -416,7 +416,7 @@ GridDeploymentResponse sendResourceRequest(final String rsrcName, IgniteUuid cls
         };
 
         GridMessageListener resLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert nodeId != null;
                 assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
index 406da2d284b63..a80cfbe2555d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
@@ -977,7 +977,7 @@ private <T extends Event> List<T> query(IgnitePredicate<T> p, Collection<? exten
 
         GridMessageListener resLsnr = new GridMessageListener() {
             @SuppressWarnings("deprecation")
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert nodeId != null;
                 assert msg != null;
 
@@ -1154,7 +1154,7 @@ private boolean[] copy(boolean[] arr) {
      */
     private class RequestListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 924ce7917b4eb..39087b628beb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -127,7 +128,7 @@ public class GridCacheIoManager extends GridCacheSharedManagerAdapter {
 
     /** Message listener. */
     private GridMessageListener lsnr = new GridMessageListener() {
-        @Override public void onMessage(final UUID nodeId, final Object msg) {
+        @Override public void onMessage(final UUID nodeId, final Object msg, final byte plc) {
             if (log.isDebugEnabled())
                 log.debug("Received unordered cache communication message [nodeId=" + nodeId +
                     ", locId=" + cctx.localNodeId() + ", msg=" + msg + ']');
@@ -206,6 +207,9 @@ else if (cacheDesc.receivedFromStartVersion() != null)
                         log.debug(msg0.toString());
                     }
 
+                    if (plc == GridIoPolicy.UTILITY_CACHE_POOL)
+                        rmtAffVer = new AffinityTopologyVersion(cctx.localNode().order());
+
                     fut = cctx.exchange().affinityReadyFuture(rmtAffVer);
                 }
             }
@@ -213,22 +217,27 @@ else if (cacheDesc.receivedFromStartVersion() != null)
             if (fut != null && !fut.isDone()) {
                 fut.listen(new CI1<IgniteInternalFuture<?>>() {
                     @Override public void apply(IgniteInternalFuture<?> t) {
-                        cctx.kernalContext().closure().runLocalSafe(new Runnable() {
-                            @Override public void run() {
-                                IgniteLogger log = cacheMsg.messageLogger(cctx);
+                        try {
+                            cctx.kernalContext().pools().poolForPolicy(plc).execute(new Runnable() {
+                                @Override public void run() {
+                                    IgniteLogger log = cacheMsg.messageLogger(cctx);
 
-                                if (log.isDebugEnabled()) {
-                                    StringBuilder msg0 = new StringBuilder("Process cache message after wait for " +
-                                        "affinity topology version [");
+                                    if (log.isDebugEnabled()) {
+                                        StringBuilder msg0 = new StringBuilder("Process cache message after wait for " +
+                                            "affinity topology version [");
 
-                                    appendMessageInfo(cacheMsg, nodeId, msg0).append(']');
+                                        appendMessageInfo(cacheMsg, nodeId, msg0).append(']');
 
-                                    log.debug(msg0.toString());
-                                }
+                                        log.debug(msg0.toString());
+                                    }
 
-                                handleMessage(nodeId, cacheMsg);
-                            }
-                        });
+                                    handleMessage(nodeId, cacheMsg);
+                                }
+                            });
+                        }
+                        catch (IgniteCheckedException e) {
+                            U.error(cacheMsg.messageLogger(cctx), "Failed to get pool for policy: " + plc, e);
+                        }
                     }
                 });
 
@@ -1336,7 +1345,7 @@ private class OrderedMessageListener implements GridMessageListener {
 
         /** {@inheritDoc} */
         @SuppressWarnings({"CatchGenericClass", "unchecked"})
-        @Override public void onMessage(final UUID nodeId, Object msg) {
+        @Override public void onMessage(final UUID nodeId, Object msg, byte plc) {
             if (log.isDebugEnabled())
                 log.debug("Received cache ordered message [nodeId=" + nodeId + ", msg=" + msg + ']');
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 1ba0102408eb0..8b029465fbf30 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -2475,7 +2475,7 @@ private CommitListener(IgniteInternalTx tx) {
     private class DeadlockDetectionListener implements GridMessageListener {
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridCacheMessage cacheMsg = (GridCacheMessage)msg;
 
             Throwable err = null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
index 07643164368ce..3586956da99dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
@@ -96,7 +96,7 @@ public GridClockSyncProcessor(GridKernalContext ctx) {
         srv.start(ctx);
 
         ctx.io().addMessageListener(TOPIC_TIME_SYNC, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert msg instanceof GridClockDeltaSnapshotMessage;
 
                 GridClockDeltaSnapshotMessage msg0 = (GridClockDeltaSnapshotMessage)msg;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 4717e8724d038..c4989a218df34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -288,7 +288,7 @@ public GridContinuousProcessor(GridKernalContext ctx) {
             });
 
         ctx.io().addMessageListener(TOPIC_CONTINUOUS, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object obj) {
+            @Override public void onMessage(UUID nodeId, Object obj, byte plc) {
                 GridContinuousMessage msg = (GridContinuousMessage)obj;
 
                 if (msg.data() == null && msg.dataBytes() != null) {
@@ -722,7 +722,7 @@ public IgniteInternalFuture<UUID> startRoutine(GridContinuousHandler hnd,
     private void registerMessageListener(GridContinuousHandler hnd) {
         if (hnd.orderedTopic() != null) {
             ctx.io().addMessageListener(hnd.orderedTopic(), new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object obj) {
+                @Override public void onMessage(UUID nodeId, Object obj, byte plc) {
                     GridContinuousMessage msg = (GridContinuousMessage)obj;
 
                     // Only notification can be ordered.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index fee4dd6616b28..6f35a52d5b673 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -80,7 +80,7 @@ public DataStreamProcessor(GridKernalContext ctx) {
 
         if (!ctx.clientNode()) {
             ctx.io().addMessageListener(TOPIC_DATASTREAM, new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     assert msg instanceof DataStreamerRequest;
 
                     processRequest(nodeId, (DataStreamerRequest)msg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index bb9ffdd5795a2..515314eb08728 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -321,7 +321,7 @@ public DataStreamerImpl(
         topic = TOPIC_DATASTREAM.topic(IgniteUuid.fromUuid(ctx.localNodeId()));
 
         ctx.io().addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert msg instanceof DataStreamerResponse;
 
                 DataStreamerResponse res = (DataStreamerResponse)msg;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index 4490a6854d79b..21a0b7f2037d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -155,7 +155,7 @@ void awaitInit() {
         topic = F.isEmpty(igfsName) ? TOPIC_IGFS : TOPIC_IGFS.topic(igfsName);
 
         igfsCtx.kernalContext().io().addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (msg instanceof IgfsBlocksMessage)
                     processBlocksMessage(nodeId, (IgfsBlocksMessage)msg);
                 else if (msg instanceof IgfsAckMessage)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
index 2e82f33024b57..f76b877f38443 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
@@ -453,7 +453,7 @@ protected FragmentizerCoordinator() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof IgfsFragmentizerResponse) {
                 IgfsFragmentizerResponse res = (IgfsFragmentizerResponse)msg;
 
@@ -673,7 +673,7 @@ protected FragmentizerWorker() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof IgfsFragmentizerRequest ||
                 msg instanceof IgfsSyncMessage) {
                 if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index ea9cbd7dbd272..eb3300ce3412c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -444,7 +444,7 @@ public Collection<ComputeJobSibling> requestJobSiblings(
         final Condition cond = lock.newCondition();
 
         GridMessageListener msgLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 String err = null;
                 GridJobSiblingsResponse res = null;
 
@@ -1842,7 +1842,7 @@ private class JobHoldListener implements GridJobHoldListener {
      */
     private class JobSessionListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
@@ -1858,7 +1858,7 @@ private class JobSessionListener implements GridMessageListener {
      */
     private class JobCancelListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
@@ -1876,7 +1876,7 @@ private class JobCancelListener implements GridMessageListener {
      */
     private class JobExecutionListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
index 947435cb58d83..88f8d4de9241e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
@@ -109,7 +109,7 @@ public GridTaskCommandHandler(final GridKernalContext ctx) {
         super(ctx);
 
         ctx.io().addMessageListener(TOPIC_REST, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!(msg instanceof GridTaskResultRequest)) {
                     U.warn(log, "Received unexpected message instead of task result request: " + msg);
 
@@ -425,7 +425,7 @@ private IgniteBiTuple<String, GridTaskResultResponse> requestTaskResult(final UU
         final Condition cond = lock.newCondition();
 
         GridMessageListener msgLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 String err = null;
                 GridTaskResultResponse res = null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 12213581b3286..a807ad9efe5dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -1225,7 +1225,7 @@ private JobMessageListener(boolean jobResOnly) {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof GridJobExecuteResponse)
                 processJobExecuteResponse(nodeId, (GridJobExecuteResponse)msg);
             else if (jobResOnly)
@@ -1269,7 +1269,7 @@ private class TaskDiscoveryListener implements GridLocalEventListener {
      */
     private class JobSiblingsMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (!(msg instanceof GridJobSiblingsRequest)) {
                 U.warn(log, "Received unexpected message instead of siblings request: " + msg);
 
@@ -1341,7 +1341,7 @@ private class JobSiblingsMessageListener implements GridMessageListener {
      */
     private class TaskCancelMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert msg != null;
 
             if (!(msg instanceof GridTaskCancelRequest)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
index f778bfcc73ae8..fca249893980a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
@@ -547,7 +547,7 @@ public void setStealingAttributes(Map<String, ? extends Serializable> stealAttrs
 
         spiCtx.addMessageListener(
             msgLsnr = new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     MessageInfo info = rcvMsgMap.get(nodeId);
 
                     if (info == null) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
index 2aed459fedc59..17ca1a7ffc7c4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
@@ -88,6 +88,8 @@ public class TestRecordingCommunicationSpi extends TcpCommunicationSpi {
 
                     blockedMsgs.add(new T2<>(node, ioMsg));
 
+                    this.notifyAll();
+
                     return;
                 }
             }
@@ -136,6 +138,33 @@ public boolean hasBlockedMessages() {
         }
     }
 
+    /**
+     * @param cls Message class.
+     * @param nodeName Node name.
+     * @throws InterruptedException If interrupted.
+     */
+    public void waitForBlocked(Class<?> cls, String nodeName) throws InterruptedException {
+        synchronized (this) {
+            while (!hasMessage(cls, nodeName))
+                wait();
+        }
+    }
+
+    /**
+     * @param cls Message class.
+     * @param nodeName Node name.
+     * @return {@code True} if has blocked message.
+     */
+    private boolean hasMessage(Class<?> cls, String nodeName) {
+        for (T2<ClusterNode, GridIoMessage> msg : blockedMsgs) {
+            if (msg.get2().message().getClass() == cls &&
+                nodeName.equals(msg.get1().attribute(ATTR_GRID_NAME)))
+                return true;
+        }
+
+        return false;
+    }
+
     /**
      * @param blockP Message block predicate.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
index 7613543335ce5..9289f866c8fb5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
@@ -159,7 +159,7 @@ private static class MessageListeningTask extends ComputeTaskSplitAdapter<Object
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             // No-op.
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
index 8503b48f5ec48..08ee3475e9f9a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
@@ -121,7 +121,7 @@ private void doSend() throws Exception {
         final CountDownLatch latch = new CountDownLatch(SAMPLE_CNT);
 
         mgr1.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 latch.countDown();
             }
         });
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
index 2cb4a005676b7..47d6092f2b2c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
@@ -226,7 +226,7 @@ private void prepare() throws Exception {
                 ((IgniteKernal)g).context().io().addMessageListener(
                     TOPIC_CACHE,
                     new GridMessageListener() {
-                        @Override public void onMessage(UUID nodeId, Object msg) {
+                        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                             info("Received message from node [nodeId=" + nodeId + ", msg=" + msg + ']');
 
                             if (msg instanceof GridNearSingleGetRequest) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java
new file mode 100644
index 0000000000000..f919ca736115c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class IgniteBinaryMetadataUpdateFromInvoke extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setMarshaller(null);
+
+        cfg.setClientMode(client);
+
+        CacheConfiguration ccfg = new CacheConfiguration("cache");
+
+        ccfg.setAtomicityMode(ATOMIC);
+        ccfg.setBackups(1);
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        cfg.setConsistentId(gridName);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMetadataUpdateFromAtomicInvoke() throws Exception {
+        for (int i = 0; i < 2; i++) {
+            client = false;
+
+            log.info("Iteration: " + i);
+
+            final int SRVS = 4;
+
+            startGrids(SRVS);
+
+            awaitPartitionMapExchange();
+
+            final CyclicBarrier b = new CyclicBarrier(3);
+
+            final AtomicBoolean stop = new AtomicBoolean();
+
+            final int META_PRIMARY_NODE = 0;
+            final int META_UPDATE_FROM_NODE = 1;
+
+            BinaryMarshaller marsh = (BinaryMarshaller)ignite(META_PRIMARY_NODE).configuration().getMarshaller();
+
+            marsh.getContext().registerClass(
+                marsh.binaryMarshaller().context().typeId(TestEnum1.class.getName()), TestEnum1.class);
+
+            TestRecordingCommunicationSpi testSpi =
+                (TestRecordingCommunicationSpi)ignite(META_UPDATE_FROM_NODE).configuration().getCommunicationSpi();
+
+            testSpi.blockMessages(GridNearTxPrepareRequest.class, getTestGridName(META_PRIMARY_NODE));
+
+            IgniteInternalFuture<?> sFut1 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    IgniteCache<Object, Object> cache = ignite(META_UPDATE_FROM_NODE).cache("cache");
+
+                    List<Integer> keys = primaryKeys(cache, 1);
+
+                    b.await();
+
+                    cache.invoke(keys.get(0), new TestEntryProcessor());
+
+                    return null;
+                }
+            }, "async-node-0");
+
+            IgniteInternalFuture<?> sFut2 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    IgniteCache<Object, Object> cache = ignite(META_PRIMARY_NODE).cache("cache");
+
+                    List<Integer> keys = primaryKeys(cache, 1);
+
+                    b.await();
+
+                    Thread.sleep(2000);
+
+                    cache.invoke(keys.get(0), new TestEntryProcessor());
+
+                    return null;
+                }
+            }, "async-node-1");
+
+            IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    client = true;
+
+                    b.await();
+
+                    for (int i = 0; i < 1; i++)
+                        startGrid(SRVS + i);
+
+                    stop.set(true);
+
+                    return null;
+                }
+            });
+
+            testSpi.waitForBlocked(GridNearTxPrepareRequest.class, getTestGridName(META_PRIMARY_NODE));
+
+            U.sleep(5000);
+
+            testSpi.stopBlock();
+
+            sFut1.get();
+            sFut2.get();
+            fut2.get();
+
+            stopAllGrids();
+        }
+    }
+
+    /**
+     *
+     */
+    static class TestEntryProcessor implements CacheEntryProcessor<Object, Object, Object> {
+        @Override public Object process(MutableEntry<Object, Object> e, Object... args) {
+            e.setValue(TestEnum1.ENUM);
+
+            return null;
+        }
+    }
+
+    /**
+     *
+     */
+    enum TestEnum1 {
+        /** */
+        ENUM
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
index eafd09f2fdbf9..f32148a272cbf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
@@ -76,7 +76,7 @@ public void testMessageUnmarshallWithoutCacheContext() throws Exception {
             final AtomicBoolean res = new AtomicBoolean();
 
             clientCtx.gridIO().addMessageListener(TOPIC, new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     if (msg instanceof TxLocksResponse) {
                         try {
                             ((TxLocksResponse)msg).finishUnmarshal(clientCtx, clientCtx.deploy().globalLoader());
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
index 723495cfa89c2..293e5789ddf3c 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
@@ -240,7 +240,7 @@ private static void receiveMessages(final IgniteKernal g) {
         GridMessageListener lsnr = new GridMessageListener() {
             private ClusterNode node;
 
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (node == null)
                     node = g.context().discovery().node(nodeId);
 
@@ -336,7 +336,7 @@ private ClusterNode awaitOther(final GridDiscoveryManager disc) throws Interrupt
      */
     private static class SenderMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             msgCntr.increment();
 
             if (testLatency)
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
index f2c62559397b4..83efd7de42098 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
@@ -130,7 +130,7 @@ public void testThroughput() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -141,7 +141,7 @@ public void testThroughput() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 msgCntr.increment();
 
                 sem.release();
@@ -224,7 +224,7 @@ public void testLatency() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -235,7 +235,7 @@ public void testLatency() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 map.get(((GridTestMessage)msg).id()).countDown();
             }
         });
@@ -324,7 +324,7 @@ public void testVariableLoad() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -335,7 +335,7 @@ public void testVariableLoad() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 msgCntr.increment();
 
                 sem.release();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
index 9c975422a2601..2d04db2fedab9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
@@ -139,7 +139,7 @@ private void doSend() throws Exception {
         final CountDownLatch latch = new CountDownLatch(SAMPLE_CNT);
 
         mgr1.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 try {
                     latch.countDown();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index 1c8acbc3960ce..0c04039753beb 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -328,7 +328,7 @@ public Serializable removeSentMessage(ClusterNode node) {
     @SuppressWarnings("deprecation")
     public void triggerMessage(ClusterNode node, Object msg) {
         for (GridMessageListener lsnr : msgLsnrs)
-            lsnr.onMessage(node.id(), msg);
+            lsnr.onMessage(node.id(), msg, (byte)0);
     }
 
     /** {@inheritDoc} */
@@ -667,7 +667,7 @@ private class GridLocalMessageListener implements GridMessageListener {
         @SuppressWarnings({
             "SynchronizationOnLocalVariableOrMethodParameter", "ConstantConditions",
             "OverlyStrongTypeCast"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridIoUserMessage ioMsg = (GridIoUserMessage)msg;
 
             ClusterNode node = locNode;
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
index 05137868443c8..fb38c5535b35b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.GridCachePutAllFailoverSelfTest;
+import org.apache.ignite.internal.processors.cache.IgniteBinaryMetadataUpdateFromInvoke;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicPutAllFailoverSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCachePutAllRestartTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteBinaryMetadataUpdateNodeRestartTest;
@@ -45,6 +46,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCachePutAllFailoverSelfTest.class);
 
         suite.addTestSuite(IgniteBinaryMetadataUpdateNodeRestartTest.class);
+        suite.addTestSuite(IgniteBinaryMetadataUpdateFromInvoke.class);
 
         suite.addTestSuite(IgniteCacheGetRestartTest.class);
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index c29239f143719..22b94c7ac796b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -138,7 +138,7 @@ protected final void initDistributedJoinMessaging(GridH2Table tbl) {
             msgTopic = new IgniteBiTuple<>(GridTopic.TOPIC_QUERY, tbl.identifier() + '.' + getName());
 
             msgLsnr = new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     GridSpinBusyLock l = desc.indexing().busyLock();
 
                     if (!l.enterBusy())
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index ac1a6a60c0200..0605287ded597 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -167,7 +167,7 @@ public void start(final GridKernalContext ctx, IgniteH2Indexing h2) throws Ignit
         }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
 
         ctx.io().addMessageListener(GridTopic.TOPIC_QUERY, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!busyLock.enterBusy())
                     return;
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index ee9976c22846a..cbfe1de8735aa 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -189,7 +189,7 @@ public void start(final GridKernalContext ctx, final IgniteH2Indexing h2) throws
         log = ctx.log(GridReduceQueryExecutor.class);
 
         ctx.io().addMessageListener(GridTopic.TOPIC_QUERY, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!busyLock.enterBusy())
                     return;
 

From 0521b7780756788d92bfa35ef00f56b5bb01367d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 27 Jun 2017 20:43:07 +0300
Subject: [PATCH 324/446] GG-12370: Fixed massive NODE_FAILED events lead to
 excessive momentary memory consumption.

---
 .../GridCachePartitionExchangeManager.java    |  5 +-
 .../CacheLateAffinityAssignmentTest.java      | 68 +++++++++++++++++++
 2 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 07805f3f1b50b..4f5feaefc1554 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -269,8 +269,9 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                                 exchFut = exchangeFuture(exchId, evt, cache, null, msg);
                             }
                         }
-                        else
-                            exchangeFuture(msg.exchangeId(), null, null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
+                        else if (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                            exchangeFuture(msg.exchangeId(), null, null, null, null)
+                                .onAffinityChangeMessage(customEvt.eventNode(), msg);
                     }
                 }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 7e37450dd5621..771ab34116030 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -69,6 +69,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
@@ -1168,6 +1169,73 @@ public void testDelayAssignmentAffinityChanged() throws Exception {
         checkAffinity(4, topVer(4, 1), true);
     }
 
+    /**
+     * Wait for rebalance, send affinity change message, but affinity already changed (new node joined).
+     *
+     * @throws Exception If failed.
+     */
+    public void testDelayAssignmentAffinityChanged2() throws Exception {
+        Ignite ignite0 = startServer(0, 1);
+
+        TestTcpDiscoverySpi discoSpi0 =
+            (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+        TestRecordingCommunicationSpi commSpi0 =
+            (TestRecordingCommunicationSpi)ignite0.configuration().getCommunicationSpi();
+
+        startClient(1, 2);
+
+        checkAffinity(2, topVer(2, 0), true);
+
+        startServer(2, 3);
+
+        checkAffinity(3, topVer(3, 1), false);
+
+        discoSpi0.blockCustomEvent();
+
+        stopNode(2, 4);
+
+        discoSpi0.waitCustomEvent();
+
+        blockSupplySend(commSpi0, CACHE_NAME1);
+
+        final IgniteInternalFuture<?> startedFuture = multithreadedAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                startServer(3, 5);
+
+                return null;
+            }
+        }, 1, "server-starter");
+
+        Thread.sleep(2_000);
+
+        discoSpi0.stopBlock();
+
+        boolean started = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return startedFuture.isDone();
+            }
+        }, 10_000);
+
+        if (!started)
+            startedFuture.cancel();
+
+        assertTrue(started);
+
+        checkAffinity(3, topVer(5, 0), false);
+
+        checkNoExchange(3, topVer(5, 1));
+
+        commSpi0.stopBlock();
+
+        checkAffinity(3, topVer(5, 1), true);
+
+        List<GridDhtPartitionsExchangeFuture> exFutures = grid(3).context().cache().context().exchange().exchangeFutures();
+
+        for (GridDhtPartitionsExchangeFuture f : exFutures)
+            //Shouldn't contains staled futures.
+            assertTrue(f.topologyVersion().topologyVersion() >= 5);
+    }
+
     /**
      * Wait for rebalance, cache is destroyed and created again.
      *

From 6abe5bf5bd732bf9f79df577e159243520dd5c0b Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 09:47:45 +0300
Subject: [PATCH 325/446] Fixed compilation.

---
 .../internal/processors/hadoop/shuffle/HadoopShuffle.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
index 8ffea8c75075d..81001a6ff0dec 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -63,7 +63,7 @@ public class HadoopShuffle extends HadoopComponent {
         super.start(ctx);
 
         ctx.kernalContext().io().addMessageListener(GridTopic.TOPIC_HADOOP_MSG, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 onMessageReceived(nodeId, (HadoopMessage)msg);
             }
         });

From 7d5217260b293b7224340349b5e44792999600f3 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 11:47:21 +0300
Subject: [PATCH 326/446] Rethrow handshake exceptions as it done for
 ConnectionException.

---
 .../tcp/TcpCommunicationSpi.java              | 32 +++++++++++++------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 10d2141ffdf15..0f910df798947 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3009,7 +3009,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
 
                     // Reconnect for the second time, if connection is not established.
                     if (!failureDetThrReached && connectAttempts < 5 &&
-                        (X.hasCause(e, ConnectException.class) || X.hasCause(e, SocketTimeoutException.class))) {
+                        (X.hasCause(e, ConnectException.class, HandshakeException.class, SocketTimeoutException.class))) {
                         U.sleep(200);
 
                         connectAttempts++;
@@ -3036,7 +3036,8 @@ else if (X.hasCause(e, SocketTimeoutException.class))
 
             if (enableForcibleNodeKill) {
                 if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
-                    X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
+                    X.hasCause(errs, ConnectException.class, HandshakeException.class,
+                        SocketTimeoutException.class, HandshakeTimeoutException.class,
                         IgniteSpiOperationTimeoutException.class)) {
                     U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
                         "cluster [" + "rmtNode=" + node + ']', errs);
@@ -3048,7 +3049,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                 }
             }
 
-            if (X.hasCause(errs, ConnectException.class))
+            if (X.hasCause(errs, ConnectException.class, HandshakeException.class))
                 throw errs;
         }
 
@@ -3101,7 +3102,7 @@ private <T> long safeHandshake(
                         sslHnd = new BlockingSslHandler(sslMeta.sslEngine(), ch, directBuf, ByteOrder.nativeOrder(), log);
 
                         if (!sslHnd.handshake())
-                            throw new IgniteCheckedException("SSL handshake is not completed.");
+                            throw new HandshakeException("SSL handshake is not completed.");
 
                         ByteBuffer handBuff = sslHnd.applicationBuffer();
 
@@ -3111,7 +3112,7 @@ private <T> long safeHandshake(
                             int read = ch.read(buf);
 
                             if (read == -1)
-                                throw new IgniteCheckedException("Failed to read remote node ID (connection closed).");
+                                throw new HandshakeException("Failed to read remote node ID (connection closed).");
 
                             buf.flip();
 
@@ -3127,7 +3128,7 @@ private <T> long safeHandshake(
                             int read = ch.read(buf);
 
                             if (read == -1)
-                                throw new IgniteCheckedException("Failed to read remote node ID (connection closed).");
+                                throw new HandshakeException("Failed to read remote node ID (connection closed).");
 
                             i += read;
                         }
@@ -3136,7 +3137,7 @@ private <T> long safeHandshake(
                     UUID rmtNodeId0 = U.bytesToUuid(buf.array(), 1);
 
                     if (!rmtNodeId.equals(rmtNodeId0))
-                        throw new IgniteCheckedException("Remote node ID is not as expected [expected=" + rmtNodeId +
+                        throw new HandshakeException("Remote node ID is not as expected [expected=" + rmtNodeId +
                             ", rcvd=" + rmtNodeId0 + ']');
                     else if (log.isDebugEnabled())
                         log.debug("Received remote node ID: " + rmtNodeId0);
@@ -3222,7 +3223,7 @@ else if (log.isDebugEnabled())
                                 int read = ch.read(buf);
 
                                 if (read == -1)
-                                    throw new IgniteCheckedException("Failed to read remote node recovery handshake " +
+                                    throw new HandshakeException("Failed to read remote node recovery handshake " +
                                         "(connection closed).");
 
                                 buf.flip();
@@ -3260,7 +3261,7 @@ else if (log.isDebugEnabled())
                                 int read = ch.read(buf);
 
                                 if (read == -1)
-                                    throw new IgniteCheckedException("Failed to read remote node recovery handshake " +
+                                    throw new HandshakeException("Failed to read remote node recovery handshake " +
                                         "(connection closed).");
 
                                 i += read;
@@ -3480,6 +3481,19 @@ private NodeIdMessage nodeIdMessage() {
         return S.toString(TcpCommunicationSpi.class, this);
     }
 
+    /** Internal exception class for proper timeout handling. */
+    private static class HandshakeException extends IgniteCheckedException {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param msg Error message.
+         */
+        HandshakeException(String msg) {
+            super(msg);
+        }
+    }
+
     /** Internal exception class for proper timeout handling. */
     private static class HandshakeTimeoutException extends IgniteCheckedException {
         /** */

From 81cdea40743131cac9dae49150c1038073595f7e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 13:00:30 +0300
Subject: [PATCH 327/446] Fixed tests.

---
 .../IgniteClientReconnectAbstractTest.java        | 10 +++++-----
 ...iteClientReconnectContinuousProcessorTest.java | 15 +--------------
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
index a793760f44fbf..17a1ad41ae448 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
@@ -69,9 +70,6 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
     /** */
     private static final long RECONNECT_TIMEOUT = 10_000;
 
-    /** Reconnect should occurs before failure detection time is out. */
-    public static final long FAILURE_DETECTION_TIMEOUT = RECONNECT_TIMEOUT +2000;
-
     /** */
     protected boolean clientMode;
 
@@ -79,8 +77,6 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
-        cfg.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT);
-
         TestTcpDiscoverySpi disco = new TestTcpDiscoverySpi();
 
         disco.setIpFinder(ipFinder);
@@ -157,6 +153,8 @@ protected BlockTcpCommunicationSpi commSpi(Ignite ignite) {
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL, "true");
+
         int srvs = serverCount();
 
         if (srvs > 0)
@@ -177,6 +175,8 @@ protected BlockTcpCommunicationSpi commSpi(Ignite ignite) {
     @Override protected void afterTestsStopped() throws Exception {
         super.afterTestsStopped();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         stopAllGrids();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
index 0ff5883d11b57..c86dcabb3a208 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
@@ -23,6 +23,7 @@
 import javax.cache.event.CacheEntryUpdatedListener;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.query.ContinuousQuery;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -110,20 +111,6 @@ public void testEventListenerReconnect() throws Exception {
         assertFalse(lsnr.latch.await(3000, MILLISECONDS));
     }
 
-    /**
-     * @throws Exception If failed.
-     */
-    public void testMessageListenerReconnectAndStopFromServer() throws Exception {
-        testMessageListenerReconnect(false);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testMessageListenerReconnectAndStopFromClient() throws Exception {
-        testMessageListenerReconnect(true);
-    }
-
     /**
      * @param stopFromClient If {@code true} stops listener from client node, otherwise from server.
      * @throws Exception If failed.

From 473abcafca568c7c898b0b1ae91fe964084fdf43 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 28 Jun 2017 14:49:49 +0300
Subject: [PATCH 328/446] Tests fixed

---
 .../IgniteCachePutRetryAbstractSelfTest.java  | 51 +++----------------
 1 file changed, 6 insertions(+), 45 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
index fc9017927d3bc..0c2b868704cd3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
@@ -57,7 +57,6 @@
 import org.apache.ignite.spi.swapspace.inmemory.GridTestSwapSpaceSpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -631,71 +630,33 @@ public SetEntryProcessor(Integer val) {
         }
     }
 
-    /**
-     *
-     */
     private static class TestStoreFactory implements Factory<CacheStore> {
         /** {@inheritDoc} */
-        @Override public CacheStore create() {
-            return new CacheStoreAdapter() {
-                @Override public Object load(Object key) throws CacheLoaderException {
-                    return null;
-                }
-
-                @Override public void write(Cache.Entry entry) throws CacheWriterException {
-                    // No-op.
-                }
-
-                @Override public void delete(Object key) throws CacheWriterException {
-                    // No-op.
-                }
-            };
-        }
-    }
-
-
-/*
-    private static class TestStoreFactory implements Factory<CacheStore> {
-        */
-/** {@inheritDoc} *//*
-
         @Override public CacheStore create() {
             return new TestCacheStore();
         }
     }
 
-    */
-/**
+    /**
      *
-     *//*
-
+     */
     private static class TestCacheStore extends CacheStoreAdapter {
-        */
-/** Store map. *//*
-
+        /** Store map. */
         private static Map STORE_MAP = new ConcurrentHashMap();
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public Object load(Object key) throws CacheLoaderException {
             return STORE_MAP.get(key);
         }
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public void write(Cache.Entry entry) throws CacheWriterException {
             STORE_MAP.put(entry.getKey(), entry.getValue());
         }
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public void delete(Object key) throws CacheWriterException {
             STORE_MAP.remove(key);
         }
     }
-*/
-
 }
\ No newline at end of file

From 4f383ae772631987c3f4ac29bb654b4811fbc407 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 28 Jun 2017 14:49:49 +0300
Subject: [PATCH 329/446] Tests fixed.

---
 .../IgniteCachePutRetryAbstractSelfTest.java  | 51 +++----------------
 1 file changed, 6 insertions(+), 45 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
index fc9017927d3bc..0c2b868704cd3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
@@ -57,7 +57,6 @@
 import org.apache.ignite.spi.swapspace.inmemory.GridTestSwapSpaceSpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -631,71 +630,33 @@ public SetEntryProcessor(Integer val) {
         }
     }
 
-    /**
-     *
-     */
     private static class TestStoreFactory implements Factory<CacheStore> {
         /** {@inheritDoc} */
-        @Override public CacheStore create() {
-            return new CacheStoreAdapter() {
-                @Override public Object load(Object key) throws CacheLoaderException {
-                    return null;
-                }
-
-                @Override public void write(Cache.Entry entry) throws CacheWriterException {
-                    // No-op.
-                }
-
-                @Override public void delete(Object key) throws CacheWriterException {
-                    // No-op.
-                }
-            };
-        }
-    }
-
-
-/*
-    private static class TestStoreFactory implements Factory<CacheStore> {
-        */
-/** {@inheritDoc} *//*
-
         @Override public CacheStore create() {
             return new TestCacheStore();
         }
     }
 
-    */
-/**
+    /**
      *
-     *//*
-
+     */
     private static class TestCacheStore extends CacheStoreAdapter {
-        */
-/** Store map. *//*
-
+        /** Store map. */
         private static Map STORE_MAP = new ConcurrentHashMap();
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public Object load(Object key) throws CacheLoaderException {
             return STORE_MAP.get(key);
         }
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public void write(Cache.Entry entry) throws CacheWriterException {
             STORE_MAP.put(entry.getKey(), entry.getValue());
         }
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public void delete(Object key) throws CacheWriterException {
             STORE_MAP.remove(key);
         }
     }
-*/
-
 }
\ No newline at end of file

From 4b11bb27d5caeb7ba7bc2929685d7c548f4c327d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 15:45:42 +0300
Subject: [PATCH 330/446] Tests fixed.

---
 .../cache/IgniteDynamicCacheStartSelfTest.java    | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
index 48e06ee7c82a0..49a49c7958e94 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheExistsException;
 import org.apache.ignite.cache.CacheMode;
@@ -144,6 +145,20 @@ public int nodeCount() {
         stopAllGrids();
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        if(grid(0).cache(DYNAMIC_CACHE_NAME)!=null)
+            grid(0).destroyCache(DYNAMIC_CACHE_NAME);
+
+        while(Ignition.allGrids().size() > nodeCount()) {
+            stopGrid(nodeCount());
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From 9b92dac5756619ed5218db269acdae1ef02ef8ae Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 18:20:02 +0300
Subject: [PATCH 331/446] Tests fixed.

---
 .../ignite/spi/communication/tcp/TcpCommunicationSpi.java       | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 0f910df798947..122700e84785c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -23,6 +23,7 @@
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -2917,6 +2918,7 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
+                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +

From 0295a1a7ecb2ef57a917ddc9015ff8b71a6ddb14 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Fri, 23 Jun 2017 21:00:45 +0300
Subject: [PATCH 332/446] IGNITE-5521: Large near caches lead to cluster
 instability with metrics enabled.

---
 .../near/GridNearCacheAdapter.java            |  2 +-
 .../near/GridCacheNearEvictionSelfTest.java   |  5 ++--
 .../near/GridCacheNearMetricsSelfTest.java    | 30 ++++++++++++++++++-
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
index 4ddad74157987..8dfe1698991a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
@@ -308,7 +308,7 @@ protected void processGetResponse(UUID nodeId, GridNearGetResponse res) {
 
     /** {@inheritDoc} */
     @Override public int size() {
-        return nearEntries().size() + dht().size();
+        return dht().size();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
index df5943611075c..6181f50962003 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
@@ -165,12 +165,13 @@ public void testNearEnabledThreeNodes() throws Exception {
 
             for (int i = 0; i < gridCnt; i++) {
                 final GridCacheAdapter cache = internalCache(i);
+                final GridCacheAdapter near =  near(i);
 
                 // Repeatedly check cache sizes because of concurrent cache updates.
                 assertTrue(GridTestUtils.waitForCondition(new PA() {
                     @Override public boolean apply() {
                         // Every node contains either near, backup, or primary.
-                        return cnt == cache.size();
+                        return cnt == cache.size() + near.nearSize();
                     }
                 }, getTestTimeout()));
 
@@ -183,4 +184,4 @@ public void testNearEnabledThreeNodes() throws Exception {
             stopAllGrids();
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
index 09ff519a67976..de2696f56392a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
@@ -21,7 +21,10 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -97,6 +100,31 @@ protected int keyCount() {
         return cc;
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNearCacheDoesNotAffectCacheSize() throws Exception {
+        IgniteCache<Integer, Integer> cache0 = grid(0).cache(null);
+
+        for (int i = 0; i < 100 ; i++)
+            cache0.put(i, i);
+
+        IgniteEx g1 = grid(1);
+
+        IgniteCache<Integer, Integer> cache1 = g1.cache(null    );
+
+        ClusterNode localNode = g1.cluster().localNode();
+
+        int beforeSize = cache1.localMetrics().getSize();
+
+        for (int i = 0; i < 100 ; i++) {
+            if (!affinity(cache1).isPrimaryOrBackup(localNode, i))
+                cache1.get(i); // put entry to near cache
+        }
+
+        assertEquals(beforeSize, cache1.localMetrics().getSize());
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -417,4 +445,4 @@ else if (affinity(jcache).isBackup(g.cluster().localNode(), key)){
             }
         }
     }
-}
\ No newline at end of file
+}

From 16fed552fc8f91de550207eeebd5850e685960a8 Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Thu, 29 Jun 2017 10:49:34 +0300
Subject: [PATCH 333/446] Fix tests.

---
 ...PartitionedBackupNodeFailureRecoveryTest.java | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java
index 6654fd93a77a8..e1ef59c5b79be 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java
@@ -33,6 +33,8 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest;
+import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.testframework.GridTestUtils;
 
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.PRIMARY;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -148,16 +150,22 @@ public void testBackUpFail() throws Exception {
 
                         IgniteEx backUp = startGrid(2);
 
-                        IgniteCache<Integer, Integer> cache3 = backUp.cache(null);
+                        final IgniteCache<Integer, Integer> cache3 = backUp.cache(null);
 
                         lock.lock();
 
                         try {
-                            Integer backUpVal = cache3.localPeek(finalKey);
+                            boolean res = GridTestUtils.waitForCondition(new PA() {
+                                @Override public boolean apply() {
+                                    Integer actl = cache3.localPeek(finalKey);
 
-                            Integer exp = cntr.get();
+                                    Integer exp = cntr.get();
 
-                            assertEquals(exp, backUpVal);
+                                    return exp.equals(actl);
+                                }
+                            }, 1000);
+
+                            assertTrue(res);
                         }
                         finally {
                             lock.unlock();

From 65d93e48c264f4bfff0a94856fdfeb83375a3976 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Fri, 23 Jun 2017 21:00:45 +0300
Subject: [PATCH 334/446] IGNITE-5521: Large near caches lead to cluster
 instability with metrics enabled

---
 .../near/GridNearCacheAdapter.java            |  2 +-
 .../near/GridCacheNearEvictionSelfTest.java   |  5 ++--
 .../near/GridCacheNearMetricsSelfTest.java    | 30 ++++++++++++++++++-
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
index 4ddad74157987..8dfe1698991a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
@@ -308,7 +308,7 @@ protected void processGetResponse(UUID nodeId, GridNearGetResponse res) {
 
     /** {@inheritDoc} */
     @Override public int size() {
-        return nearEntries().size() + dht().size();
+        return dht().size();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
index df5943611075c..6181f50962003 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
@@ -165,12 +165,13 @@ public void testNearEnabledThreeNodes() throws Exception {
 
             for (int i = 0; i < gridCnt; i++) {
                 final GridCacheAdapter cache = internalCache(i);
+                final GridCacheAdapter near =  near(i);
 
                 // Repeatedly check cache sizes because of concurrent cache updates.
                 assertTrue(GridTestUtils.waitForCondition(new PA() {
                     @Override public boolean apply() {
                         // Every node contains either near, backup, or primary.
-                        return cnt == cache.size();
+                        return cnt == cache.size() + near.nearSize();
                     }
                 }, getTestTimeout()));
 
@@ -183,4 +184,4 @@ public void testNearEnabledThreeNodes() throws Exception {
             stopAllGrids();
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
index 09ff519a67976..de2696f56392a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
@@ -21,7 +21,10 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -97,6 +100,31 @@ protected int keyCount() {
         return cc;
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNearCacheDoesNotAffectCacheSize() throws Exception {
+        IgniteCache<Integer, Integer> cache0 = grid(0).cache(null);
+
+        for (int i = 0; i < 100 ; i++)
+            cache0.put(i, i);
+
+        IgniteEx g1 = grid(1);
+
+        IgniteCache<Integer, Integer> cache1 = g1.cache(null    );
+
+        ClusterNode localNode = g1.cluster().localNode();
+
+        int beforeSize = cache1.localMetrics().getSize();
+
+        for (int i = 0; i < 100 ; i++) {
+            if (!affinity(cache1).isPrimaryOrBackup(localNode, i))
+                cache1.get(i); // put entry to near cache
+        }
+
+        assertEquals(beforeSize, cache1.localMetrics().getSize());
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -417,4 +445,4 @@ else if (affinity(jcache).isBackup(g.cluster().localNode(), key)){
             }
         }
     }
-}
\ No newline at end of file
+}

From ff1afad1278e939aef71b274e959fd4256904971 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 27 Jun 2017 10:34:42 +0300
Subject: [PATCH 335/446] Partially reverted GG-12352.

(cherry picked from commit 3668b91)
---
 .../ignite/spi/discovery/tcp/ServerImpl.java      | 10 +---------
 ...IgniteBinaryMetadataUpdateNodeRestartTest.java | 10 ++++++++++
 .../IgniteCacheNearRestartRollbackSelfTest.java   | 15 +++++++++++++++
 ...riteSynchronizationModesMultithreadedTest.java |  5 +++++
 4 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 2f7e9b470aae6..6e5af2986dcff 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1242,7 +1242,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
                 errs.add(e);
 
-                if (X.hasCause(e, SSLException.class)) {
+                if (X.hasCause(e, SSLException.class, StreamCorruptedException.class)) {
                     if (--sslConnectAttempts == 0)
                         throw new IgniteException("Unable to establish secure connection. " +
                             "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
@@ -1250,14 +1250,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                     continue;
                 }
 
-                if (X.hasCause(e, StreamCorruptedException.class)) {
-                    if (--sslConnectAttempts == 0)
-                        throw new IgniteException("Unable to establish plain connection. " +
-                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
-
-                    continue;
-                }
-
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
index 814fb08eacba5..3d552847fd71d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheEntryProcessor;
 import org.apache.ignite.cluster.ClusterTopologyException;
@@ -88,10 +89,19 @@ public class IgniteBinaryMetadataUpdateNodeRestartTest extends GridCommonAbstrac
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         super.afterTestsStopped();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index a48497d8f39c4..3f242b5726867 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -31,6 +31,7 @@
 import javax.cache.processor.MutableEntry;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -92,6 +93,20 @@ public class IgniteCacheNearRestartRollbackSelfTest extends GridCommonAbstractTe
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+    }
+
     /**
      * @param gridName Grid name.
      * @return Cache configuration.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
index 08396daa5efeb..c214a772d422c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cache.store.CacheStoreAdapter;
@@ -104,6 +105,8 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+
         startGrids(SRVS);
 
         clientMode = true;
@@ -119,6 +122,8 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         super.afterTestsStopped();
     }
 

From 7fbb95d1698c534957c9f0b7c5558f43985cfe68 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 27 Jun 2017 20:43:07 +0300
Subject: [PATCH 336/446] GG-12370: Fixed massive NODE_FAILED events lead to
 excessive momentary memory consumption.

(cherry picked from commit 0521b77)
---
 .../GridCachePartitionExchangeManager.java    |  5 +-
 .../CacheLateAffinityAssignmentTest.java      | 68 +++++++++++++++++++
 2 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 92142c0731240..3df0bafc06c25 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -269,8 +269,9 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                                 exchFut = exchangeFuture(exchId, e, null, msg);
                             }
                         }
-                        else
-                            exchangeFuture(msg.exchangeId(), null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
+                        else if (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                            exchangeFuture(msg.exchangeId(), null, null, null)
+                                .onAffinityChangeMessage(customEvt.eventNode(), msg);
                     }
                 }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 7e37450dd5621..771ab34116030 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -69,6 +69,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
@@ -1168,6 +1169,73 @@ public void testDelayAssignmentAffinityChanged() throws Exception {
         checkAffinity(4, topVer(4, 1), true);
     }
 
+    /**
+     * Wait for rebalance, send affinity change message, but affinity already changed (new node joined).
+     *
+     * @throws Exception If failed.
+     */
+    public void testDelayAssignmentAffinityChanged2() throws Exception {
+        Ignite ignite0 = startServer(0, 1);
+
+        TestTcpDiscoverySpi discoSpi0 =
+            (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+        TestRecordingCommunicationSpi commSpi0 =
+            (TestRecordingCommunicationSpi)ignite0.configuration().getCommunicationSpi();
+
+        startClient(1, 2);
+
+        checkAffinity(2, topVer(2, 0), true);
+
+        startServer(2, 3);
+
+        checkAffinity(3, topVer(3, 1), false);
+
+        discoSpi0.blockCustomEvent();
+
+        stopNode(2, 4);
+
+        discoSpi0.waitCustomEvent();
+
+        blockSupplySend(commSpi0, CACHE_NAME1);
+
+        final IgniteInternalFuture<?> startedFuture = multithreadedAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                startServer(3, 5);
+
+                return null;
+            }
+        }, 1, "server-starter");
+
+        Thread.sleep(2_000);
+
+        discoSpi0.stopBlock();
+
+        boolean started = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return startedFuture.isDone();
+            }
+        }, 10_000);
+
+        if (!started)
+            startedFuture.cancel();
+
+        assertTrue(started);
+
+        checkAffinity(3, topVer(5, 0), false);
+
+        checkNoExchange(3, topVer(5, 1));
+
+        commSpi0.stopBlock();
+
+        checkAffinity(3, topVer(5, 1), true);
+
+        List<GridDhtPartitionsExchangeFuture> exFutures = grid(3).context().cache().context().exchange().exchangeFutures();
+
+        for (GridDhtPartitionsExchangeFuture f : exFutures)
+            //Shouldn't contains staled futures.
+            assertTrue(f.topologyVersion().topologyVersion() >= 5);
+    }
+
     /**
      * Wait for rebalance, cache is destroyed and created again.
      *

From 710db327c027d43c0984b7007447cecca71f3cca Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 11:47:21 +0300
Subject: [PATCH 337/446] Rethrow handshake exceptions as it done for
 ConnectionException.

(cherry picked from commit 7d52172)
---
 .../tcp/TcpCommunicationSpi.java              | 32 +++++++++++++------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 1bfa56accf2e0..9925b3d0c8736 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2572,7 +2572,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
 
                     // Reconnect for the second time, if connection is not established.
                     if (!failureDetThrReached && connectAttempts < 5 &&
-                        (X.hasCause(e, ConnectException.class) || X.hasCause(e, SocketTimeoutException.class))) {
+                        (X.hasCause(e, ConnectException.class, HandshakeException.class, SocketTimeoutException.class))) {
                         U.sleep(200);
 
                         connectAttempts++;
@@ -2599,7 +2599,8 @@ else if (X.hasCause(e, SocketTimeoutException.class))
 
             if (enableForcibleNodeKill) {
                 if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
-                    X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
+                    X.hasCause(errs, ConnectException.class, HandshakeException.class,
+                        SocketTimeoutException.class, HandshakeTimeoutException.class,
                         IgniteSpiOperationTimeoutException.class)) {
                     U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
                         "cluster [" + "rmtNode=" + node + ']', errs);
@@ -2611,7 +2612,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                 }
             }
 
-            if (X.hasCause(errs, ConnectException.class))
+            if (X.hasCause(errs, ConnectException.class, HandshakeException.class))
                 throw errs;
         }
 
@@ -2662,7 +2663,7 @@ private <T> long safeHandshake(
                         sslHnd = new BlockingSslHandler(sslMeta.sslEngine(), ch, directBuf, ByteOrder.nativeOrder(), log);
 
                         if (!sslHnd.handshake())
-                            throw new IgniteCheckedException("SSL handshake is not completed.");
+                            throw new HandshakeException("SSL handshake is not completed.");
 
                         ByteBuffer handBuff = sslHnd.applicationBuffer();
 
@@ -2672,7 +2673,7 @@ private <T> long safeHandshake(
                             int read = ch.read(buf);
 
                             if (read == -1)
-                                throw new IgniteCheckedException("Failed to read remote node ID (connection closed).");
+                                throw new HandshakeException("Failed to read remote node ID (connection closed).");
 
                             buf.flip();
 
@@ -2688,7 +2689,7 @@ private <T> long safeHandshake(
                             int read = ch.read(buf);
 
                             if (read == -1)
-                                throw new IgniteCheckedException("Failed to read remote node ID (connection closed).");
+                                throw new HandshakeException("Failed to read remote node ID (connection closed).");
 
                             i += read;
                         }
@@ -2697,7 +2698,7 @@ private <T> long safeHandshake(
                     UUID rmtNodeId0 = U.bytesToUuid(buf.array(), 1);
 
                     if (!rmtNodeId.equals(rmtNodeId0))
-                        throw new IgniteCheckedException("Remote node ID is not as expected [expected=" + rmtNodeId +
+                        throw new HandshakeException("Remote node ID is not as expected [expected=" + rmtNodeId +
                             ", rcvd=" + rmtNodeId0 + ']');
                     else if (log.isDebugEnabled())
                         log.debug("Received remote node ID: " + rmtNodeId0);
@@ -2768,7 +2769,7 @@ else if (log.isDebugEnabled())
                                 int read = ch.read(buf);
 
                                 if (read == -1)
-                                    throw new IgniteCheckedException("Failed to read remote node recovery handshake " +
+                                    throw new HandshakeException("Failed to read remote node recovery handshake " +
                                         "(connection closed).");
 
                                 buf.flip();
@@ -2802,7 +2803,7 @@ else if (log.isDebugEnabled())
                                 int read = ch.read(buf);
 
                                 if (read == -1)
-                                    throw new IgniteCheckedException("Failed to read remote node recovery handshake " +
+                                    throw new HandshakeException("Failed to read remote node recovery handshake " +
                                         "(connection closed).");
 
                                 i += read;
@@ -2990,6 +2991,19 @@ private ClientKey(UUID nodeId, long order) {
         }
     }
 
+    /** Internal exception class for proper timeout handling. */
+    private static class HandshakeException extends IgniteCheckedException {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param msg Error message.
+         */
+        HandshakeException(String msg) {
+            super(msg);
+        }
+    }
+
     /** Internal exception class for proper timeout handling. */
     private static class HandshakeTimeoutException extends IgniteCheckedException {
         /** */

From 7c619f77009cc6876405839f9f6201d332d94b47 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 15:45:42 +0300
Subject: [PATCH 338/446] Fixed tests.

(cherry picked from commit 81cdea4)
---
 .../IgniteClientReconnectAbstractTest.java        |  5 +++++
 ...iteClientReconnectContinuousProcessorTest.java | 15 +--------------
 .../cache/IgniteDynamicCacheStartSelfTest.java    | 15 +++++++++++++++
 3 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
index 4d49366e1ad23..17a1ad41ae448 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
@@ -152,6 +153,8 @@ protected BlockTcpCommunicationSpi commSpi(Ignite ignite) {
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL, "true");
+
         int srvs = serverCount();
 
         if (srvs > 0)
@@ -172,6 +175,8 @@ protected BlockTcpCommunicationSpi commSpi(Ignite ignite) {
     @Override protected void afterTestsStopped() throws Exception {
         super.afterTestsStopped();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         stopAllGrids();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
index 0ff5883d11b57..c86dcabb3a208 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
@@ -23,6 +23,7 @@
 import javax.cache.event.CacheEntryUpdatedListener;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.query.ContinuousQuery;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -110,20 +111,6 @@ public void testEventListenerReconnect() throws Exception {
         assertFalse(lsnr.latch.await(3000, MILLISECONDS));
     }
 
-    /**
-     * @throws Exception If failed.
-     */
-    public void testMessageListenerReconnectAndStopFromServer() throws Exception {
-        testMessageListenerReconnect(false);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testMessageListenerReconnectAndStopFromClient() throws Exception {
-        testMessageListenerReconnect(true);
-    }
-
     /**
      * @param stopFromClient If {@code true} stops listener from client node, otherwise from server.
      * @throws Exception If failed.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
index 48e06ee7c82a0..49a49c7958e94 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheExistsException;
 import org.apache.ignite.cache.CacheMode;
@@ -144,6 +145,20 @@ public int nodeCount() {
         stopAllGrids();
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        if(grid(0).cache(DYNAMIC_CACHE_NAME)!=null)
+            grid(0).destroyCache(DYNAMIC_CACHE_NAME);
+
+        while(Ignition.allGrids().size() > nodeCount()) {
+            stopGrid(nodeCount());
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From 5219a35208b95a2d7dea3825dd4cb4edc74b3542 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Mon, 26 Jun 2017 15:37:26 +0300
Subject: [PATCH 339/446] IGNITE-5076: improved multi-threaded start of nodes

---
 .../ignite/testframework/junits/GridAbstractTest.java  | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
 mode change 100644 => 100755 modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
old mode 100644
new mode 100755
index ffaec4fbc89af..5c43ecbfc9bb2
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -646,12 +646,14 @@ protected final Ignite startGrids(int cnt) throws Exception {
      * @throws Exception If failed.
      */
     protected Ignite startGridsMultiThreaded(int cnt) throws Exception {
-        if (cnt == 1)
-            return startGrids(1);
+        assert cnt > 0 : "Number of grids must be a positive number";
 
-        Ignite ignite = startGridsMultiThreaded(0, cnt);
+        Ignite ignite = startGrids(1);
 
-        checkTopology(cnt);
+        if (cnt > 1) {
+            startGridsMultiThreaded(1, cnt - 1);
+            checkTopology(cnt);
+        }
 
         return ignite;
     }

From a4fc555b118d91ec0348154b88764f010dbbae52 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 30 Jun 2017 14:38:54 +0300
Subject: [PATCH 340/446] Fixed "IGNITE-5424 GridServiceProxy does not unwraps
 exception message from InvocationTargetException." This closes #2168

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../processors/service/GridServiceProxy.java  |  9 ++-
 .../GridServiceProcessorProxySelfTest.java    | 66 +++++++++++++++++++
 2 files changed, 74 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index 2286cff9717aa..6e8fb4e0ae667 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -23,6 +23,7 @@
 import java.io.ObjectOutput;
 import java.io.Serializable;
 import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.util.ArrayList;
@@ -403,7 +404,13 @@ private ServiceProxyCallable(String mtdName, String svcName, Class[] argTypes, O
             if (mtd == null)
                 throw new GridServiceMethodNotFoundException(svcName, mtdName, argTypes);
 
-            return mtd.invoke(svcCtx.service(), args);
+            try {
+                return mtd.invoke(svcCtx.service(), args);
+            }
+            catch (InvocationTargetException e) {
+                // Get error message.
+                throw new IgniteCheckedException(e.getCause().getMessage(), e);
+            }
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
index 6fc7e02e42946..850cb5702b5b0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
@@ -18,12 +18,15 @@
 package org.apache.ignite.internal.processors.service;
 
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
 
 /**
  * Service proxy test.
@@ -63,6 +66,31 @@ public void testNodeSingletonProxy() throws Exception {
         }
     }
 
+    /**
+     * Unwraps error message from InvocationTargetException.
+     *
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    public void testException() throws Exception {
+        String name = "errorService";
+
+        Ignite ignite = grid(0);
+
+        ignite.services(ignite.cluster().forRemotes()).deployNodeSingleton(name, new ErrorServiceImpl());
+
+        final ErrorService svc = ignite.services().serviceProxy(name, ErrorService.class, false);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                svc.go();
+
+                return null;
+            }
+        }, IgniteException.class, "Test exception");
+
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -353,6 +381,7 @@ protected static class MapServiceImpl<K, V> implements MapService<K, V>, Service
             map.clear();
         }
 
+        /** {@inheritDoc} */
         @Override public int size() {
             return map.size();
         }
@@ -372,4 +401,41 @@ protected static class MapServiceImpl<K, V> implements MapService<K, V>, Service
             X.println("Executing cache service: " + ctx.name());
         }
     }
+
+    /**
+     *
+     */
+    protected interface ErrorService extends Service {
+        /**
+         *
+         */
+        void go() throws Exception;
+    }
+
+    /**
+     *
+     */
+    protected class ErrorServiceImpl implements ErrorService {
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(ServiceContext ctx) throws Exception {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void go() throws Exception {
+            throw new Exception("Test exception");
+        }
+    }
+
+
 }
\ No newline at end of file

From 62876fefc8dbd334db9dc6741c2bc91b3409343d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 30 Jun 2017 14:45:18 +0300
Subject: [PATCH 341/446] IGNITE-5473 partial fix: Create ignite
 troubleshooting logger.

---
 .../apache/ignite/IgniteSystemProperties.java |  7 +++++++
 .../tcp/TcpCommunicationSpi.java              | 20 ++++++++++++++-----
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 24df237db6a75..aaba91e15d304 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -136,6 +136,13 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_QUIET = "IGNITE_QUIET";
 
+    /**
+     * Setting this option to {@code true} will enable troubleshooting logger.
+     * Troubleshooting logger makes logging more verbose without enabling debug mode
+     * to provide more detailed logs without performance penalty.
+     */
+    public static final String IGNITE_TROUBLESHOOTING_LOGGER = "IGNITE_TROUBLESHOOTING_LOGGER";
+
     /**
      * Setting to {@code true} enables writing sensitive information in {@code toString()} output.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 122700e84785c..7c49a35b5c62a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -349,6 +349,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     private boolean enableForcibleNodeKill = IgniteSystemProperties
         .getBoolean(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
 
+    /** */
+    private boolean enableTroubleshootingLog = IgniteSystemProperties
+        .getBoolean(IgniteSystemProperties.IGNITE_TROUBLESHOOTING_LOGGER);
+
     /** Server listener. */
     private final GridNioServerListener<Message> srvLsnr =
         new GridNioServerListenerAdapter<Message>() {
@@ -2989,9 +2993,10 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
                     boolean failureDetThrReached = timeoutHelper.checkFailureTimeoutReached(e);
 
-                    U.error(log, "Failed to establish connection to a remote node [node=" + node +
-                        ", addr=" + addr + ", connectAttempts=" + connectAttempts +
-                        ", failureDetThrReached" + failureDetThrReached + ']', e);
+                    if (enableTroubleshootingLog)
+                        U.error(log, "Failed to establish connection to a remote node [node=" + node +
+                            ", addr=" + addr + ", connectAttempts=" + connectAttempts +
+                            ", failureDetThrReached=" + failureDetThrReached + ']', e);
 
                     if (failureDetThrReached)
                         LT.warn(log, "Connect timed out (consider increasing 'failureDetectionTimeout' " +
@@ -3041,8 +3046,13 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     X.hasCause(errs, ConnectException.class, HandshakeException.class,
                         SocketTimeoutException.class, HandshakeTimeoutException.class,
                         IgniteSpiOperationTimeoutException.class)) {
-                    U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
-                        "cluster [" + "rmtNode=" + node + ']', errs);
+                    String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
+                        "cluster [" + "rmtNode=" + node + ']';
+
+                    if(enableTroubleshootingLog)
+                        U.error(log, msg, errs);
+                    else
+                        U.warn(log, msg);
 
                     getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
                         "rmtNode=" + node +

From 752b1368adbb5c77b5d5caca3c07a72decff5111 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 30 Jun 2017 14:45:18 +0300
Subject: [PATCH 342/446] IGNITE-5473 partial fix: Create ignite
 troubleshooting logger.

---
 .../apache/ignite/IgniteSystemProperties.java |  7 +++++++
 .../tcp/TcpCommunicationSpi.java              | 20 ++++++++++++++-----
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index e9bbf5a10df4c..c4208a76f1a7d 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -135,6 +135,13 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_QUIET = "IGNITE_QUIET";
 
+    /**
+     * Setting this option to {@code true} will enable troubleshooting logger.
+     * Troubleshooting logger makes logging more verbose without enabling debug mode
+     * to provide more detailed logs without performance penalty.
+     */
+    public static final String IGNITE_TROUBLESHOOTING_LOGGER = "IGNITE_TROUBLESHOOTING_LOGGER";
+
     /**
      * Setting to {@code true} enables writing sensitive information in {@code toString()} output.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 9925b3d0c8736..b16842e0d1129 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -333,6 +333,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     private boolean enableForcibleNodeKill = IgniteSystemProperties
         .getBoolean(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
 
+    /** */
+    private boolean enableTroubleshootingLog = IgniteSystemProperties
+        .getBoolean(IgniteSystemProperties.IGNITE_TROUBLESHOOTING_LOGGER);
+
     /** Server listener. */
     private final GridNioServerListener<Message> srvLsnr =
         new GridNioServerListenerAdapter<Message>() {
@@ -2550,9 +2554,10 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
 
                     boolean failureDetThrReached = timeoutHelper.checkFailureTimeoutReached(e);
 
-                    U.error(log, "Failed to establish connection to a remote node [node=" + node +
-                        ", addr=" + addr + ", connectAttempts=" + connectAttempts +
-                        ", failureDetThrReached" + failureDetThrReached + ']', e);
+                    if (enableTroubleshootingLog)
+                        U.error(log, "Failed to establish connection to a remote node [node=" + node +
+                            ", addr=" + addr + ", connectAttempts=" + connectAttempts +
+                            ", failureDetThrReached=" + failureDetThrReached + ']', e);
 
                     if (failureDetThrReached)
                         LT.warn(log, "Connect timed out (consider increasing 'failureDetectionTimeout' " +
@@ -2602,8 +2607,13 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     X.hasCause(errs, ConnectException.class, HandshakeException.class,
                         SocketTimeoutException.class, HandshakeTimeoutException.class,
                         IgniteSpiOperationTimeoutException.class)) {
-                    U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
-                        "cluster [" + "rmtNode=" + node + ']', errs);
+                    String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
+                        "cluster [" + "rmtNode=" + node + ']';
+
+                    if(enableTroubleshootingLog)
+                        U.error(log, msg, errs);
+                    else
+                        U.warn(log, msg);
 
                     getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
                         "rmtNode=" + node +

From 1525c6cf2cb015289392eb54fec4029e9b53b438 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 30 Jun 2017 18:36:50 +0300
Subject: [PATCH 343/446] Fixed service deployment tests.

---
 .../IgniteServiceDynamicCachesSelfTest.java    | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
index c41f2f03c460e..acd2b252d8f9d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
@@ -43,8 +43,8 @@ public class IgniteServiceDynamicCachesSelfTest extends GridCommonAbstractTest {
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
         TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
 
@@ -83,7 +83,7 @@ public void testDeployCalledAfterCacheStart() throws Exception {
 
             final String svcName = "myService";
 
-            svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, "key");
+            svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, primaryKey(ig.cache(cacheName)));
 
             boolean res = GridTestUtils.waitForCondition(new PA() {
                 @Override public boolean apply() {
@@ -125,7 +125,15 @@ public void testDeployCalledBeforeCacheStart() throws Exception {
 
         final String svcName = "myService";
 
-        svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, "key");
+        ig.createCache(ccfg);
+
+        Object key = primaryKey(ig.cache(cacheName));
+
+        ig.destroyCache(cacheName);
+
+        awaitPartitionMapExchange();
+
+        svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, key);
 
         assert svcs.service(svcName) == null;
 
@@ -140,6 +148,8 @@ public void testDeployCalledBeforeCacheStart() throws Exception {
 
             assertTrue("Service was not deployed", res);
 
+            info("stopping cache: " + cacheName);
+
             ig.destroyCache(cacheName);
 
             res = GridTestUtils.waitForCondition(new PA() {

From a24ca24dd7aaa34707d74a4e660d769d3d5b0ed8 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 19 Apr 2017 12:46:31 +0300
Subject: [PATCH 344/446] Attempt to fix awaitPartitionMapExchange: wait for
 last exchange completion to avoid races with cache destroy.

(cherry picked from commit d383484)
---
 .../junits/common/GridCommonAbstractTest.java | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 90fabd95a1b47..6616daf297388 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -58,6 +58,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
@@ -450,6 +451,27 @@ protected void awaitPartitionMapExchange(boolean waitEvicts,
 
         Set<String> names = new HashSet<>();
 
+        Ignite crd = null;
+
+        for (Ignite g : G.allGrids()) {
+            ClusterNode node = g.cluster().localNode();
+
+            if (crd == null || node.order() < crd.cluster().localNode().order()) {
+                crd = g;
+
+                if (node.order() == 1)
+                    break;
+            }
+        }
+
+        if (crd == null)
+            return;
+
+        AffinityTopologyVersion waitTopVer = ((IgniteKernal)crd).context().discovery().topologyVersionEx();
+
+        if (waitTopVer.topologyVersion() <= 0)
+            waitTopVer = new AffinityTopologyVersion(1, 0);
+
         for (Ignite g : G.allGrids()) {
             if (nodes != null && !nodes.contains(g.cluster().localNode()))
                 continue;
@@ -466,6 +488,19 @@ protected void awaitPartitionMapExchange(boolean waitEvicts,
             else
                 startTime = g0.context().discovery().gridStartTime();
 
+            IgniteInternalFuture<?> exchFut =
+                g0.context().cache().context().exchange().affinityReadyFuture(waitTopVer);
+
+            if (exchFut != null && !exchFut.isDone()) {
+                try {
+                    exchFut.get(timeout);
+                }
+                catch (IgniteCheckedException e) {
+                    log.error("Failed to wait for exchange [topVer=" + waitTopVer +
+                        ", node=" + g0.name() + ']', e);
+                }
+            }
+
             for (IgniteCacheProxy<?, ?> c : g0.context().cache().jcaches()) {
                 CacheConfiguration cfg = c.context().config();
 

From ef0a874ceb5c8bfa53e16337f6fd1699afaf2a39 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 30 Jun 2017 20:39:01 +0300
Subject: [PATCH 345/446] Fixed
 CacheSerializableTransactionsTest#testTxConflictRemoveWithOldValue test.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../ignite/internal/processors/cache/GridCacheAdapter.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 220c192f21c46..cde6309d3f9ff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3011,7 +3011,7 @@ protected boolean remove0(final K key, final CacheEntryPredicate filter) throws
                     Collections.singletonList(key),
                     /*retval*/false,
                     filter,
-                    /*singleRmv*/true).get().success();
+                    /*singleRmv*/false).get().success();
             }
 
             @Override public String toString() {

From 4dce965ea86374cba7265cb5d22e975aeac7d480 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 30 Jun 2017 21:36:02 +0300
Subject: [PATCH 346/446] Fixed org.jsr107.tck.PutTest tests.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../internal/processors/cache/GridCacheAdapter.java   | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index cde6309d3f9ff..7c5a54dfcb852 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -642,7 +642,7 @@ public void onKernalStop() {
 
         return (IgniteInternalFuture)getAsync(
             key,
-            /*force primary*/false,
+            /*force primary*/ !ctx.config().isReadFromBackup(),
             /*skip tx*/false,
             /*subj id*/null,
             /*task name*/null,
@@ -669,7 +669,7 @@ public void onKernalStop() {
 
         return getAllAsync(
             keys,
-            /*force primary*/false,
+            /*force primary*/ !ctx.config().isReadFromBackup(),
             /*skip tx*/false,
             /*subj id*/null,
             /*task name*/null,
@@ -1137,7 +1137,7 @@ public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, bool
                     execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
 
                     for (int i = 1; i < jobs.size(); i++)
-                        execSvc.submit(jobs.get(i));
+                        execSvc.execute(jobs.get(i));
                 }
 
                 jobs.get(0).run();
@@ -2715,6 +2715,8 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
 
     /** {@inheritDoc} */
     @Override public void putAll(@Nullable final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        A.notNull(m, "map");
+
         if (F.isEmpty(m))
             return;
 
@@ -3000,6 +3002,7 @@ public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws
 
     /**
      * @param key Key.
+     * @param filter Filter.
      * @return {@code True} if entry was removed.
      * @throws IgniteCheckedException If failed.
      */
@@ -3011,7 +3014,7 @@ protected boolean remove0(final K key, final CacheEntryPredicate filter) throws
                     Collections.singletonList(key),
                     /*retval*/false,
                     filter,
-                    /*singleRmv*/false).get().success();
+                    /*singleRmv*/filter == null).get().success();
             }
 
             @Override public String toString() {

From 50887fed508e03a8b7df32569afb6d84ab3eb670 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 4 Jul 2017 20:01:01 +0300
Subject: [PATCH 347/446] IGNITE-5663: ODBC: Closing cursor do not reset
 prepared statement anymore

---
 .../cpp/odbc-test/src/queries_test.cpp        | 62 +++++++++++++++++++
 modules/platforms/cpp/odbc/src/statement.cpp  |  3 -
 2 files changed, 62 insertions(+), 3 deletions(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 9aca77de784ed..f0e7c76133ed9 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -1754,4 +1754,66 @@ BOOST_AUTO_TEST_CASE(TestParamsNum)
     CheckParamsNum("INSERT INTO TestType(_key, strField) VALUES(?, ?)", 2);
 }
 
+BOOST_AUTO_TEST_CASE(TestExecuteAfterCursorClose)
+{
+    TestType in(1, 2, 3, 4, "5", 6.0f, 7.0, true, Guid(8, 9), BinaryUtils::MakeDateGmt(1987, 6, 5),
+        BinaryUtils::MakeTimestampGmt(1998, 12, 27, 1, 2, 3, 456));
+
+    cache1.Put(1, in);
+
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+
+    int64_t key = 0;
+    char strField[1024] = { 0 };
+    SQLLEN strFieldLen = 0;
+
+    // Binding columns.
+    SQLRETURN ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    // Binding columns.
+    ret = SQLBindCol(stmt, 2, SQL_C_CHAR, &strField, sizeof(strField), &strFieldLen);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    // Just selecting everything to make sure everything is OK
+    SQLCHAR selectReq[] = "SELECT _key, strField FROM TestType";
+
+    ret = SQLPrepare(stmt, selectReq, sizeof(selectReq));
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLExecute(stmt);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLFreeStmt(stmt, SQL_CLOSE);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLExecute(stmt);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLFetch(stmt);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    BOOST_CHECK_EQUAL(key, 1);
+
+    BOOST_CHECK_EQUAL(std::string(strField, strFieldLen), "5");
+
+    ret = SQLFetch(stmt);
+
+    BOOST_CHECK_EQUAL(ret, SQL_NO_DATA);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp
index 6154f9126568e..78b8a1ef8a2da 100644
--- a/modules/platforms/cpp/odbc/src/statement.cpp
+++ b/modules/platforms/cpp/odbc/src/statement.cpp
@@ -684,9 +684,6 @@ namespace ignite
 
             SqlResult result = currentQuery->Close();
 
-            if (result == SQL_RESULT_SUCCESS)
-                currentQuery.reset();
-
             return result;
         }
 

From da290cee855ef45a90ad539515e039f2826a6c00 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Wed, 5 Jul 2017 13:21:12 +0300
Subject: [PATCH 348/446] IGNITE-5663: Fix for test

---
 modules/platforms/cpp/odbc-test/src/queries_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index f0e7c76133ed9..4c0284f978ca9 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -1761,7 +1761,7 @@ BOOST_AUTO_TEST_CASE(TestExecuteAfterCursorClose)
 
     cache1.Put(1, in);
 
-    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
 
     int64_t key = 0;
     char strField[1024] = { 0 };

From 3536a58982e4c264bb72b2ccc1953049d2b5c67f Mon Sep 17 00:00:00 2001
From: Alexey Kukushkin <AlexeyKukushkin@yahoo.com>
Date: Wed, 5 Jul 2017 19:36:41 +0300
Subject: [PATCH 349/446] IGNITE-4901 Decrease logging level for DataStremer
 retry

---
 .../datastreamer/DataStreamProcessor.java     |   4 +-
 .../datastreamer/DataStreamerImpl.java        |  14 +-
 .../DataStreamerImplSelfTest.java             | 121 +++++++++++++++++-
 3 files changed, 131 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index 6f35a52d5b673..dec228d520ec1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -21,10 +21,10 @@
 import java.util.UUID;
 import java.util.concurrent.DelayQueue;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.communication.GridIoManager;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.deployment.GridDeployment;
@@ -336,7 +336,7 @@ private void localUpdate(final UUID nodeId,
                 AffinityTopologyVersion topVer = fut.topologyVersion();
 
                 if (!allowOverwrite && !topVer.equals(req.topologyVersion())) {
-                    Exception err = new IgniteCheckedException(
+                    Exception err = new ClusterTopologyCheckedException(
                         "DataStreamer will retry data transfer at stable topology " +
                             "[reqTop=" + req.topologyVersion() + ", topVer=" + topVer + ", node=remote]");
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index 515314eb08728..8978fbf34dffe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -1774,10 +1774,16 @@ void onResponse(DataStreamerResponse res, UUID nodeId) {
                 try {
                     GridPeerDeployAware jobPda0 = jobPda;
 
-                    err = new IgniteCheckedException("DataStreamer request failed [node=" + nodeId + "]",
-                        (Throwable)U.unmarshal(ctx,
-                            errBytes,
-                            U.resolveClassLoader(jobPda0 != null ? jobPda0.classLoader() : null, ctx.config())));
+                    final Throwable cause = U.unmarshal(
+                        ctx,
+                        errBytes,
+                        U.resolveClassLoader(jobPda0 != null ? jobPda0.classLoader() : null, ctx.config()));
+
+                    final String msg = "DataStreamer request failed [node=" + nodeId + "]";
+
+                    err = cause instanceof ClusterTopologyCheckedException ?
+                        new ClusterTopologyCheckedException(msg, cause) :
+                        new IgniteCheckedException(msg, cause);
                 }
                 catch (IgniteCheckedException e) {
                     f.onDone(null, new IgniteCheckedException("Failed to unmarshal response.", e));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java
index a6a9f5488cefc..b26ccf24526eb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.datastreamer;
 
 import java.io.Serializable;
+import java.io.StringWriter;
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.Callable;
@@ -26,17 +27,29 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.cache.CacheServerNotFoundException;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.log4j.Appender;
+import org.apache.log4j.Logger;
+import org.apache.log4j.SimpleLayout;
+import org.apache.log4j.WriterAppender;
 
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
@@ -51,12 +64,18 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest {
     /** Number of keys to load via data streamer. */
     private static final int KEYS_COUNT = 1000;
 
+    /** Next nodes after MAX_CACHE_COUNT start without cache */
+    private static final int MAX_CACHE_COUNT = 4;
+
     /** Started grid counter. */
     private static int cnt;
 
     /** No nodes filter. */
     private static volatile boolean noNodesFilter;
 
+    /** Indicates whether we need to make the topology stale */
+    private static boolean needStaleTop = false;
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         super.afterTest();
@@ -73,8 +92,9 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest {
 
         cfg.setDiscoverySpi(discoSpi);
 
-        // Forth node goes without cache.
-        if (cnt < 4)
+        cfg.setCommunicationSpi(new StaleTopologyCommunicationSpi());
+
+        if (cnt < MAX_CACHE_COUNT)
             cfg.setCacheConfiguration(cacheConfiguration());
 
         cnt++;
@@ -232,6 +252,44 @@ public void testNoDataNodesOnFlush() throws Exception {
         }
     }
 
+    /**
+     * Cluster topology mismatch shall result in DataStreamer retrying cache update with the latest topology and
+     * no error logged to the console.
+     *
+     * @throws Exception if failed
+     */
+    public void testRetryWhenTopologyMismatch() throws Exception {
+        final int KEY = 1;
+        final String VAL = "1";
+
+        cnt = 0;
+
+        StringWriter logWriter = new StringWriter();
+        Appender logAppender = new WriterAppender(new SimpleLayout(), logWriter);
+
+        Logger.getRootLogger().addAppender(logAppender);
+
+        startGrids(MAX_CACHE_COUNT - 1); // cache-enabled nodes
+
+        try (Ignite ignite = startGrid(MAX_CACHE_COUNT);
+             IgniteDataStreamer<Integer, String> streamer = ignite.dataStreamer(null)) {
+
+            needStaleTop = true; // simulate stale topology for the next action
+
+            streamer.addData(KEY, VAL);
+        } finally {
+            needStaleTop = false;
+
+            logWriter.flush();
+
+            Logger.getRootLogger().removeAppender(logAppender);
+
+            logAppender.close();
+        }
+
+        assertFalse(logWriter.toString().contains("DataStreamer will retry data transfer at stable topology"));
+    }
+
     /**
      * Gets cache configuration.
      *
@@ -284,4 +342,63 @@ public Integer val() {
             return obj instanceof TestObject && ((TestObject)obj).val == val;
         }
     }
+
+    /**
+     * Simulate stale (not up-to-date) topology
+     */
+    private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+            // Send stale topology only in the first request to avoid indefinitely getting failures.
+            if (needStaleTop) {
+                if (msg instanceof GridIoMessage) {
+                    GridIoMessage ioMsg = (GridIoMessage)msg;
+
+                    Message appMsg = ioMsg.message();
+
+                    if (appMsg != null && appMsg instanceof DataStreamerRequest) {
+                        DataStreamerRequest req = (DataStreamerRequest)appMsg;
+
+                        AffinityTopologyVersion validTop = req.topologyVersion();
+
+                        // Simulate situation when a node did not receive the latest "node joined" topology update causing
+                        // topology mismatch
+                        AffinityTopologyVersion staleTop = new AffinityTopologyVersion(
+                            validTop.topologyVersion() - 1,
+                            validTop.minorTopologyVersion());
+
+                        appMsg = new DataStreamerRequest(
+                            req.requestId(),
+                            req.responseTopicBytes(),
+                            req.cacheName(),
+                            req.updaterBytes(),
+                            req.entries(),
+                            req.ignoreDeploymentOwnership(),
+                            req.skipStore(),
+                            req.keepBinary(),
+                            req.deploymentMode(),
+                            req.sampleClassName(),
+                            req.userVersion(),
+                            req.participants(),
+                            req.classLoaderId(),
+                            req.forceLocalDeployment(),
+                            staleTop);
+
+                        msg = new GridIoMessage(
+                            GridTestUtils.<Byte>getFieldValue(ioMsg, "plc"),
+                            GridTestUtils.getFieldValue(ioMsg, "topic"),
+                            GridTestUtils.<Integer>getFieldValue(ioMsg, "topicOrd"),
+                            appMsg,
+                            GridTestUtils.<Boolean>getFieldValue(ioMsg, "ordered"),
+                            ioMsg.timeout(),
+                            ioMsg.skipOnTimeout());
+
+                        needStaleTop = false;
+                    }
+                }
+            }
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
 }
\ No newline at end of file

From 6d3a3ff2d99697882232070e715928336a9180cd Mon Sep 17 00:00:00 2001
From: Alexey Kukushkin <AlexeyKukushkin@yahoo.com>
Date: Wed, 5 Jul 2017 20:05:02 +0300
Subject: [PATCH 350/446] Fixed merge conflicts

---
 .../cache/GridCachePartitionExchangeManager.java    |  2 +-
 .../spi/communication/tcp/TcpCommunicationSpi.java  | 13 -------------
 2 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 8ddbb59b540f3..c62ffd2aec79c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -269,7 +269,7 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                                 exchFut = exchangeFuture(exchId, evt, cache, null, msg);
                             }
                         }
-                        elseif (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                        else if (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
                             exchangeFuture(msg.exchangeId(), null, null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
                     }
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index f96cd1d79f538..0d80f4778df36 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3504,19 +3504,6 @@ private static class HandshakeException extends IgniteCheckedException {
         }
     }
 
-    /** Internal exception class for proper timeout handling. */
-    private static class HandshakeException extends IgniteCheckedException {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /**
-         * @param msg Error message.
-         */
-        HandshakeException(String msg) {
-            super(msg);
-        }
-    }
-
     /** Internal exception class for proper timeout handling. */
     private static class HandshakeTimeoutException extends IgniteCheckedException {
         /** */

From 355a5283559c885f57c4557bba2c6d9170a9b5fc Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Fri, 30 Jun 2017 20:23:55 +0300
Subject: [PATCH 351/446] IGNITE-5554 ServiceProcessor may process failed
 reassignments in timeout thread

---
 .../service/GridServiceProcessor.java         | 21 +++++++++----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 20bcff7c780d3..c452da31086d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1581,16 +1581,11 @@ private void onDeployment(final GridServiceDeployment dep, final AffinityTopolog
                 }
 
                 @Override public void onTimeout() {
-                    if (!busyLock.enterBusy())
-                        return;
-
-                    try {
-                        // Try again.
-                        onDeployment(dep, topVer);
-                    }
-                    finally {
-                        busyLock.leaveBusy();
-                    }
+                    depExe.execute(new DepRunnable() {
+                        @Override public void run0() {
+                            onDeployment(dep, topVer);
+                        }
+                    });
                 }
             });
         }
@@ -1777,7 +1772,11 @@ private void onReassignmentFailed(final AffinityTopologyVersion topVer,
                         }
 
                         @Override public void onTimeout() {
-                            onReassignmentFailed(topVer, retries);
+                            depExe.execute(new Runnable() {
+                                public void run() {
+                                    onReassignmentFailed(topVer, retries);
+                                }
+                            });
                         }
                     });
                 }

From 92aa7c6e3c0d9b5cc68002433861b175d54f9421 Mon Sep 17 00:00:00 2001
From: agura <agura@gridgain.com>
Date: Tue, 4 Jul 2017 16:56:40 +0300
Subject: [PATCH 352/446] ignite-5685 JDBC prepared statement shouldn't clear
 parameters after execution

---
 .../jdbc2/JdbcPreparedStatementSelfTest.java  | 35 +++++++++++++++++++
 .../jdbc/JdbcPreparedStatementSelfTest.java   | 35 +++++++++++++++++++
 .../internal/jdbc/JdbcPreparedStatement.java  |  6 +---
 .../internal/jdbc2/JdbcPreparedStatement.java |  6 ++--
 4 files changed, 73 insertions(+), 9 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
index ea586b297a579..56c248817e12a 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
@@ -137,6 +137,41 @@ public class JdbcPreparedStatementSelfTest extends GridCommonAbstractTest {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRepeatableUsage() throws Exception {
+        stmt = conn.prepareStatement("select * from TestObject where id = ?");
+
+        stmt.setInt(1, 1);
+
+        ResultSet rs = stmt.executeQuery();
+
+        int cnt = 0;
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assertEquals(1, rs.getInt(1));
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+
+        cnt = 0;
+
+        rs = stmt.executeQuery();
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assertEquals(1, rs.getInt(1));
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java
index 0c701ef3e7ce3..cd8648833c3d1 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java
@@ -155,6 +155,41 @@ public class JdbcPreparedStatementSelfTest extends GridCommonAbstractTest {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRepeatableUsage() throws Exception {
+        stmt = conn.prepareStatement("select * from TestObject where id = ?");
+
+        stmt.setInt(1, 1);
+
+        ResultSet rs = stmt.executeQuery();
+
+        int cnt = 0;
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assertEquals(1, rs.getInt(1));
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+
+        cnt = 0;
+
+        rs = stmt.executeQuery();
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assertEquals(1, rs.getInt(1));
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java
index 7e5358becab93..93cda1e4768cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java
@@ -69,11 +69,7 @@ public class JdbcPreparedStatement extends JdbcStatement implements PreparedStat
 
     /** {@inheritDoc} */
     @Override public ResultSet executeQuery() throws SQLException {
-        ResultSet rs = executeQuery(sql);
-
-        args = null;
-
-        return rs;
+        return executeQuery(sql);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
index a99f24c8633d7..72375cd939a88 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
@@ -50,11 +50,9 @@ public class JdbcPreparedStatement extends JdbcStatement implements PreparedStat
 
     /** {@inheritDoc} */
     @Override public ResultSet executeQuery() throws SQLException {
-        ResultSet rs = executeQuery(sql);
-
-        args = null;
+        ensureNotClosed();
 
-        return rs;
+        return executeQuery(sql);
     }
 
     /** {@inheritDoc} */

From 9165a0f93b5173b543cc6b4fad5fde37bd215f91 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 7 Jul 2017 15:35:33 +0300
Subject: [PATCH 353/446] ignite-5562: assert statements were changed to the
 'if' blocks

---
 .../spi/discovery/tcp/internal/TcpDiscoveryStatistics.java  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
index a69dbd9aa89aa..42d2ae6f8617e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
@@ -437,7 +437,8 @@ public synchronized String maxProcessingTimeMessageClass() {
      * @param initTime Time socket was initialized in.
      */
     public synchronized void onServerSocketInitialized(long initTime) {
-        assert initTime >= 0;
+        if (initTime < 0)
+            initTime = 0;
 
         if (maxSrvSockInitTime < initTime)
             maxSrvSockInitTime = initTime;
@@ -449,7 +450,8 @@ public synchronized void onServerSocketInitialized(long initTime) {
      * @param initTime Time socket was initialized in.
      */
     public synchronized void onClientSocketInitialized(long initTime) {
-        assert initTime >= 0;
+        if (initTime < 0)
+            initTime = 0;
 
         clientSockCreatedCnt++;
 

From d9fc20a61d5ac0a6e63b26faa7fa0af753b2fa06 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dgovorukhin@gridgain.com>
Date: Fri, 7 Apr 2017 14:28:22 +0300
Subject: [PATCH 354/446] IGNITE-4889 - Changed Hibernate integration to use
 custom keys

(cherry picked from commit 6b62a20)
---
 .../HibernateAccessStrategyAdapter.java       |   8 +-
 .../cache/hibernate/HibernateCacheProxy.java  | 818 ++++++++++++++++++
 .../hibernate/HibernateCollectionRegion.java  |   3 +-
 .../hibernate/HibernateEntityRegion.java      |   3 +-
 .../hibernate/HibernateGeneralDataRegion.java |   3 +-
 .../hibernate/HibernateKeyTransformer.java    |  29 +
 .../cache/hibernate/HibernateKeyWrapper.java  |  72 ++
 .../hibernate/HibernateNaturalIdRegion.java   |   3 +-
 .../HibernateNonStrictAccessStrategy.java     |   5 +-
 .../HibernateQueryResultsRegion.java          |   3 +-
 .../HibernateReadOnlyAccessStrategy.java      |   3 +-
 .../HibernateReadWriteAccessStrategy.java     |   3 +-
 .../cache/hibernate/HibernateRegion.java      |   6 +-
 .../hibernate/HibernateRegionFactory.java     |  29 +-
 .../hibernate/HibernateTimestampsRegion.java  |   3 +-
 .../HibernateTransactionalAccessStrategy.java |   2 +-
 .../HibernateTransactionalDataRegion.java     |   3 +-
 .../IgniteBinaryHibernateTestSuite.java       |  37 +
 18 files changed, 1000 insertions(+), 33 deletions(-)
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
 create mode 100644 modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java

diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
index 27734d9ad9ca5..f6c1d0e15eb43 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
@@ -92,7 +92,7 @@
  */
 public abstract class HibernateAccessStrategyAdapter {
     /** */
-    protected final IgniteInternalCache<Object, Object> cache;
+    protected final HibernateCacheProxy cache;
 
     /** Grid. */
     protected final Ignite ignite;
@@ -104,7 +104,7 @@ public abstract class HibernateAccessStrategyAdapter {
      * @param ignite Grid.
      * @param cache Cache.
      */
-    protected HibernateAccessStrategyAdapter(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    protected HibernateAccessStrategyAdapter(Ignite ignite, HibernateCacheProxy cache) {
         this.cache = cache;
         this.ignite = ignite;
 
@@ -292,8 +292,10 @@ protected final void removeAll() throws CacheException {
      * @param key Key.
      * @throws CacheException If failed.
      */
-    static void evict(Ignite ignite, IgniteInternalCache<Object,Object> cache, Object key) throws CacheException {
+    static void evict(Ignite ignite, HibernateCacheProxy cache, Object key) throws CacheException {
         try {
+            key = cache.keyTransformer().transform(key);
+
             ignite.compute(ignite.cluster()).call(new ClearKeyCallable(key, cache.name()));
         }
         catch (IgniteException e) {
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
new file mode 100644
index 0000000000000..871c4a100ac6f
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
@@ -0,0 +1,818 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorResult;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheEntry;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.mxbean.CacheMetricsMXBean;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Hibernate cache proxy.
+ */
+public class HibernateCacheProxy implements IgniteInternalCache<Object, Object> {
+    /** Delegate. */
+    private final IgniteInternalCache<Object, Object> delegate;
+
+    /** Transformer. */
+    private final HibernateKeyTransformer keyTransformer;
+
+    /**
+     * @param delegate Delegate.
+     * @param keyTransformer Key keyTransformer.
+     */
+    HibernateCacheProxy(
+        IgniteInternalCache<Object, Object> delegate,
+        HibernateKeyTransformer keyTransformer
+    ) {
+        assert delegate != null;
+        assert keyTransformer != null;
+
+        this.delegate = delegate;
+        this.keyTransformer = keyTransformer;
+    }
+
+    /**
+     * @return HibernateKeyTransformer
+     */
+    HibernateKeyTransformer keyTransformer(){
+        return keyTransformer;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String name() {
+        return delegate.name();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean skipStore() {
+        return delegate.skipStore();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache setSkipStore(boolean skipStore) {
+        return delegate.setSkipStore(skipStore);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean containsKey(Object key) {
+        return delegate.containsKey(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> containsKeyAsync(Object key) {
+        return delegate.containsKeyAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void promoteAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.promoteAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public long overflowSize() throws IgniteCheckedException {
+        return delegate.overflowSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long swapSize() throws IgniteCheckedException {
+        return delegate.swapSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long swapKeys() throws IgniteCheckedException {
+        return delegate.swapKeys();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean containsKeys(Collection keys) {
+        return delegate.containsKey(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> containsKeysAsync(Collection keys) {
+        return delegate.containsKeysAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object localPeek(
+        Object key,
+        CachePeekMode[] peekModes,
+        @Nullable IgniteCacheExpiryPolicy plc
+    ) throws IgniteCheckedException {
+        return delegate.localPeek(keyTransformer.transform(key), peekModes, plc);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable<Cache.Entry<Object, Object>> localEntries(
+        CachePeekMode[] peekModes
+    ) throws IgniteCheckedException {
+        return delegate.localEntries(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object get(Object key) throws IgniteCheckedException {
+        return delegate.get(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public CacheEntry getEntry(Object key) throws IgniteCheckedException {
+        return delegate.getEntry(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAsync(Object key) {
+        return delegate.getAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<CacheEntry<Object, Object>> getEntryAsync(Object key) {
+        return delegate.getEntryAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map getAll(@Nullable Collection keys) throws IgniteCheckedException {
+        return delegate.getAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<CacheEntry<Object, Object>> getEntries(
+        @Nullable Collection keys) throws IgniteCheckedException {
+        return delegate.getEntries(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map<Object, Object>> getAllAsync(@Nullable Collection keys) {
+        return delegate.getAllAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Collection<CacheEntry<Object,Object>>> getEntriesAsync(
+        @Nullable Collection keys
+    ) {
+        return delegate.getEntriesAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndPut(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndPut(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndPutAsync(Object key, Object val) {
+        return delegate.getAndPutAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean put(Object key, Object val) throws IgniteCheckedException {
+        return delegate.put(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> putAsync(Object key, Object val) {
+        return delegate.putAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndPutIfAbsent(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndPutIfAbsent(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndPutIfAbsentAsync(Object key, Object val) {
+        return delegate.getAndPutIfAbsentAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean putIfAbsent(Object key, Object val) throws IgniteCheckedException {
+        return delegate.putIfAbsent(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(Object key, Object val) {
+        return delegate.putIfAbsentAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndReplace(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndReplace(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndReplaceAsync(Object key, Object val) {
+        return delegate.getAndReplaceAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean replace(Object key, Object val) throws IgniteCheckedException {
+        return delegate.replace(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> replaceAsync(Object key, Object val) {
+        return delegate.replaceAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean replace(Object key, Object oldVal, Object newVal) throws IgniteCheckedException {
+        return delegate.replace(keyTransformer.transform(key), oldVal, newVal);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> replaceAsync(Object key, Object oldVal, Object newVal) {
+        return delegate.replaceAsync(keyTransformer.transform(key), oldVal, newVal);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void putAll(@Nullable Map m) throws IgniteCheckedException {
+        delegate.putAll(transform(m));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> putAllAsync(@Nullable Map m) {
+        return delegate.putAllAsync(transform(m));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set keySet() {
+        return delegate.keySet();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set keySetx() {
+        return delegate.keySetx();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set primaryKeySet() {
+        return delegate.primaryKeySet();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable values() {
+        return delegate.values();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set<Cache.Entry<Object, Object>> entrySet() {
+        return delegate.entrySet();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Set<Cache.Entry<Object,Object>> entrySet(int part) {
+        return delegate.entrySet(part);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set<Cache.Entry<Object, Object>> entrySetx(CacheEntryPredicate... filter) {
+        return delegate.entrySetx(filter);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Transaction txStart(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation
+    ) {
+        return delegate.txStart(concurrency, isolation);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalTx txStartEx(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation
+    ) {
+        return delegate.txStartEx(concurrency, isolation);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Transaction txStart(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation,
+        long timeout,
+        int txSize
+    ) {
+        return delegate.txStart(concurrency, isolation, timeout, txSize);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Transaction tx() {
+        return delegate.tx();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean evict(Object key) {
+        return delegate.evict(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void evictAll(@Nullable Collection keys) {
+        delegate.evictAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearLocally(boolean srv, boolean near, boolean readers) {
+        delegate.clearLocally(srv, near, readers);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean clearLocally(Object key) {
+        return delegate.clearLocally(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearLocallyAll(Set keys, boolean srv, boolean near, boolean readers) {
+        delegate.clearLocallyAll((Set<?>)transform(keys), srv, near, readers);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear(Object key) throws IgniteCheckedException {
+        delegate.clear(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearAll(Set keys) throws IgniteCheckedException {
+        delegate.clearAll((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear() throws IgniteCheckedException {
+        delegate.clear();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAsync() {
+        return delegate.clearAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAsync(Object key) {
+        return delegate.clearAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAllAsync(Set keys) {
+        return delegate.clearAllAsync((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndRemove(Object key) throws IgniteCheckedException {
+        return delegate.getAndRemove(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndRemoveAsync(Object key) {
+        return delegate.getAndRemoveAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(Object key) throws IgniteCheckedException {
+        return delegate.remove(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> removeAsync(Object key) {
+        return delegate.removeAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(Object key, Object val) throws IgniteCheckedException {
+        return delegate.remove(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> removeAsync(Object key, Object val) {
+        return delegate.removeAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.removeAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllAsync(@Nullable Collection keys) {
+        return delegate.removeAllAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAll() throws IgniteCheckedException {
+        delegate.removeAll();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllAsync() {
+        return delegate.removeAllAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lock(Object key, long timeout) throws IgniteCheckedException {
+        return delegate.lock(keyTransformer.transform(key), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> lockAsync(Object key, long timeout) {
+        return delegate.lockAsync(keyTransformer.transform(key), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lockAll(@Nullable Collection keys, long timeout) throws IgniteCheckedException {
+        return delegate.lockAll(transform(keys), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> lockAllAsync(@Nullable Collection keys, long timeout) {
+        return delegate.lockAllAsync(transform(keys), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlock(Object key) throws IgniteCheckedException {
+        delegate.unlock(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlockAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.unlockAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLocked(Object key) {
+        return delegate.isLocked(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLockedByThread(Object key) {
+        return delegate.isLockedByThread(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size() {
+        return delegate.size();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong() {
+        return delegate.sizeLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int localSize(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSize(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long localSizeLong(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSizeLong(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long localSizeLong(int partition, CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSizeLong(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.size(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.sizeLong(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong(int partition, CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.sizeLong(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Integer> sizeAsync(CachePeekMode[] peekModes) {
+        return delegate.sizeAsync(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Long> sizeLongAsync(CachePeekMode[] peekModes) {
+        return delegate.sizeLongAsync(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Long> sizeLongAsync(int partition, CachePeekMode[] peekModes) {
+        return delegate.sizeLongAsync(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int nearSize() {
+        return delegate.nearSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int primarySize() {
+        return delegate.primarySize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long primarySizeLong() {
+        return delegate.primarySizeLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheConfiguration configuration() {
+        return delegate.configuration();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Affinity affinity() {
+        return delegate.affinity();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics clusterMetrics() {
+        return delegate.clusterMetrics();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics clusterMetrics(ClusterGroup grp) {
+        return delegate.clusterMetrics(grp);
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics localMetrics() {
+        return delegate.localMetrics();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetricsMXBean clusterMxBean() {
+        return delegate.clusterMxBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetricsMXBean localMxBean() {
+        return delegate.localMxBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long offHeapEntriesCount() {
+        return delegate.offHeapEntriesCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long offHeapAllocatedSize() {
+        return delegate.offHeapAllocatedSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> rebalance() {
+        return delegate.rebalance();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache forSubjectId(UUID subjId) {
+        return delegate.forSubjectId(subjId);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getForcePrimary(Object key) throws IgniteCheckedException {
+        return delegate.getForcePrimary(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getForcePrimaryAsync(Object key) {
+        return delegate.getForcePrimaryAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map getAllOutTx(Set keys) throws IgniteCheckedException {
+        return delegate.getAllOutTx((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map<Object, Object>> getAllOutTxAsync(Set keys) {
+        return delegate.getAllOutTxAsync((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isIgfsDataCache() {
+        return delegate.isIgfsDataCache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long igfsDataSpaceUsed() {
+        return delegate.igfsDataSpaceUsed();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long igfsDataSpaceMax() {
+        return delegate.igfsDataSpaceMax();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMongoDataCache() {
+        return delegate.isMongoDataCache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMongoMetaCache() {
+        return delegate.isMongoMetaCache();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public ExpiryPolicy expiry() {
+        return delegate.expiry();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache withExpiryPolicy(ExpiryPolicy plc) {
+        return delegate.withExpiryPolicy(plc);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache withNoRetries() {
+        return delegate.withNoRetries();
+    }
+
+    /** {@inheritDoc} */
+    @Override public GridCacheContext context() {
+        return delegate.context();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void localLoadCache(
+        @Nullable IgniteBiPredicate p,
+        @Nullable Object... args
+    ) throws IgniteCheckedException {
+        delegate.localLoadCache(p, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> localLoadCacheAsync(
+        @Nullable IgniteBiPredicate p,
+        @Nullable Object... args
+    ) {
+        return delegate.localLoadCacheAsync(p, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Object getTopologySafe(Object key) throws IgniteCheckedException {
+        return delegate.getTopologySafe(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object tryGetAndPut(Object key, Object val) throws IgniteCheckedException {
+        return delegate.tryGetAndPut(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryProcessorResult invoke(
+        @Nullable AffinityTopologyVersion topVer,
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) throws IgniteCheckedException {
+        return delegate.invoke(topVer, key, entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map> invokeAllAsync(Map map, Object... args) {
+        return delegate.invokeAllAsync(map, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map invokeAll(Map map, Object... args) throws IgniteCheckedException {
+        return delegate.invokeAll(map, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map> invokeAllAsync(Set keys, EntryProcessor entryProcessor, Object... args) {
+        return delegate.invokeAllAsync((Set<?>)transform(keys), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map invokeAll(Set keys, EntryProcessor entryProcessor, Object... args) throws IgniteCheckedException {
+        return delegate.invokeAll((Set<?>)transform(keys), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<EntryProcessorResult> invokeAsync(
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) {
+        return delegate.invokeAsync(keyTransformer.transform(key), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryProcessorResult invoke(
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) throws IgniteCheckedException {
+        return delegate.invoke(keyTransformer.transform(key), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllConflictAsync(Map drMap) throws IgniteCheckedException {
+        return delegate.removeAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAllConflict(Map drMap) throws IgniteCheckedException {
+        delegate.removeAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> putAllConflictAsync(Map drMap) throws IgniteCheckedException {
+        return delegate.putAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void putAllConflict(Map drMap) throws IgniteCheckedException {
+        delegate.putAllConflict(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache keepBinary() {
+        return delegate.keepBinary();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache cache() {
+        return delegate.cache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterator iterator() {
+        return delegate.iterator();
+    }
+
+    /**
+     * @param keys Keys.
+     */
+    private Collection<Object> transform(Collection<Object> keys) {
+        Collection<Object> res = new LinkedList<>();
+
+        for (Object o : keys)
+            res.add(keyTransformer.transform(o));
+
+        return res;
+    }
+
+    /**
+     * @param map Map.
+     */
+    private Map<Object, Object> transform(Map<Object, Object> map) {
+        Map<Object, Object> res = new HashMap<>();
+
+        Set<Map.Entry<Object, Object>> ents = map.entrySet();
+
+        for (Map.Entry<Object, Object> e : ents)
+            res.put(keyTransformer.transform(e.getKey()), e.getValue());
+
+        return res;
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
index 045f401d92f80..eb35a2c4709d7 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.CollectionRegion;
@@ -72,7 +71,7 @@ public class HibernateCollectionRegion extends HibernateTransactionalDataRegion
      * @param dataDesc Region data description.
      */
     public HibernateCollectionRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
index 1ceda144a8355..ad5b1919f51b3 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.EntityRegion;
@@ -62,7 +61,7 @@ public class HibernateEntityRegion extends HibernateTransactionalDataRegion impl
      * @param dataDesc Region data description.
      */
     public HibernateEntityRegion(HibernateRegionFactory factory, String name, Ignite ignite,
-        IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
index fbac624f66af1..2f1a11dc8baa3 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.GeneralDataRegion;
 import org.hibernate.cache.spi.QueryResultsRegion;
@@ -38,7 +37,7 @@ public class HibernateGeneralDataRegion extends HibernateRegion implements Gener
      * @param cache Region cache.
      */
     public HibernateGeneralDataRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite, HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
new file mode 100644
index 0000000000000..97fc0e9eb15e6
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+/**
+ * An interface for transforming hibernate keys to Ignite keys.
+ */
+public interface HibernateKeyTransformer {
+    /**
+     * @param key Hibernate key.
+     * @return Transformed key.
+     */
+    public Object transform(Object key);
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
new file mode 100644
index 0000000000000..7de440ebb032c
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Hibernate cache key wrapper.
+ */
+public class HibernateKeyWrapper {
+    /** Key. */
+    private final Object key;
+
+    /** Entry. */
+    private final String entry;
+
+    /** */
+    private final String tenantId;
+
+    /**
+     * @param key Key.
+     * @param entry Entry.
+     * @param tenantId Tenant ID.
+     */
+    HibernateKeyWrapper(Object key, String entry, String tenantId) {
+        this.key = key;
+        this.entry = entry;
+        this.tenantId = tenantId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o) return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        HibernateKeyWrapper that = (HibernateKeyWrapper) o;
+
+        return (key != null ? key.equals(that.key) : that.key == null) &&
+            (entry != null ? entry.equals(that.entry) : that.entry == null) &&
+            (tenantId != null ? tenantId.equals(that.tenantId) : that.tenantId == null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = key != null ? key.hashCode() : 0;
+        res = 31 * res + (entry != null ? entry.hashCode() : 0);
+        res = 31 * res + (tenantId != null ? tenantId.hashCode() : 0);
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HibernateKeyWrapper.class, this);
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
index 99d5348dea479..862a4228f2767 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.NaturalIdRegion;
@@ -52,7 +51,7 @@ public class HibernateNaturalIdRegion extends HibernateTransactionalDataRegion i
      * @param dataDesc Region data description.
      */
     public HibernateNaturalIdRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
index 1cb8d48e0be2f..a36d7e786d739 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
@@ -21,7 +21,6 @@
 import java.util.Set;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridLeanMap;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.typedef.F;
@@ -67,7 +66,7 @@ public class HibernateNonStrictAccessStrategy extends HibernateAccessStrategyAda
      * @param cache Cache.
      * @param writeCtx Thread local instance used to track updates done during one Hibernate transaction.
      */
-    protected HibernateNonStrictAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache, ThreadLocal writeCtx) {
+    protected HibernateNonStrictAccessStrategy(Ignite ignite, HibernateCacheProxy cache, ThreadLocal writeCtx) {
         super(ignite, cache);
 
         this.writeCtx = (ThreadLocal<WriteContext>)writeCtx;
@@ -212,7 +211,7 @@ void removed(Object key) {
          * @param cache Cache.
          * @throws IgniteCheckedException If failed.
          */
-        void updateCache(IgniteInternalCache<Object, Object> cache) throws IgniteCheckedException {
+        void updateCache(HibernateCacheProxy cache) throws IgniteCheckedException {
             if (!F.isEmpty(rmvs))
                 cache.removeAll(rmvs);
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
index e3303a71ff413..0b9a43d6a13d6 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.Query;
 import org.hibernate.cache.spi.QueryResultsRegion;
 
@@ -65,7 +64,7 @@ public class HibernateQueryResultsRegion extends HibernateGeneralDataRegion impl
      * @param cache Region cache.
      */
     public HibernateQueryResultsRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite, HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
index 58a2c4b8779d4..cdef80e0a0c0b 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.access.AccessType;
 import org.hibernate.cache.spi.access.SoftLock;
@@ -60,7 +59,7 @@ public class HibernateReadOnlyAccessStrategy extends HibernateAccessStrategyAdap
      * @param ignite Grid.
      * @param cache Cache.
      */
-    public HibernateReadOnlyAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    public HibernateReadOnlyAccessStrategy(Ignite ignite, HibernateCacheProxy cache) {
         super(ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
index bbb1d4e5dfdc2..625b05061ff0f 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
@@ -21,7 +21,6 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.transactions.Transaction;
 import org.hibernate.cache.CacheException;
@@ -68,7 +67,7 @@ public class HibernateReadWriteAccessStrategy extends HibernateAccessStrategyAda
      * @param cache Cache.
      * @param txCtx Thread local instance used to track updates done during one Hibernate transaction.
      */
-    protected HibernateReadWriteAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache, ThreadLocal txCtx) {
+    protected HibernateReadWriteAccessStrategy(Ignite ignite, HibernateCacheProxy cache, ThreadLocal txCtx) {
         super(ignite, cache);
 
         this.txCtx = (ThreadLocal<TxContext>)txCtx;
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
index 27479e90d8b7b..11a96d09096fb 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
@@ -20,7 +20,6 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.Region;
 
@@ -35,7 +34,7 @@ public class HibernateRegion implements Region {
     private final String name;
 
     /** Cache instance. */
-    protected final IgniteInternalCache<Object, Object> cache;
+    protected final HibernateCacheProxy cache;
 
     /** Grid instance. */
     protected Ignite ignite;
@@ -46,8 +45,7 @@ public class HibernateRegion implements Region {
      * @param ignite Grid.
      * @param cache Region cache.
      */
-    public HibernateRegion(HibernateRegionFactory factory, String name, Ignite ignite,
-        IgniteInternalCache<Object, Object> cache) {
+    public HibernateRegion(HibernateRegionFactory factory, String name, Ignite ignite, HibernateCacheProxy cache) {
         this.factory = factory;
         this.name = name;
         this.ignite = ignite;
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
index 825abee7e2590..4e4be36ec7c49 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.util.typedef.G;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
+import org.hibernate.cache.spi.CacheKey;
 import org.hibernate.cache.spi.CollectionRegion;
 import org.hibernate.cache.spi.EntityRegion;
 import org.hibernate.cache.spi.NaturalIdRegion;
@@ -88,7 +89,7 @@ public class HibernateRegionFactory implements RegionFactory {
     private Ignite ignite;
 
     /** Default cache. */
-    private IgniteInternalCache<Object, Object> dfltCache;
+    private HibernateCacheProxy dfltCache;
 
     /** Default region access type. */
     private AccessType dfltAccessType;
@@ -99,6 +100,23 @@ public class HibernateRegionFactory implements RegionFactory {
     /** Map needed to provide the same transaction context for different regions. */
     private final ThreadLocal threadLoc = new ThreadLocal();
 
+    /** Key transformer. */
+    private final HibernateKeyTransformer hibernate4transformer = new HibernateKeyTransformer() {
+        @Override public Object transform(Object key) {
+            if (key instanceof CacheKey) {
+                CacheKey cacheKey = (CacheKey)key;
+
+                return new HibernateKeyWrapper(
+                    cacheKey.getKey(),
+                    cacheKey.getEntityOrRoleName(),
+                    cacheKey.getTenantId()
+                );
+            }
+
+            return key;
+        }
+    };
+
     /** {@inheritDoc} */
     @Override public void start(Settings settings, Properties props) throws CacheException {
         String gridCfg = props.getProperty(GRID_CONFIG_PROPERTY);
@@ -138,10 +156,12 @@ public class HibernateRegionFactory implements RegionFactory {
         String dfltCacheName = props.getProperty(DFLT_CACHE_NAME_PROPERTY);
 
         if (dfltCacheName != null) {
-            dfltCache = ((IgniteKernal)ignite).getCache(dfltCacheName);
+            IgniteInternalCache<Object, Object> dfltCache = ((IgniteKernal)ignite).getCache(dfltCacheName);
 
             if (dfltCache == null)
                 throw new CacheException("Cache specified as default is not configured: " + dfltCacheName);
+
+            this.dfltCache = new HibernateCacheProxy(dfltCache, hibernate4transformer);
         }
 
         IgniteLogger log = ignite.log().getLogger(HibernateRegionFactory.class);
@@ -152,6 +172,7 @@ public class HibernateRegionFactory implements RegionFactory {
 
     /** {@inheritDoc} */
     @Override public void stop() {
+        // No-op.
     }
 
     /** {@inheritDoc} */
@@ -213,7 +234,7 @@ ThreadLocal threadLocalForCache(String cacheName) {
      * @return Cache for given region.
      * @throws CacheException If cache for given region is not configured.
      */
-    private IgniteInternalCache<Object, Object> regionCache(String regionName) throws CacheException {
+    private HibernateCacheProxy regionCache(String regionName) throws CacheException {
         String cacheName = regionCaches.get(regionName);
 
         if (cacheName == null) {
@@ -228,6 +249,6 @@ private IgniteInternalCache<Object, Object> regionCache(String regionName) throw
         if (cache == null)
             throw new CacheException("Cache '" + cacheName + "' for region '" + regionName + "' is not configured.");
 
-        return cache;
+        return new HibernateCacheProxy(cache, hibernate4transformer);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
index 4cedae2070e31..8b4c243277ec9 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.spi.TimestampsRegion;
 
 /**
@@ -34,7 +33,7 @@ public class HibernateTimestampsRegion extends HibernateGeneralDataRegion implem
      * @param cache Region cache.
      */
     public HibernateTimestampsRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite,  HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
index 80f75a71afed6..ca5284917aa68 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
@@ -61,7 +61,7 @@ public class HibernateTransactionalAccessStrategy extends HibernateAccessStrateg
      * @param ignite Grid.
      * @param cache Cache.
      */
-    public HibernateTransactionalAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    public HibernateTransactionalAccessStrategy(Ignite ignite, HibernateCacheProxy cache) {
         super(ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
index ed2ee01cf2280..581076a3fdbb6 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.TransactionConfiguration;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.CollectionRegion;
@@ -48,7 +47,7 @@ public class HibernateTransactionalDataRegion extends HibernateRegion implements
      * @param dataDesc Region data description.
      */
     public HibernateTransactionalDataRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache);
 
         this.dataDesc = dataDesc;
diff --git a/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java b/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java
new file mode 100644
index 0000000000000..3791baed9c93c
--- /dev/null
+++ b/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.testframework.config.GridTestProperties;
+
+/**
+ *
+ */
+public class IgniteBinaryHibernateTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception If failed.
+     */
+    public static TestSuite suite() throws Exception {
+        GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, BinaryMarshaller.class.getName());
+
+        return IgniteHibernateTestSuite.suite();
+    }
+}

From 16067300c9124b79bfee42139eb881ae585c0914 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dgovorukhin@gridgain.com>
Date: Fri, 7 Apr 2017 14:28:22 +0300
Subject: [PATCH 355/446] IGNITE-4889 - Changed Hibernate integration to use
 custom keys

(cherry picked from commit 6b62a20)
---
 .../HibernateAccessStrategyAdapter.java       |   8 +-
 .../cache/hibernate/HibernateCacheProxy.java  | 818 ++++++++++++++++++
 .../hibernate/HibernateCollectionRegion.java  |   3 +-
 .../hibernate/HibernateEntityRegion.java      |   3 +-
 .../hibernate/HibernateGeneralDataRegion.java |   3 +-
 .../hibernate/HibernateKeyTransformer.java    |  29 +
 .../cache/hibernate/HibernateKeyWrapper.java  |  72 ++
 .../hibernate/HibernateNaturalIdRegion.java   |   3 +-
 .../HibernateNonStrictAccessStrategy.java     |   5 +-
 .../HibernateQueryResultsRegion.java          |   3 +-
 .../HibernateReadOnlyAccessStrategy.java      |   3 +-
 .../HibernateReadWriteAccessStrategy.java     |   3 +-
 .../cache/hibernate/HibernateRegion.java      |   6 +-
 .../hibernate/HibernateRegionFactory.java     |  29 +-
 .../hibernate/HibernateTimestampsRegion.java  |   3 +-
 .../HibernateTransactionalAccessStrategy.java |   2 +-
 .../HibernateTransactionalDataRegion.java     |   3 +-
 .../IgniteBinaryHibernateTestSuite.java       |  37 +
 18 files changed, 1000 insertions(+), 33 deletions(-)
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
 create mode 100644 modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java

diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
index 27734d9ad9ca5..f6c1d0e15eb43 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
@@ -92,7 +92,7 @@
  */
 public abstract class HibernateAccessStrategyAdapter {
     /** */
-    protected final IgniteInternalCache<Object, Object> cache;
+    protected final HibernateCacheProxy cache;
 
     /** Grid. */
     protected final Ignite ignite;
@@ -104,7 +104,7 @@ public abstract class HibernateAccessStrategyAdapter {
      * @param ignite Grid.
      * @param cache Cache.
      */
-    protected HibernateAccessStrategyAdapter(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    protected HibernateAccessStrategyAdapter(Ignite ignite, HibernateCacheProxy cache) {
         this.cache = cache;
         this.ignite = ignite;
 
@@ -292,8 +292,10 @@ protected final void removeAll() throws CacheException {
      * @param key Key.
      * @throws CacheException If failed.
      */
-    static void evict(Ignite ignite, IgniteInternalCache<Object,Object> cache, Object key) throws CacheException {
+    static void evict(Ignite ignite, HibernateCacheProxy cache, Object key) throws CacheException {
         try {
+            key = cache.keyTransformer().transform(key);
+
             ignite.compute(ignite.cluster()).call(new ClearKeyCallable(key, cache.name()));
         }
         catch (IgniteException e) {
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
new file mode 100644
index 0000000000000..871c4a100ac6f
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
@@ -0,0 +1,818 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorResult;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheEntry;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.mxbean.CacheMetricsMXBean;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Hibernate cache proxy.
+ */
+public class HibernateCacheProxy implements IgniteInternalCache<Object, Object> {
+    /** Delegate. */
+    private final IgniteInternalCache<Object, Object> delegate;
+
+    /** Transformer. */
+    private final HibernateKeyTransformer keyTransformer;
+
+    /**
+     * @param delegate Delegate.
+     * @param keyTransformer Key keyTransformer.
+     */
+    HibernateCacheProxy(
+        IgniteInternalCache<Object, Object> delegate,
+        HibernateKeyTransformer keyTransformer
+    ) {
+        assert delegate != null;
+        assert keyTransformer != null;
+
+        this.delegate = delegate;
+        this.keyTransformer = keyTransformer;
+    }
+
+    /**
+     * @return HibernateKeyTransformer
+     */
+    HibernateKeyTransformer keyTransformer(){
+        return keyTransformer;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String name() {
+        return delegate.name();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean skipStore() {
+        return delegate.skipStore();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache setSkipStore(boolean skipStore) {
+        return delegate.setSkipStore(skipStore);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean containsKey(Object key) {
+        return delegate.containsKey(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> containsKeyAsync(Object key) {
+        return delegate.containsKeyAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void promoteAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.promoteAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public long overflowSize() throws IgniteCheckedException {
+        return delegate.overflowSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long swapSize() throws IgniteCheckedException {
+        return delegate.swapSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long swapKeys() throws IgniteCheckedException {
+        return delegate.swapKeys();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean containsKeys(Collection keys) {
+        return delegate.containsKey(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> containsKeysAsync(Collection keys) {
+        return delegate.containsKeysAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object localPeek(
+        Object key,
+        CachePeekMode[] peekModes,
+        @Nullable IgniteCacheExpiryPolicy plc
+    ) throws IgniteCheckedException {
+        return delegate.localPeek(keyTransformer.transform(key), peekModes, plc);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable<Cache.Entry<Object, Object>> localEntries(
+        CachePeekMode[] peekModes
+    ) throws IgniteCheckedException {
+        return delegate.localEntries(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object get(Object key) throws IgniteCheckedException {
+        return delegate.get(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public CacheEntry getEntry(Object key) throws IgniteCheckedException {
+        return delegate.getEntry(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAsync(Object key) {
+        return delegate.getAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<CacheEntry<Object, Object>> getEntryAsync(Object key) {
+        return delegate.getEntryAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map getAll(@Nullable Collection keys) throws IgniteCheckedException {
+        return delegate.getAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<CacheEntry<Object, Object>> getEntries(
+        @Nullable Collection keys) throws IgniteCheckedException {
+        return delegate.getEntries(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map<Object, Object>> getAllAsync(@Nullable Collection keys) {
+        return delegate.getAllAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Collection<CacheEntry<Object,Object>>> getEntriesAsync(
+        @Nullable Collection keys
+    ) {
+        return delegate.getEntriesAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndPut(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndPut(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndPutAsync(Object key, Object val) {
+        return delegate.getAndPutAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean put(Object key, Object val) throws IgniteCheckedException {
+        return delegate.put(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> putAsync(Object key, Object val) {
+        return delegate.putAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndPutIfAbsent(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndPutIfAbsent(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndPutIfAbsentAsync(Object key, Object val) {
+        return delegate.getAndPutIfAbsentAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean putIfAbsent(Object key, Object val) throws IgniteCheckedException {
+        return delegate.putIfAbsent(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(Object key, Object val) {
+        return delegate.putIfAbsentAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndReplace(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndReplace(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndReplaceAsync(Object key, Object val) {
+        return delegate.getAndReplaceAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean replace(Object key, Object val) throws IgniteCheckedException {
+        return delegate.replace(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> replaceAsync(Object key, Object val) {
+        return delegate.replaceAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean replace(Object key, Object oldVal, Object newVal) throws IgniteCheckedException {
+        return delegate.replace(keyTransformer.transform(key), oldVal, newVal);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> replaceAsync(Object key, Object oldVal, Object newVal) {
+        return delegate.replaceAsync(keyTransformer.transform(key), oldVal, newVal);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void putAll(@Nullable Map m) throws IgniteCheckedException {
+        delegate.putAll(transform(m));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> putAllAsync(@Nullable Map m) {
+        return delegate.putAllAsync(transform(m));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set keySet() {
+        return delegate.keySet();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set keySetx() {
+        return delegate.keySetx();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set primaryKeySet() {
+        return delegate.primaryKeySet();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable values() {
+        return delegate.values();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set<Cache.Entry<Object, Object>> entrySet() {
+        return delegate.entrySet();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Set<Cache.Entry<Object,Object>> entrySet(int part) {
+        return delegate.entrySet(part);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set<Cache.Entry<Object, Object>> entrySetx(CacheEntryPredicate... filter) {
+        return delegate.entrySetx(filter);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Transaction txStart(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation
+    ) {
+        return delegate.txStart(concurrency, isolation);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalTx txStartEx(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation
+    ) {
+        return delegate.txStartEx(concurrency, isolation);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Transaction txStart(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation,
+        long timeout,
+        int txSize
+    ) {
+        return delegate.txStart(concurrency, isolation, timeout, txSize);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Transaction tx() {
+        return delegate.tx();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean evict(Object key) {
+        return delegate.evict(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void evictAll(@Nullable Collection keys) {
+        delegate.evictAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearLocally(boolean srv, boolean near, boolean readers) {
+        delegate.clearLocally(srv, near, readers);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean clearLocally(Object key) {
+        return delegate.clearLocally(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearLocallyAll(Set keys, boolean srv, boolean near, boolean readers) {
+        delegate.clearLocallyAll((Set<?>)transform(keys), srv, near, readers);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear(Object key) throws IgniteCheckedException {
+        delegate.clear(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearAll(Set keys) throws IgniteCheckedException {
+        delegate.clearAll((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear() throws IgniteCheckedException {
+        delegate.clear();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAsync() {
+        return delegate.clearAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAsync(Object key) {
+        return delegate.clearAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAllAsync(Set keys) {
+        return delegate.clearAllAsync((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndRemove(Object key) throws IgniteCheckedException {
+        return delegate.getAndRemove(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndRemoveAsync(Object key) {
+        return delegate.getAndRemoveAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(Object key) throws IgniteCheckedException {
+        return delegate.remove(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> removeAsync(Object key) {
+        return delegate.removeAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(Object key, Object val) throws IgniteCheckedException {
+        return delegate.remove(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> removeAsync(Object key, Object val) {
+        return delegate.removeAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.removeAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllAsync(@Nullable Collection keys) {
+        return delegate.removeAllAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAll() throws IgniteCheckedException {
+        delegate.removeAll();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllAsync() {
+        return delegate.removeAllAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lock(Object key, long timeout) throws IgniteCheckedException {
+        return delegate.lock(keyTransformer.transform(key), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> lockAsync(Object key, long timeout) {
+        return delegate.lockAsync(keyTransformer.transform(key), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lockAll(@Nullable Collection keys, long timeout) throws IgniteCheckedException {
+        return delegate.lockAll(transform(keys), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> lockAllAsync(@Nullable Collection keys, long timeout) {
+        return delegate.lockAllAsync(transform(keys), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlock(Object key) throws IgniteCheckedException {
+        delegate.unlock(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlockAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.unlockAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLocked(Object key) {
+        return delegate.isLocked(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLockedByThread(Object key) {
+        return delegate.isLockedByThread(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size() {
+        return delegate.size();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong() {
+        return delegate.sizeLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int localSize(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSize(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long localSizeLong(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSizeLong(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long localSizeLong(int partition, CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSizeLong(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.size(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.sizeLong(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong(int partition, CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.sizeLong(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Integer> sizeAsync(CachePeekMode[] peekModes) {
+        return delegate.sizeAsync(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Long> sizeLongAsync(CachePeekMode[] peekModes) {
+        return delegate.sizeLongAsync(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Long> sizeLongAsync(int partition, CachePeekMode[] peekModes) {
+        return delegate.sizeLongAsync(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int nearSize() {
+        return delegate.nearSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int primarySize() {
+        return delegate.primarySize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long primarySizeLong() {
+        return delegate.primarySizeLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheConfiguration configuration() {
+        return delegate.configuration();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Affinity affinity() {
+        return delegate.affinity();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics clusterMetrics() {
+        return delegate.clusterMetrics();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics clusterMetrics(ClusterGroup grp) {
+        return delegate.clusterMetrics(grp);
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics localMetrics() {
+        return delegate.localMetrics();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetricsMXBean clusterMxBean() {
+        return delegate.clusterMxBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetricsMXBean localMxBean() {
+        return delegate.localMxBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long offHeapEntriesCount() {
+        return delegate.offHeapEntriesCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long offHeapAllocatedSize() {
+        return delegate.offHeapAllocatedSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> rebalance() {
+        return delegate.rebalance();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache forSubjectId(UUID subjId) {
+        return delegate.forSubjectId(subjId);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getForcePrimary(Object key) throws IgniteCheckedException {
+        return delegate.getForcePrimary(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getForcePrimaryAsync(Object key) {
+        return delegate.getForcePrimaryAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map getAllOutTx(Set keys) throws IgniteCheckedException {
+        return delegate.getAllOutTx((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map<Object, Object>> getAllOutTxAsync(Set keys) {
+        return delegate.getAllOutTxAsync((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isIgfsDataCache() {
+        return delegate.isIgfsDataCache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long igfsDataSpaceUsed() {
+        return delegate.igfsDataSpaceUsed();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long igfsDataSpaceMax() {
+        return delegate.igfsDataSpaceMax();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMongoDataCache() {
+        return delegate.isMongoDataCache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMongoMetaCache() {
+        return delegate.isMongoMetaCache();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public ExpiryPolicy expiry() {
+        return delegate.expiry();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache withExpiryPolicy(ExpiryPolicy plc) {
+        return delegate.withExpiryPolicy(plc);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache withNoRetries() {
+        return delegate.withNoRetries();
+    }
+
+    /** {@inheritDoc} */
+    @Override public GridCacheContext context() {
+        return delegate.context();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void localLoadCache(
+        @Nullable IgniteBiPredicate p,
+        @Nullable Object... args
+    ) throws IgniteCheckedException {
+        delegate.localLoadCache(p, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> localLoadCacheAsync(
+        @Nullable IgniteBiPredicate p,
+        @Nullable Object... args
+    ) {
+        return delegate.localLoadCacheAsync(p, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Object getTopologySafe(Object key) throws IgniteCheckedException {
+        return delegate.getTopologySafe(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object tryGetAndPut(Object key, Object val) throws IgniteCheckedException {
+        return delegate.tryGetAndPut(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryProcessorResult invoke(
+        @Nullable AffinityTopologyVersion topVer,
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) throws IgniteCheckedException {
+        return delegate.invoke(topVer, key, entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map> invokeAllAsync(Map map, Object... args) {
+        return delegate.invokeAllAsync(map, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map invokeAll(Map map, Object... args) throws IgniteCheckedException {
+        return delegate.invokeAll(map, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map> invokeAllAsync(Set keys, EntryProcessor entryProcessor, Object... args) {
+        return delegate.invokeAllAsync((Set<?>)transform(keys), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map invokeAll(Set keys, EntryProcessor entryProcessor, Object... args) throws IgniteCheckedException {
+        return delegate.invokeAll((Set<?>)transform(keys), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<EntryProcessorResult> invokeAsync(
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) {
+        return delegate.invokeAsync(keyTransformer.transform(key), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryProcessorResult invoke(
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) throws IgniteCheckedException {
+        return delegate.invoke(keyTransformer.transform(key), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllConflictAsync(Map drMap) throws IgniteCheckedException {
+        return delegate.removeAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAllConflict(Map drMap) throws IgniteCheckedException {
+        delegate.removeAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> putAllConflictAsync(Map drMap) throws IgniteCheckedException {
+        return delegate.putAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void putAllConflict(Map drMap) throws IgniteCheckedException {
+        delegate.putAllConflict(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache keepBinary() {
+        return delegate.keepBinary();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache cache() {
+        return delegate.cache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterator iterator() {
+        return delegate.iterator();
+    }
+
+    /**
+     * @param keys Keys.
+     */
+    private Collection<Object> transform(Collection<Object> keys) {
+        Collection<Object> res = new LinkedList<>();
+
+        for (Object o : keys)
+            res.add(keyTransformer.transform(o));
+
+        return res;
+    }
+
+    /**
+     * @param map Map.
+     */
+    private Map<Object, Object> transform(Map<Object, Object> map) {
+        Map<Object, Object> res = new HashMap<>();
+
+        Set<Map.Entry<Object, Object>> ents = map.entrySet();
+
+        for (Map.Entry<Object, Object> e : ents)
+            res.put(keyTransformer.transform(e.getKey()), e.getValue());
+
+        return res;
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
index 045f401d92f80..eb35a2c4709d7 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.CollectionRegion;
@@ -72,7 +71,7 @@ public class HibernateCollectionRegion extends HibernateTransactionalDataRegion
      * @param dataDesc Region data description.
      */
     public HibernateCollectionRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
index 1ceda144a8355..ad5b1919f51b3 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.EntityRegion;
@@ -62,7 +61,7 @@ public class HibernateEntityRegion extends HibernateTransactionalDataRegion impl
      * @param dataDesc Region data description.
      */
     public HibernateEntityRegion(HibernateRegionFactory factory, String name, Ignite ignite,
-        IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
index fbac624f66af1..2f1a11dc8baa3 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.GeneralDataRegion;
 import org.hibernate.cache.spi.QueryResultsRegion;
@@ -38,7 +37,7 @@ public class HibernateGeneralDataRegion extends HibernateRegion implements Gener
      * @param cache Region cache.
      */
     public HibernateGeneralDataRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite, HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
new file mode 100644
index 0000000000000..97fc0e9eb15e6
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+/**
+ * An interface for transforming hibernate keys to Ignite keys.
+ */
+public interface HibernateKeyTransformer {
+    /**
+     * @param key Hibernate key.
+     * @return Transformed key.
+     */
+    public Object transform(Object key);
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
new file mode 100644
index 0000000000000..7de440ebb032c
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Hibernate cache key wrapper.
+ */
+public class HibernateKeyWrapper {
+    /** Key. */
+    private final Object key;
+
+    /** Entry. */
+    private final String entry;
+
+    /** */
+    private final String tenantId;
+
+    /**
+     * @param key Key.
+     * @param entry Entry.
+     * @param tenantId Tenant ID.
+     */
+    HibernateKeyWrapper(Object key, String entry, String tenantId) {
+        this.key = key;
+        this.entry = entry;
+        this.tenantId = tenantId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o) return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        HibernateKeyWrapper that = (HibernateKeyWrapper) o;
+
+        return (key != null ? key.equals(that.key) : that.key == null) &&
+            (entry != null ? entry.equals(that.entry) : that.entry == null) &&
+            (tenantId != null ? tenantId.equals(that.tenantId) : that.tenantId == null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = key != null ? key.hashCode() : 0;
+        res = 31 * res + (entry != null ? entry.hashCode() : 0);
+        res = 31 * res + (tenantId != null ? tenantId.hashCode() : 0);
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HibernateKeyWrapper.class, this);
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
index 99d5348dea479..862a4228f2767 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.NaturalIdRegion;
@@ -52,7 +51,7 @@ public class HibernateNaturalIdRegion extends HibernateTransactionalDataRegion i
      * @param dataDesc Region data description.
      */
     public HibernateNaturalIdRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
index 1cb8d48e0be2f..a36d7e786d739 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
@@ -21,7 +21,6 @@
 import java.util.Set;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridLeanMap;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.typedef.F;
@@ -67,7 +66,7 @@ public class HibernateNonStrictAccessStrategy extends HibernateAccessStrategyAda
      * @param cache Cache.
      * @param writeCtx Thread local instance used to track updates done during one Hibernate transaction.
      */
-    protected HibernateNonStrictAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache, ThreadLocal writeCtx) {
+    protected HibernateNonStrictAccessStrategy(Ignite ignite, HibernateCacheProxy cache, ThreadLocal writeCtx) {
         super(ignite, cache);
 
         this.writeCtx = (ThreadLocal<WriteContext>)writeCtx;
@@ -212,7 +211,7 @@ void removed(Object key) {
          * @param cache Cache.
          * @throws IgniteCheckedException If failed.
          */
-        void updateCache(IgniteInternalCache<Object, Object> cache) throws IgniteCheckedException {
+        void updateCache(HibernateCacheProxy cache) throws IgniteCheckedException {
             if (!F.isEmpty(rmvs))
                 cache.removeAll(rmvs);
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
index e3303a71ff413..0b9a43d6a13d6 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.Query;
 import org.hibernate.cache.spi.QueryResultsRegion;
 
@@ -65,7 +64,7 @@ public class HibernateQueryResultsRegion extends HibernateGeneralDataRegion impl
      * @param cache Region cache.
      */
     public HibernateQueryResultsRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite, HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
index 58a2c4b8779d4..cdef80e0a0c0b 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.access.AccessType;
 import org.hibernate.cache.spi.access.SoftLock;
@@ -60,7 +59,7 @@ public class HibernateReadOnlyAccessStrategy extends HibernateAccessStrategyAdap
      * @param ignite Grid.
      * @param cache Cache.
      */
-    public HibernateReadOnlyAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    public HibernateReadOnlyAccessStrategy(Ignite ignite, HibernateCacheProxy cache) {
         super(ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
index bbb1d4e5dfdc2..625b05061ff0f 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
@@ -21,7 +21,6 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.transactions.Transaction;
 import org.hibernate.cache.CacheException;
@@ -68,7 +67,7 @@ public class HibernateReadWriteAccessStrategy extends HibernateAccessStrategyAda
      * @param cache Cache.
      * @param txCtx Thread local instance used to track updates done during one Hibernate transaction.
      */
-    protected HibernateReadWriteAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache, ThreadLocal txCtx) {
+    protected HibernateReadWriteAccessStrategy(Ignite ignite, HibernateCacheProxy cache, ThreadLocal txCtx) {
         super(ignite, cache);
 
         this.txCtx = (ThreadLocal<TxContext>)txCtx;
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
index 27479e90d8b7b..11a96d09096fb 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
@@ -20,7 +20,6 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.Region;
 
@@ -35,7 +34,7 @@ public class HibernateRegion implements Region {
     private final String name;
 
     /** Cache instance. */
-    protected final IgniteInternalCache<Object, Object> cache;
+    protected final HibernateCacheProxy cache;
 
     /** Grid instance. */
     protected Ignite ignite;
@@ -46,8 +45,7 @@ public class HibernateRegion implements Region {
      * @param ignite Grid.
      * @param cache Region cache.
      */
-    public HibernateRegion(HibernateRegionFactory factory, String name, Ignite ignite,
-        IgniteInternalCache<Object, Object> cache) {
+    public HibernateRegion(HibernateRegionFactory factory, String name, Ignite ignite, HibernateCacheProxy cache) {
         this.factory = factory;
         this.name = name;
         this.ignite = ignite;
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
index 825abee7e2590..4e4be36ec7c49 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.util.typedef.G;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
+import org.hibernate.cache.spi.CacheKey;
 import org.hibernate.cache.spi.CollectionRegion;
 import org.hibernate.cache.spi.EntityRegion;
 import org.hibernate.cache.spi.NaturalIdRegion;
@@ -88,7 +89,7 @@ public class HibernateRegionFactory implements RegionFactory {
     private Ignite ignite;
 
     /** Default cache. */
-    private IgniteInternalCache<Object, Object> dfltCache;
+    private HibernateCacheProxy dfltCache;
 
     /** Default region access type. */
     private AccessType dfltAccessType;
@@ -99,6 +100,23 @@ public class HibernateRegionFactory implements RegionFactory {
     /** Map needed to provide the same transaction context for different regions. */
     private final ThreadLocal threadLoc = new ThreadLocal();
 
+    /** Key transformer. */
+    private final HibernateKeyTransformer hibernate4transformer = new HibernateKeyTransformer() {
+        @Override public Object transform(Object key) {
+            if (key instanceof CacheKey) {
+                CacheKey cacheKey = (CacheKey)key;
+
+                return new HibernateKeyWrapper(
+                    cacheKey.getKey(),
+                    cacheKey.getEntityOrRoleName(),
+                    cacheKey.getTenantId()
+                );
+            }
+
+            return key;
+        }
+    };
+
     /** {@inheritDoc} */
     @Override public void start(Settings settings, Properties props) throws CacheException {
         String gridCfg = props.getProperty(GRID_CONFIG_PROPERTY);
@@ -138,10 +156,12 @@ public class HibernateRegionFactory implements RegionFactory {
         String dfltCacheName = props.getProperty(DFLT_CACHE_NAME_PROPERTY);
 
         if (dfltCacheName != null) {
-            dfltCache = ((IgniteKernal)ignite).getCache(dfltCacheName);
+            IgniteInternalCache<Object, Object> dfltCache = ((IgniteKernal)ignite).getCache(dfltCacheName);
 
             if (dfltCache == null)
                 throw new CacheException("Cache specified as default is not configured: " + dfltCacheName);
+
+            this.dfltCache = new HibernateCacheProxy(dfltCache, hibernate4transformer);
         }
 
         IgniteLogger log = ignite.log().getLogger(HibernateRegionFactory.class);
@@ -152,6 +172,7 @@ public class HibernateRegionFactory implements RegionFactory {
 
     /** {@inheritDoc} */
     @Override public void stop() {
+        // No-op.
     }
 
     /** {@inheritDoc} */
@@ -213,7 +234,7 @@ ThreadLocal threadLocalForCache(String cacheName) {
      * @return Cache for given region.
      * @throws CacheException If cache for given region is not configured.
      */
-    private IgniteInternalCache<Object, Object> regionCache(String regionName) throws CacheException {
+    private HibernateCacheProxy regionCache(String regionName) throws CacheException {
         String cacheName = regionCaches.get(regionName);
 
         if (cacheName == null) {
@@ -228,6 +249,6 @@ private IgniteInternalCache<Object, Object> regionCache(String regionName) throw
         if (cache == null)
             throw new CacheException("Cache '" + cacheName + "' for region '" + regionName + "' is not configured.");
 
-        return cache;
+        return new HibernateCacheProxy(cache, hibernate4transformer);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
index 4cedae2070e31..8b4c243277ec9 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.spi.TimestampsRegion;
 
 /**
@@ -34,7 +33,7 @@ public class HibernateTimestampsRegion extends HibernateGeneralDataRegion implem
      * @param cache Region cache.
      */
     public HibernateTimestampsRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite,  HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
index 80f75a71afed6..ca5284917aa68 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
@@ -61,7 +61,7 @@ public class HibernateTransactionalAccessStrategy extends HibernateAccessStrateg
      * @param ignite Grid.
      * @param cache Cache.
      */
-    public HibernateTransactionalAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    public HibernateTransactionalAccessStrategy(Ignite ignite, HibernateCacheProxy cache) {
         super(ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
index ed2ee01cf2280..581076a3fdbb6 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.TransactionConfiguration;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.CollectionRegion;
@@ -48,7 +47,7 @@ public class HibernateTransactionalDataRegion extends HibernateRegion implements
      * @param dataDesc Region data description.
      */
     public HibernateTransactionalDataRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache);
 
         this.dataDesc = dataDesc;
diff --git a/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java b/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java
new file mode 100644
index 0000000000000..3791baed9c93c
--- /dev/null
+++ b/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.testframework.config.GridTestProperties;
+
+/**
+ *
+ */
+public class IgniteBinaryHibernateTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception If failed.
+     */
+    public static TestSuite suite() throws Exception {
+        GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, BinaryMarshaller.class.getName());
+
+        return IgniteHibernateTestSuite.suite();
+    }
+}

From a352951d91edde9c0029a8bf435d61b4a7cd8c11 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 4 Jul 2017 20:24:52 +0300
Subject: [PATCH 356/446] IGNITE-4831: Add an option to disable MBeans.

---
 .../apache/ignite/IgniteSystemProperties.java |   8 ++
 .../org/apache/ignite/cache/CacheManager.java |  11 ++
 .../apache/ignite/internal/IgniteKernal.java  |  35 +++--
 .../apache/ignite/internal/IgnitionEx.java    |  10 +-
 .../client/router/impl/GridTcpRouterImpl.java |  57 ++++++---
 .../processors/cache/GridCacheProcessor.java  |  10 +-
 .../ignite/internal/util/IgniteUtils.java     |  17 ++-
 .../apache/ignite/spi/IgniteSpiAdapter.java   |   3 +
 .../util/mbeans/GridMBeanDisableSelfTest.java | 121 ++++++++++++++++++
 9 files changed, 238 insertions(+), 34 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index c4208a76f1a7d..e38d236f49964 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -382,6 +382,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_MBEAN_APPEND_CLASS_LOADER_ID = "IGNITE_MBEAN_APPEND_CLASS_LOADER_ID";
 
+    /**
+     * If property is set to {@code true}, then Ignite will disable MBeans registration.
+     * This may be helpful if MBeans are not allowed e.g. for security reasons.
+     *
+     * Default is {@code false}
+     */
+    public static final String IGNITE_MBEANS_DISABLED = "IGNITE_MBEANS_DISABLED";
+
     /**
      * Property controlling size of buffer holding last exception. Default value of {@code 1000}.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java
index 6e09d7276f3ad..a5e612ef986fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java
@@ -44,7 +44,9 @@
 import org.apache.ignite.internal.IgnitionEx;
 import org.apache.ignite.internal.mxbean.IgniteStandardMXBean;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
+import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.jetbrains.annotations.Nullable;
@@ -314,6 +316,9 @@ private ObjectName getObjectName(String cacheName, String objName) {
 
     /** {@inheritDoc} */
     @Override public void enableManagement(String cacheName, boolean enabled) {
+        if(IgniteUtils.IGNITE_MBEANS_DISABLED)
+            return;
+
         kernalGateway.readLock();
 
         try {
@@ -336,6 +341,9 @@ private ObjectName getObjectName(String cacheName, String objName) {
 
     /** {@inheritDoc} */
     @Override public void enableStatistics(String cacheName, boolean enabled) {
+        if(IgniteUtils.IGNITE_MBEANS_DISABLED)
+            return;
+
         kernalGateway.readLock();
 
         try {
@@ -389,6 +397,9 @@ private void registerCacheObject(Object mxbean, String name, String beanType) {
      * @param beanType Mxbean name.
      */
     private void unregisterCacheObject(String name, String beanType) {
+        if(IgniteUtils.IGNITE_MBEANS_DISABLED)
+            return;
+
         MBeanServer mBeanSrv = ignite.configuration().getMBeanServer();
 
         Set<ObjectName> registeredObjNames = mBeanSrv.queryNames(getObjectName(name, beanType), null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 2aa05f1b9ab11..9bada6c35ce3b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1494,6 +1494,9 @@ private void addSpiAttributes(IgniteSpi... spiList) throws IgniteCheckedExceptio
 
     /** @throws IgniteCheckedException If registration failed. */
     private void registerKernalMBean() throws IgniteCheckedException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
+
         try {
             kernalMBean = U.registerMBean(
                 cfg.getMBeanServer(),
@@ -1515,6 +1518,9 @@ private void registerKernalMBean() throws IgniteCheckedException {
 
     /** @throws IgniteCheckedException If registration failed. */
     private void registerLocalNodeMBean() throws IgniteCheckedException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
+
         ClusterLocalNodeMetricsMXBean mbean = new ClusterLocalNodeMetricsMXBeanImpl(ctx.discovery().localNode());
 
         try {
@@ -1546,6 +1552,8 @@ private void registerExecutorMBeans(
         ExecutorService utilityExecSvc,
         ExecutorService marshallerExecSvc
     ) throws IgniteCheckedException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
         pubExecSvcMBean = registerExecutorMBean(execSvc, "GridExecutionExecutor");
         sysExecSvcMBean = registerExecutorMBean(sysExecSvc, "GridSystemExecutor");
         mgmtExecSvcMBean = registerExecutorMBean(mgmtExecSvc, "GridManagementExecutor");
@@ -1567,6 +1575,7 @@ private void registerExecutorMBeans(
      */
     private ObjectName registerExecutorMBean(ExecutorService exec, String name) throws IgniteCheckedException {
         assert exec != null;
+        assert !U.IGNITE_MBEANS_DISABLED;
 
         try {
             ObjectName res = U.registerMBean(
@@ -1595,22 +1604,24 @@ private ObjectName registerExecutorMBean(ExecutorService exec, String name) thro
      * @return {@code True} if successfully unregistered, {@code false} otherwise.
      */
     private boolean unregisterMBean(@Nullable ObjectName mbean) {
-        if (mbean != null)
-            try {
-                cfg.getMBeanServer().unregisterMBean(mbean);
+        if (mbean == null)
+            return true;
 
-                if (log.isDebugEnabled())
-                    log.debug("Unregistered MBean: " + mbean);
+        assert !U.IGNITE_MBEANS_DISABLED;
 
-                return true;
-            }
-            catch (JMException e) {
-                U.error(log, "Failed to unregister MBean.", e);
+        try {
+            cfg.getMBeanServer().unregisterMBean(mbean);
 
-                return false;
-            }
+            if (log.isDebugEnabled())
+                log.debug("Unregistered MBean: " + mbean);
+
+            return true;
+        }
+        catch (JMException e) {
+            U.error(log, "Failed to unregister MBean.", e);
 
-        return true;
+            return false;
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 5b2c3fc53faea..8d54ba57b8762 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -1950,7 +1950,7 @@ private IgniteConfiguration initializeConfiguration(IgniteConfiguration cfg)
             if (myCfg.getUserAttributes() == null)
                 myCfg.setUserAttributes(Collections.<String, Object>emptyMap());
 
-            if (myCfg.getMBeanServer() == null)
+            if (myCfg.getMBeanServer() == null && !U.IGNITE_MBEANS_DISABLED)
                 myCfg.setMBeanServer(ManagementFactory.getPlatformMBeanServer());
 
             Marshaller marsh = myCfg.getMarshaller();
@@ -2430,6 +2430,11 @@ private void stopExecutors0(IgniteLogger log) {
          * @throws IgniteCheckedException If registration failed.
          */
         private void registerFactoryMbean(MBeanServer srv) throws IgniteCheckedException {
+            if(U.IGNITE_MBEANS_DISABLED)
+                return;
+
+            assert srv != null;
+
             synchronized (mbeans) {
                 GridMBeanServerData data = mbeans.get(srv);
 
@@ -2480,6 +2485,9 @@ private void registerFactoryMbean(MBeanServer srv) throws IgniteCheckedException
          * Unregister delegate Mbean instance for {@link Ignition}.
          */
         private void unregisterFactoryMBean() {
+            if(U.IGNITE_MBEANS_DISABLED)
+                return;
+
             synchronized (mbeans) {
                 Iterator<Entry<MBeanServer, GridMBeanServerData>> iter = mbeans.entrySet().iterator();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
index 06a492924f408..6210bb06b765b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
@@ -169,6 +169,32 @@ public GridTcpRouterImpl(GridTcpRouterConfiguration cfg) {
                 "are in use) [firstPort=" + cfg.getPort() + ", lastPort=" + (cfg.getPort() + cfg.getPortRange()) +
                 ", addr=" + hostAddr + ']');
 
+        registerMBean();
+    }
+
+    /**
+     * Stops this router.
+     */
+    @Override public void stop() {
+        if (srv != null)
+            srv.stop();
+
+        if (client != null)
+            client.stop(true);
+
+        unregisterMBean();
+
+        if (log.isInfoEnabled())
+            log.info("TCP router successfully stopped.");
+    }
+
+    /**
+     * Try to register MBean.
+     */
+    private void registerMBean() {
+        if (U.IGNITE_MBEANS_DISABLED)
+            return;
+
         try {
             ObjectName objName = U.registerMBean(
                 ManagementFactory.getPlatformMBeanServer(),
@@ -189,28 +215,23 @@ public GridTcpRouterImpl(GridTcpRouterConfiguration cfg) {
     }
 
     /**
-     * Stops this router.
+     * Unregister MBean.
      */
-    @Override public void stop() {
-        if (srv != null)
-            srv.stop();
-
-        if (client != null)
-            client.stop(true);
+    private void unregisterMBean() {
+        if (mbeanName == null)
+            return;
 
-        if (mbeanName != null)
-            try {
-                ManagementFactory.getPlatformMBeanServer().unregisterMBean(mbeanName);
+        assert !U.IGNITE_MBEANS_DISABLED;
 
-                if (log.isDebugEnabled())
-                    log.debug("Unregistered MBean: " + mbeanName);
-            }
-            catch (JMException e) {
-                U.error(log, "Failed to unregister MBean.", e);
-            }
+        try {
+            ManagementFactory.getPlatformMBeanServer().unregisterMBean(mbeanName);
 
-        if (log.isInfoEnabled())
-            log.info("TCP router successfully stopped.");
+            if (log.isDebugEnabled())
+                log.debug("Unregistered MBean: " + mbeanName);
+        }
+        catch (JMException e) {
+            U.error(log, "Failed to unregister MBean.", e);
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index ccd7ae0bf500d..341b2c19d38e4 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3497,6 +3497,9 @@ public void createMissingCaches() throws IgniteCheckedException {
     @SuppressWarnings("unchecked")
     private void registerMbean(Object o, @Nullable String cacheName, boolean near)
         throws IgniteCheckedException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
+
         assert o != null;
 
         MBeanServer srvr = ctx.config().getMBeanServer();
@@ -3513,7 +3516,7 @@ private void registerMbean(Object o, @Nullable String cacheName, boolean near)
                     U.registerCacheMBean(srvr, ctx.gridName(), cacheName, o.getClass().getName(), o,
                         (Class<Object>)itf);
                 }
-                catch (JMException e) {
+                catch (Throwable e) {
                     throw new IgniteCheckedException("Failed to register MBean for component: " + o, e);
                 }
 
@@ -3530,6 +3533,9 @@ private void registerMbean(Object o, @Nullable String cacheName, boolean near)
      * @param near Near flag.
      */
     private void unregisterMbean(Object o, @Nullable String cacheName, boolean near) {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
+
         assert o != null;
 
         MBeanServer srvr = ctx.config().getMBeanServer();
@@ -3545,7 +3551,7 @@ private void unregisterMbean(Object o, @Nullable String cacheName, boolean near)
                 try {
                     srvr.unregisterMBean(U.makeCacheMBeanName(ctx.gridName(), cacheName, o.getClass().getName()));
                 }
-                catch (JMException e) {
+                catch (Throwable e) {
                     U.error(log, "Failed to unregister MBean for component: " + o, e);
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index c2efb953c9135..84566872e5abf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -134,6 +134,7 @@
 import java.util.zip.ZipOutputStream;
 import javax.management.DynamicMBean;
 import javax.management.JMException;
+import javax.management.MBeanRegistrationException;
 import javax.management.MBeanServer;
 import javax.management.MalformedObjectNameException;
 import javax.management.ObjectName;
@@ -150,10 +151,10 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteDeploymentException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteIllegalStateException;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryRawReader;
 import org.apache.ignite.binary.BinaryRawWriter;
 import org.apache.ignite.cluster.ClusterGroupEmptyException;
@@ -509,6 +510,9 @@ public abstract class IgniteUtils {
         }
     };
 
+    /** Ignite MBeans disabled flag. */
+    public static boolean IGNITE_MBEANS_DISABLED = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_MBEANS_DISABLED);
+
     /**
      * Initializes enterprise check.
      */
@@ -4423,10 +4427,13 @@ public static ObjectName makeCacheMBeanName(@Nullable String gridName, @Nullable
      * @param impl MBean implementation.
      * @param itf MBean interface.
      * @return JMX object name.
+     * @throws MBeanRegistrationException if MBeans are disabled.
      * @throws JMException If MBean creation failed.
      */
     public static <T> ObjectName registerMBean(MBeanServer mbeanSrv, @Nullable String gridName, @Nullable String grp,
         String name, T impl, @Nullable Class<T> itf) throws JMException {
+        if(IGNITE_MBEANS_DISABLED)
+            throw new MBeanRegistrationException(new IgniteIllegalStateException("No MBeans are allowed."));
         assert mbeanSrv != null;
         assert name != null;
         assert itf != null;
@@ -4447,10 +4454,14 @@ public static <T> ObjectName registerMBean(MBeanServer mbeanSrv, @Nullable Strin
      * @param impl MBean implementation.
      * @param itf MBean interface.
      * @return JMX object name.
+     * @throws MBeanRegistrationException if MBeans are disabled.
      * @throws JMException If MBean creation failed.
      */
     public static <T> ObjectName registerMBean(MBeanServer mbeanSrv, ObjectName name, T impl, Class<T> itf)
         throws JMException {
+        if(IGNITE_MBEANS_DISABLED)
+            throw new MBeanRegistrationException(new IgniteIllegalStateException("MBeans are disabled."));
+
         assert mbeanSrv != null;
         assert name != null;
         assert itf != null;
@@ -4473,10 +4484,14 @@ public static <T> ObjectName registerMBean(MBeanServer mbeanSrv, ObjectName name
      * @param impl MBean implementation.
      * @param itf MBean interface.
      * @return JMX object name.
+     * @throws MBeanRegistrationException if MBeans are disabled.
      * @throws JMException If MBean creation failed.
      */
     public static <T> ObjectName registerCacheMBean(MBeanServer mbeanSrv, @Nullable String gridName,
         @Nullable String cacheName, String name, T impl, Class<T> itf) throws JMException {
+        if(IGNITE_MBEANS_DISABLED)
+            throw new MBeanRegistrationException(new IgniteIllegalStateException("MBeans are disabled."));
+
         assert mbeanSrv != null;
         assert name != null;
         assert itf != null;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index 8879364dca5dd..e2350f0a50a3d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -410,6 +410,8 @@ private static String format(String msg, Object locVal, Object rmtVal) {
      */
     protected final <T extends IgniteSpiManagementMBean> void registerMBean(String gridName, T impl, Class<T> mbeanItf)
         throws IgniteSpiException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
         MBeanServer jmx = ignite.configuration().getMBeanServer();
 
         assert mbeanItf == null || mbeanItf.isInterface();
@@ -434,6 +436,7 @@ protected final <T extends IgniteSpiManagementMBean> void registerMBean(String g
     protected final void unregisterMBean() throws IgniteSpiException {
         // Unregister SPI MBean.
         if (spiMBean != null) {
+            assert !U.IGNITE_MBEANS_DISABLED;
             MBeanServer jmx = ignite.configuration().getMBeanServer();
 
             assert jmx != null;
diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java
new file mode 100644
index 0000000000000..f08f58bd9258a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util.mbeans;
+
+import java.util.concurrent.Callable;
+import javax.management.MBeanRegistrationException;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Disabling MBeans test.
+ */
+public class GridMBeanDisableSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        IgniteUtils.IGNITE_MBEANS_DISABLED = true;
+
+        super.beforeTestsStarted();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        IgniteUtils.IGNITE_MBEANS_DISABLED = false;
+    }
+
+    /**
+     * Test MBean registration.
+     *
+     * @throws Exception Thrown if test fails.
+     */
+    public void testCorrectMBeanInfo() throws Exception {
+        // Node should start and stopped with no errors.
+        try (final Ignite ignite = startGrid(0)) {
+
+            // Cache should be created and closed with no errors.
+            try (IgniteCache cache = ignite.getOrCreateCache(new CacheConfiguration("MyCache"))) {
+
+                final MBeanServer server = ignite.configuration().getMBeanServer();
+
+                GridTestUtils.assertThrowsWithCause(
+                    new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            U.registerMBean(server, ignite.name(), "dummy", "DummyMbean1", new DummyMBeanImpl(), DummyMBean.class);
+
+                            return null;
+
+                        }
+                    }, MBeanRegistrationException.class);
+
+                GridTestUtils.assertThrowsWithCause(
+                    new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            ObjectName objName = U.makeMBeanName(
+                                ignite.name(),
+                                "dummy",
+                                "DummyMbean2"
+                            );
+
+                            U.registerMBean(server, objName, new DummyMBeanImpl(), DummyMBean.class);
+
+                            return null;
+
+                        }
+                    }, MBeanRegistrationException.class);
+
+                GridTestUtils.assertThrowsWithCause(
+                    new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            U.registerCacheMBean(server, ignite.name(), "MyCache", "DummyMbean3",
+                                new DummyMBeanImpl(), DummyMBean.class);
+
+                            return null;
+
+                        }
+                    }, MBeanRegistrationException.class);
+            }
+        }
+    }
+
+    /**
+     * MBean dummy interface.
+     */
+    interface DummyMBean {
+        /** */
+        void noop();
+    }
+
+    /**
+     * MBean stub.
+     */
+    static class DummyMBeanImpl implements DummyMBean {
+        /** {@inheritDoc} */
+        @Override public void noop() {
+            // No op.
+        }
+    }
+}
\ No newline at end of file

From e4d141e97ab4ec34b5fe6a7bc599413223944438 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 14 Jul 2017 14:40:02 +0300
Subject: [PATCH 357/446] IGNITE-5103 - Server drops client node from cluster
 when no heartbeat messages received in interval heartBeatsFrequency *
 maxMissedClientHeartBeats.

---
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  46 +++++-
 ...entDiscoverySpiFailureTimeoutSelfTest.java |  14 ++
 .../tcp/TcpClientDiscoverySpiSelfTest.java    |   2 +-
 .../TcpDiscoveryClientSuspensionSelfTest.java | 133 ++++++++++++++++++
 4 files changed, 193 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 6e5af2986dcff..75d8f83344d25 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6386,6 +6386,12 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
         /** Current client metrics. */
         private volatile ClusterMetrics metrics;
 
+        /** Last heartbeat message receive time. */
+        private volatile long lastHbMsgTime;
+
+        /** Max heartbeat message receive interval. */
+        private final long maxHbInterval;
+
         /** */
         private final AtomicReference<GridFutureAdapter<Boolean>> pingFut = new AtomicReference<>();
 
@@ -6397,10 +6403,13 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
          * @param clientNodeId Node ID.
          */
         protected ClientMessageWorker(Socket sock, UUID clientNodeId) throws IOException {
-            super("tcp-disco-client-message-worker", 2000);
+            super("tcp-disco-client-message-worker", Math.max(spi.hbFreq, 10));
 
             this.sock = sock;
             this.clientNodeId = clientNodeId;
+
+            maxHbInterval = spi.hbFreq * spi.maxMissedClientHbs;
+            lastHbMsgTime = U.currentTimeMillis();
         }
 
         /**
@@ -6421,6 +6430,9 @@ ClusterMetrics metrics() {
          * @param metrics New current client metrics.
          */
         void metrics(ClusterMetrics metrics) {
+            // Updates only on heartbeat message.
+            lastHbMsgTime = U.currentTimeMillis();
+
             this.metrics = metrics;
         }
 
@@ -6597,6 +6609,38 @@ public boolean ping(IgniteSpiOperationTimeoutHelper timeoutHelper) throws Interr
             }
         }
 
+        /** {@inheritDoc} */
+        @Override protected void noMessageLoop() {
+            if (U.currentTimeMillis() - lastHbMsgTime > maxHbInterval) {
+                TcpDiscoveryNode clientNode = ring.node(clientNodeId);
+
+                if (clientNode != null) {
+                    boolean failedNode;
+
+                    synchronized (mux) {
+                        failedNode = failedNodes.containsKey(clientNode);
+                    }
+
+                    if (!failedNode) {
+                        String msg = "Client node considered as unreachable " +
+                            "and will be dropped from cluster, " +
+                            "because no heartbeat messages received in interval: " +
+                            "TcpDiscoverySpi.getHeartbeatFrequency() * TcpDiscoverySpi.getMaxMissedClientHeartbeats() ms. " +
+                            "It maybe caused by network problems or long GC pause on client node, try to increase mentioned " +
+                            "parameters. " +
+                            "[nodeId=" + clientNodeId +
+                            ", heartBeatFrequency=" + spi.hbFreq +
+                            ", maxMissedClientHeartbeats=" + spi.maxMissedClientHbs +
+                            ']';
+
+                        failNode(clientNodeId, msg);
+
+                        U.warn(log, msg);
+                    }
+                }
+            }
+        }
+
         /** {@inheritDoc} */
         @Override protected void cleanup() {
             super.cleanup();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiFailureTimeoutSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiFailureTimeoutSelfTest.java
index 35aa934e912a2..cc26c1cda6288 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiFailureTimeoutSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiFailureTimeoutSelfTest.java
@@ -185,6 +185,20 @@ public void testClientReconnectOnCoordinatorRouterFail2() throws Exception {
         clientReconnectOnCoordinatorRouterFail(2);
     }
 
+    /** {@inheritDoc} */
+    @Override public void testPingFailedClientNode() throws Exception {
+        int hb = maxMissedClientHbs;
+
+        maxMissedClientHbs = Integer.MAX_VALUE;
+
+        try {
+            super.testPingFailedClientNode();
+        }
+        finally {
+            maxMissedClientHbs = hb;
+        }
+    }
+
     /**
      * Test tries to provoke scenario when client sends reconnect message before router failure detected.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 0483a1ce8fbc6..419497753bbc2 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -142,7 +142,7 @@ public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     private boolean longSockTimeouts;
 
     /** */
-    private int maxMissedClientHbs = TcpDiscoverySpi.DFLT_MAX_MISSED_CLIENT_HEARTBEATS;
+    protected int maxMissedClientHbs = TcpDiscoverySpi.DFLT_MAX_MISSED_CLIENT_HEARTBEATS;
 
     /** */
     private IgniteInClosure2X<TcpDiscoveryAbstractMessage, Socket> afterWrite;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
new file mode 100644
index 0000000000000..7a1dd1ba23e5c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.Timer;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for missed client heartbeats.
+ */
+public class TcpDiscoveryClientSuspensionSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+        disco.setHeartbeatFrequency(200);
+        disco.setMaxMissedClientHeartbeats(10);
+
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY, "10000");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneServer() throws Exception {
+        doTestClientSuspension(1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTwoServers() throws Exception {
+        doTestClientSuspension(2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testThreeServers() throws Exception {
+        doTestClientSuspension(3);
+    }
+
+    /**
+     * @param serverCnt Servers count.
+     * @throws Exception If failed.
+     */
+    private void doTestClientSuspension(int serverCnt) throws Exception {
+        startGrids(serverCnt);
+
+        Ignition.setClientMode(true);
+
+        Ignite client = startGrid("client");
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(1, grid(i).cluster().forClients().nodes().size());
+
+        Thread.sleep(3000);
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(1, grid(i).cluster().forClients().nodes().size());
+
+        suspendClientHeartbeats(client);
+
+        Thread.sleep(3000);
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(0, grid(i).cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @param client Client.
+     */
+    private void suspendClientHeartbeats(Ignite client) {
+        assert client.cluster().localNode().isClient();
+
+        ClientImpl impl = U.field(client.configuration().getDiscoverySpi(), "impl");
+
+        Timer timer = U.field(impl, "timer");
+
+        timer.cancel();
+
+        System.out.println("Heartbeats suspended");
+    }
+}

From 45573945066113fd29548699f23c2bc9f22cef36 Mon Sep 17 00:00:00 2001
From: Tikhonov Nikolay <tikhonovnicolay@gmail.com>
Date: Wed, 21 Jun 2017 17:55:05 +0300
Subject: [PATCH 358/446] ignite-5489 Fixed possible connection leaks when
 loadPreviousValue set to true

---
 .../store/GridCacheStoreManagerAdapter.java   |   9 +-
 .../cache/CacheConnectionLeakStoreTxTest.java | 287 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   2 +
 3 files changed, 295 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 11d9816ff1e6d..c35b8fb9d2c01 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -338,7 +338,12 @@ private CacheStore cacheStoreWrapper(GridKernalContext ctx,
                 throw new IgniteCheckedException(new CacheLoaderException(e));
             }
             finally {
-                sessionEnd0(tx, threwEx);
+                IgniteInternalTx tx0 = tx;
+
+                if (tx0 != null && (tx0.dht() && tx0.local()))
+                    tx0 = null;
+
+                sessionEnd0(tx0, threwEx);
             }
 
             if (log.isDebugEnabled())
@@ -867,8 +872,6 @@ private void sessionEnd0(@Nullable IgniteInternalTx tx, boolean threwEx) throws
                         lsnr.onSessionEnd(locSes, !threwEx);
                 }
 
-                assert !sesHolder.get().ended(store);
-
                 store.sessionEnd(!threwEx);
             }
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java
new file mode 100644
index 0000000000000..f6b735077e1d8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.Serializable;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.cache.store.CacheStoreSession;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.resources.CacheStoreSessionResource;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.cache.TestCacheSession;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ *
+ */
+public class CacheConnectionLeakStoreTxTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int CLIENT_NODE = 1;
+
+    /** */
+    private static boolean client;
+
+    /** */
+    private static volatile boolean isLoadFromStore;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(disco);
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrid(0);
+
+        client = true;
+
+        startGrid(CLIENT_NODE);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        isLoadFromStore = false;
+        TestStore.sessions.clear();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupAtomic() throws Exception {
+        checkConnectionLeak(CacheAtomicityMode.ATOMIC, null, null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupAtomicLoadFromStore() throws Exception {
+        isLoadFromStore = true;
+
+        checkConnectionLeak(CacheAtomicityMode.ATOMIC, null, null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupOptimisticRepeatableRead() throws Exception {
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, OPTIMISTIC, REPEATABLE_READ);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupOptimisticRepeatableReadLoadFromStore() throws Exception {
+        isLoadFromStore = true;
+
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, OPTIMISTIC, REPEATABLE_READ);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupOptimisticReadCommitted() throws Exception {
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, OPTIMISTIC, READ_COMMITTED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupOptimisticReadCommittedLoadFromStore() throws Exception {
+        isLoadFromStore = true;
+
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, OPTIMISTIC, READ_COMMITTED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupPessimisticRepeatableRead() throws Exception {
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, PESSIMISTIC, REPEATABLE_READ);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupPessimisticReadCommitted() throws Exception {
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, PESSIMISTIC, READ_COMMITTED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupPessimisticReadCommittedLoadFromStore() throws Exception {
+        isLoadFromStore = true;
+
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, PESSIMISTIC, READ_COMMITTED);
+    }
+
+    /**
+     * @param atomicityMode Atomicity mode.
+     * @param txConcurrency Transaction concurrency.
+     * @param txIsolation Transaction isolation.
+     *
+     * @throws Exception If failed.
+     */
+    private void checkConnectionLeak(
+            CacheAtomicityMode atomicityMode,
+            TransactionConcurrency txConcurrency,
+            TransactionIsolation txIsolation
+    ) throws Exception {
+        CacheConfiguration<Integer, Integer> cacheCfg = new CacheConfiguration<>();
+
+        cacheCfg.setCacheMode(CacheMode.PARTITIONED);
+        cacheCfg.setAtomicityMode(atomicityMode);
+        cacheCfg.setCacheStoreFactory(new TestStoreFactory());
+        cacheCfg.setReadThrough(true);
+        cacheCfg.setWriteThrough(false);
+        cacheCfg.setLoadPreviousValue(true);
+
+        Ignite ignite = ignite(CLIENT_NODE);
+        IgniteCache<Integer, Integer> cache = ignite.createCache(cacheCfg);
+
+        try {
+            assertEquals(0, cache.size());
+
+            if (atomicityMode == CacheAtomicityMode.TRANSACTIONAL) {
+                try (Transaction tx = ignite.transactions().txStart(txConcurrency, txIsolation)) {
+                    cacheOp(cache);
+
+                    tx.commit();
+                }
+            }
+            else {
+                cacheOp(cache);
+            }
+
+            assertTrue("Session was leak on nodes: " + TestStore.sessions, TestStore.sessions.isEmpty());
+        }
+        finally {
+            cache.destroy();
+        }
+    }
+
+    /**
+     * @param cache Cache.
+     */
+    private void cacheOp(IgniteCache<Integer, Integer> cache) {
+        boolean b = cache.putIfAbsent(42, 42);
+
+        log.info("PutIfAbsent: " + b);
+
+        Integer val = cache.get(42);
+
+        log.info("Get: " + val);
+    }
+
+    /**
+     *
+     */
+    private static class TestStoreFactory implements Factory<CacheStoreAdapter<Integer, Integer>> {
+        /** {@inheritDoc} */
+        @Override public CacheStoreAdapter<Integer, Integer> create() {
+            return new TestStore();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestStore extends CacheStoreAdapter<Integer, Integer> implements Serializable {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** */
+        @CacheStoreSessionResource
+        private CacheStoreSession ses;
+
+        /** */
+        private CacheStoreSession NULL = new TestCacheSession();
+
+        /** */
+        public static ConcurrentHashMap<CacheStoreSession, ClusterNode> sessions = new ConcurrentHashMap<>();
+
+        /** {@inheritDoc} */
+        @Override public Integer load(Integer key) throws CacheLoaderException {
+            addSession();
+
+            return isLoadFromStore ? key : null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Integer, ? extends Integer> e) throws CacheWriterException {
+            addSession();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            addSession();
+        }
+
+        /**  */
+        private void addSession() {
+            sessions.put(ses == null ? NULL : ses, ignite.cluster().localNode());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sessionEnd(boolean commit) {
+            sessions.remove(ses == null ? NULL : ses);
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 2b446bbea8e6d..c1905af9db331 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.cache.store.jdbc.CacheJdbcStoreSessionListenerSelfTest;
 import org.apache.ignite.internal.processors.GridCacheTxLoadFromStoreOnLockSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheClientStoreSelfTest;
+import org.apache.ignite.internal.processors.cache.CacheConnectionLeakStoreTxTest;
 import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticReadCommittedSeltTest;
 import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticRepeatableReadSeltTest;
 import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticSerializableSeltTest;
@@ -279,6 +280,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheStoreUsageMultinodeStaticStartTxTest.class);
         suite.addTestSuite(CacheStoreUsageMultinodeDynamicStartAtomicTest.class);
         suite.addTestSuite(CacheStoreUsageMultinodeDynamicStartTxTest.class);
+        suite.addTestSuite(CacheConnectionLeakStoreTxTest.class);
 
         suite.addTestSuite(GridCacheStoreManagerDeserializationTest.class);
         suite.addTestSuite(GridLocalCacheStoreManagerDeserializationTest.class);

From 37535634ef3325aaf9923fd17d24038dfd5cee38 Mon Sep 17 00:00:00 2001
From: agura <agura@gridgain.com>
Date: Tue, 11 Jul 2017 16:24:54 +0300
Subject: [PATCH 359/446] ignite-5722 Cache entries stay in onheap after scan
 query execution for OFFHEAP_TIRED cache with expiry policy

---
 .../cache/query/GridCacheQueryManager.java    |   7 +-
 .../ScanQueryOffheapExpiryPolicySelfTest.java | 112 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite2.java       |   3 +
 3 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/ScanQueryOffheapExpiryPolicySelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 14b1106eb5e1c..7efb746d4428e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -3678,7 +3678,12 @@ private CacheObject value(GridCacheEntryEx entry, KeyCacheObject key) throws Ign
                     if (expiryPlc != null)
                         entry.unswap();
 
-                    return entry.peek(true, true, true, topVer, expiryPlc);
+                    CacheObject cacheObj = entry.peek(true, true, true, topVer, expiryPlc);
+
+                    if (expiryPlc != null)
+                        cctx.evicts().touch(entry, topVer);
+
+                    return cacheObj;
                 }
                 catch (GridCacheEntryRemovedException ignore) {
                     entry = null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/ScanQueryOffheapExpiryPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/ScanQueryOffheapExpiryPolicySelfTest.java
new file mode 100644
index 0000000000000..e59e458b2371c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/ScanQueryOffheapExpiryPolicySelfTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import javax.cache.expiry.CreatedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.ignite.cache.CachePeekMode.OFFHEAP;
+import static org.apache.ignite.cache.CachePeekMode.ONHEAP;
+
+/**
+ *
+ */
+public class ScanQueryOffheapExpiryPolicySelfTest extends GridCommonAbstractTest {
+
+    /** Nodes count. */
+    private static final int NODES_CNT = 2;
+
+    /** Entries count */
+    private static final int ENTRIES_CNT = 1024;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = defaultCacheConfiguration();
+
+        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+        ccfg.setCacheMode(CacheMode.PARTITIONED);
+        ccfg.setBackups(1);
+        ccfg.setMemoryMode(CacheMemoryMode.OFFHEAP_TIERED);
+        ccfg.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.MINUTES, 10)));
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGridsMultiThreaded(NODES_CNT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testEntriesMovedFromOnHeap() throws Exception {
+        Ignite ignite0 = grid(0);
+        Ignite ignite1 = grid(1);
+
+        IgniteCache<Integer, Integer> cache0 = ignite0.cache(null);
+        IgniteCache<Integer, Integer> cache1 = ignite1.cache(null);
+
+        populateCache(cache0);
+
+        assertEquals(0, cache0.localSize(ONHEAP));
+        assertEquals(0, cache1.localSize(ONHEAP));
+
+        assertEquals(ENTRIES_CNT, cache0.localSize(OFFHEAP) + cache1.localSize(OFFHEAP));
+
+        cache0.query(new ScanQuery<>()).getAll();
+        cache1.query(new ScanQuery<>()).getAll();
+
+        assertEquals(0, cache0.localSize(ONHEAP));
+        assertEquals(0, cache1.localSize(ONHEAP));
+
+        assertEquals(ENTRIES_CNT, cache0.localSize(OFFHEAP) + cache1.localSize(OFFHEAP));
+    }
+
+    /**
+     * @param cache Cache instance.
+     */
+    private static void populateCache(IgniteCache<Integer, Integer> cache) {
+        for (int i = 0; i < ENTRIES_CNT; i++)
+            cache.put(i, i);
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index 8ac219f89f199..0241f86198420 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -50,6 +50,7 @@
 import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalQueryCancelOrTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.query.GridCacheSwapScanQuerySelfTest;
+import org.apache.ignite.internal.processors.cache.query.ScanQueryOffheapExpiryPolicySelfTest;
 import org.apache.ignite.internal.processors.query.h2.sql.BaseH2CompareQueryTest;
 import org.apache.ignite.internal.processors.query.h2.sql.H2CompareBigQueryTest;
 import org.apache.ignite.spi.communication.tcp.GridOrderedMessageCancelSelfTest;
@@ -88,6 +89,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(CacheQueryOffheapEvictDataLostTest.class);
 
+        suite.addTestSuite(ScanQueryOffheapExpiryPolicySelfTest.class);
+
         // Ignite cache and H2 comparison.
         suite.addTestSuite(BaseH2CompareQueryTest.class);
         suite.addTestSuite(H2CompareBigQueryTest.class);

From c3e2eebeccbdc4bb3a7a0a70d09a8a7b63399c2c Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Tue, 18 Jul 2017 18:50:48 +0300
Subject: [PATCH 360/446] IGNITE 5776 Add option to turn on filter reachable
 addresses in TcpCommunicationSpi

---
 .../tcp/TcpCommunicationSpi.java              | 77 ++++++++++++++-----
 1 file changed, 56 insertions(+), 21 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 0d80f4778df36..7a5f7c1f16f3a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -313,6 +313,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     /** Default value for {@code TCP_NODELAY} socket option (value is <tt>true</tt>). */
     public static final boolean DFLT_TCP_NODELAY = true;
 
+    /** Default value for {@code FILTER_REACHABLE_ADDRESSES} socket option (value is <tt>false</tt>). */
+    public static final boolean DFLT_FILTER_REACHABLE_ADDRESSES = false;
+
     /** Default received messages threshold for sending ack. */
     public static final int DFLT_ACK_SND_THRESHOLD = 32;
 
@@ -996,6 +999,9 @@ class ConnectClosure implements IgniteInClosure<Boolean> {
     /** {@code TCP_NODELAY} option value for created sockets. */
     private boolean tcpNoDelay = DFLT_TCP_NODELAY;
 
+    /** {@code FILTER_REACHABLE_ADDRESSES} option value for created sockets. */
+    private boolean filterReachableAddresses = DFLT_FILTER_REACHABLE_ADDRESSES;
+
     /** Number of received messages after which acknowledgment is sent. */
     private int ackSndThreshold = DFLT_ACK_SND_THRESHOLD;
 
@@ -1515,6 +1521,33 @@ public void setTcpNoDelay(boolean tcpNoDelay) {
         return tcpNoDelay;
     }
 
+    /**
+     * Gets value for {@code FILTER_REACHABLE_ADDRESSES} socket option.
+     *
+     * @return {@code True} if needed to filter reachable addresses.
+     */
+    public boolean isFilterReachableAddresses() {
+        return filterReachableAddresses;
+    }
+
+    /**
+     * Setting this option to {@code true} enables filter for reachable
+     * addresses on creating tcp client.
+     * <p>
+     * Usually its advised to set this value to {@code false}.
+     * <p>
+     * If not provided, default value is {@link #DFLT_FILTER_REACHABLE_ADDRESSES}.
+     *
+     * @param filterReachableAddresses {@code True} to filter reachable addresses.
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public TcpCommunicationSpi setFilterReachableAddresses(boolean filterReachableAddresses) {
+        this.filterReachableAddresses = filterReachableAddresses;
+
+        return this;
+    }
+
     /**
      * Sets receive buffer size for sockets created or accepted by this SPI.
      * <p>
@@ -2780,35 +2813,37 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
         if (isExtAddrsExist)
             addrs.addAll(extAddrs);
 
-        Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
+        if (filterReachableAddresses) {
+            Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
 
-        for (InetSocketAddress addr : addrs) {
-            // Skip unresolved as addr.getAddress() can return null.
-            if(!addr.isUnresolved())
-                allInetAddrs.add(addr.getAddress());
-        }
+            for (InetSocketAddress addr : addrs) {
+                // Skip unresolved as addr.getAddress() can return null.
+                if (!addr.isUnresolved())
+                    allInetAddrs.add(addr.getAddress());
+            }
 
-        List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
+            List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
 
-        if (reachableInetAddrs.size() < allInetAddrs.size()) {
-            LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
+            if (reachableInetAddrs.size() < allInetAddrs.size()) {
+                LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
 
-            List<InetSocketAddress> unreachableInetAddr = new ArrayList<>(allInetAddrs.size() - reachableInetAddrs.size());
+                List<InetSocketAddress> unreachableInetAddr = new ArrayList<>(allInetAddrs.size() - reachableInetAddrs.size());
 
-            for (InetSocketAddress addr : addrs) {
-                if (reachableInetAddrs.contains(addr.getAddress()))
-                    addrs0.add(addr);
-                else
-                    unreachableInetAddr.add(addr);
-            }
+                for (InetSocketAddress addr : addrs) {
+                    if (reachableInetAddrs.contains(addr.getAddress()))
+                        addrs0.add(addr);
+                    else
+                        unreachableInetAddr.add(addr);
+                }
 
-            addrs0.addAll(unreachableInetAddr);
+                addrs0.addAll(unreachableInetAddr);
 
-            addrs = addrs0;
-        }
+                addrs = addrs0;
+            }
 
-        if (log.isDebugEnabled())
-            log.debug("Addresses to connect for node [rmtNode=" + node.id() + ", addrs=" + addrs.toString() + ']');
+            if (log.isDebugEnabled())
+                log.debug("Addresses to connect for node [rmtNode=" + node.id() + ", addrs=" + addrs.toString() + ']');
+        }
 
         boolean conn = false;
         GridCommunicationClient client = null;

From 97d3f42c1c95a6aafce1d0c300ccfe6708398c17 Mon Sep 17 00:00:00 2001
From: shtykh_roman <rshtykh@yahoo.com>
Date: Wed, 7 Sep 2016 08:35:31 +0300
Subject: [PATCH 361/446] IGNITE-3809: Fix for ArrayIndexOutOfBoundsException
 in GridUnsafeLru.

(cherry picked from commit 31b9bb8)
---
 .../util/offheap/unsafe/GridUnsafeLru.java    | 30 ++++++++++++++++---
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java
index aaff4f93a94f7..ea652171d8b2c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java
@@ -28,8 +28,7 @@
 /**
  * Striped LRU queue.
  */
-@SuppressWarnings("ForLoopReplaceableByForEach")
-class GridUnsafeLru {
+@SuppressWarnings("ForLoopReplaceableByForEach") class GridUnsafeLru {
     /** Number of stripes. */
     private final short cnt;
 
@@ -47,6 +46,9 @@ class GridUnsafeLru {
     /** Current round-robin remove stripe index. */
     private final AtomicInteger rmvIdx;
 
+    /** Max stripe index count. */
+    private final int maxIdxCnt;
+
     /** Released flag. */
     private AtomicBoolean released = new AtomicBoolean(false);
 
@@ -68,6 +70,8 @@ class GridUnsafeLru {
 
         addIdx = new AtomicInteger();
         rmvIdx = new AtomicInteger(cnt / 2);
+
+        maxIdxCnt = cnt - 1;
     }
 
     /**
@@ -156,7 +160,7 @@ long entry(short order, long qAddr) {
      * @throws GridOffHeapOutOfMemoryException If failed.
      */
     long offer(int part, long addr, int hash) throws GridOffHeapOutOfMemoryException {
-        return lrus[addIdx.getAndIncrement() % cnt].offer(part, addr, hash);
+        return lrus[incrementAndGet(addIdx, maxIdxCnt)].offer(part, addr, hash);
     }
 
     /**
@@ -165,7 +169,7 @@ long offer(int part, long addr, int hash) throws GridOffHeapOutOfMemoryException
      * @return Queue node address.
      */
     long prePoll() {
-        int idx = rmvIdx.getAndIncrement();
+        int idx = incrementAndGet(rmvIdx, maxIdxCnt);
 
         // Must try to poll from each LRU.
         for (int i = 0; i < lrus.length; i++) {
@@ -180,6 +184,7 @@ long prePoll() {
 
     /**
      * Removes polling node from the queue.
+     *
      * @param qAddr Queue node address.
      */
     void poll(long qAddr) {
@@ -215,6 +220,23 @@ void destruct() {
         }
     }
 
+    /**
+     * Atomically increments the given value by one, re-starting from 0 when the specified maximum is reached.
+     *
+     * @param value Value to increment.
+     * @param max Maximum after reaching which the value is reset to 0.
+     * @return Incremented value.
+     */
+    private int incrementAndGet(AtomicInteger value, int max) {
+        while (true) {
+            int cur = value.get();
+            int next = cur == max ? 0 : cur + 1;
+
+            if (value.compareAndSet(cur, next))
+                return next;
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridUnsafeLru.class, this);

From c2062d52a227dda5afee560d80c3bb4dd2ce09eb Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 19 Jul 2017 08:41:46 +0300
Subject: [PATCH 362/446] Remove empty test_utils.cpp

---
 modules/platforms/cpp/core-test/src/test_utils.cpp | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 modules/platforms/cpp/core-test/src/test_utils.cpp

diff --git a/modules/platforms/cpp/core-test/src/test_utils.cpp b/modules/platforms/cpp/core-test/src/test_utils.cpp
deleted file mode 100644
index e69de29bb2d1d..0000000000000

From 45cbba4853bab1ba4ffe2ea0d3add99a9d454aab Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 19 Jul 2017 10:44:04 +0300
Subject: [PATCH 363/446] IGNITE-5768 - Retry resolving class name from
 marshaller cache and .classname file.

---
 .../apache/ignite/IgniteSystemProperties.java |  3 +
 .../internal/MarshallerContextImpl.java       | 61 ++++++++++++++-----
 2 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index aaba91e15d304..d3c9a6b64c650 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -580,6 +580,9 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_ENABLE_FORCIBLE_NODE_KILL = "IGNITE_ENABLE_FORCIBLE_NODE_KILL";
 
+    /** Ignite marshaller cache reread pause. */
+    public static final String IGNITE_MARSHALLER_CACHE_REREAD_PAUSE = "IGNITE_MARSHALLER_CACHE_REREAD_PAUSE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java
index f3e368d859a04..dae0d4edad1d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java
@@ -38,6 +38,7 @@
 import javax.cache.event.CacheEntryUpdatedListener;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.internal.processors.cache.CachePartialUpdateCheckedException;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheTryPutFailedException;
@@ -47,10 +48,18 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.plugin.PluginProvider;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_MARSHALLER_CACHE_REREAD_PAUSE;
+
 /**
  * Marshaller context implementation.
  */
 public class MarshallerContextImpl extends MarshallerContextAdapter {
+    /** Cache reread tries. */
+    private static final int CACHE_REREAD_TRIES = 5;
+
+    /** Cache reread pause. */
+    private static final int CACHE_REREAD_PAUSE = IgniteSystemProperties.getInteger(IGNITE_MARSHALLER_CACHE_REREAD_PAUSE, 20);
+
     /** */
     private static final GridStripedLock fileLock = new GridStripedLock(32);
 
@@ -193,35 +202,59 @@ public void onKernalStop() {
                 throw new IllegalStateException("Failed to initialize marshaller context (grid is stopping).");
         }
 
-        String clsName = cache0.getTopologySafe(id);
+        String clsName = null;
+
+        for (int i = 0; i < CACHE_REREAD_TRIES; i++) {
+            clsName = cache0.getTopologySafe(id);
+
+            if (clsName != null)
+                break;
+
+            U.sleep(CACHE_REREAD_PAUSE);
+        }
 
         if (clsName == null) {
             String fileName = id + ".classname";
 
-            Lock lock = fileLock(fileName);
+            // Class name may be not in the file yet.
+            for (int i = 0; i < 2; i++) {
+                Lock lock = fileLock(fileName);
+
+                lock.lock();
 
-            lock.lock();
+                File file;
 
-            try {
-                File file = new File(workDir, fileName);
+                try {
+                    file = new File(workDir, fileName);
 
-                try (FileInputStream in = new FileInputStream(file)) {
-                    FileLock fileLock = fileLock(in.getChannel(), true);
+                    try (FileInputStream in = new FileInputStream(file)) {
+                        FileLock fileLock = fileLock(in.getChannel(), true);
 
-                    assert fileLock != null : fileName;
+                        assert fileLock != null : fileName;
 
-                    try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
-                        clsName = reader.readLine();
+                        try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
+                            clsName = reader.readLine();
+                        }
+                    }
+                    catch (IOException ignore) {
+                        // Will fail on last try.
                     }
                 }
-                catch (IOException e) {
+                finally {
+                    lock.unlock();
+                }
+
+                if (clsName != null)
+                    break;
+
+                // Fail on second unsuccessful try.
+                if (i == 1) {
                     throw new IgniteCheckedException("Class definition was not found " +
                         "at marshaller cache and local file. " +
                         "[id=" + id + ", file=" + file.getAbsolutePath() + ']');
                 }
-            }
-            finally {
-                lock.unlock();
+
+                U.sleep(20);
             }
 
             // Must explicitly put entry to cache to invoke other continuous queries.

From f24969f7e908645444df622642967a5f7fd3db23 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Wed, 19 Jul 2017 19:30:07 +0300
Subject: [PATCH 364/446] IGNITE 5775 JobsProcessor fix bug with delay in
 compute

---
 .../processors/job/GridJobProcessor.java      | 10 +--
 .../IgniteComputeJobOneThreadTest.java        | 79 +++++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |  2 +
 3 files changed, 86 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeJobOneThreadTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index ea9cbd7dbd272..09f80847832a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -1780,6 +1780,11 @@ private class JobEventListener implements GridJobEventListener {
                         return;
                     }
 
+                    if (!activeJobs.remove(worker.getJobId(), worker))
+                        cancelledJobs.remove(worker.getJobId(), worker);
+
+                    heldJobs.remove(worker.getJobId());
+
                     try {
                         handleCollisions();
                     }
@@ -1787,11 +1792,6 @@ private class JobEventListener implements GridJobEventListener {
                         rwLock.readUnlock();
                     }
                 }
-
-                if (!activeJobs.remove(worker.getJobId(), worker))
-                    cancelledJobs.remove(worker.getJobId(), worker);
-
-                heldJobs.remove(worker.getJobId());
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeJobOneThreadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeJobOneThreadTest.java
new file mode 100644
index 0000000000000..ccd8fa3765f14
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeJobOneThreadTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.spi.collision.fifoqueue.FifoQueueCollisionSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test of absence of gaps between jobs in compute
+ */
+public class IgniteComputeJobOneThreadTest extends GridCommonAbstractTest {
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        FifoQueueCollisionSpi colSpi = new FifoQueueCollisionSpi();
+        colSpi.setParallelJobsNumber(1);
+
+        return super.getConfiguration(name)
+            .setMetricsUpdateFrequency(10000)
+            .setCollisionSpi(colSpi);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        startGrid(0);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 10000;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoTimeout() throws Exception {
+        Ignite ignite = ignite(0);
+
+        IgniteFuture fut = null;
+
+        IgniteCompute compute = ignite.compute().withAsync();
+
+        for (int i = 0; i < 10000; i++) {
+            compute.run(new IgniteRunnable() {
+                @Override public void run() {
+
+                }
+            });
+        }
+
+        fut = compute.future();
+        fut.get();
+
+        assertTrue(true);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 8a501fdb801d6..c72bfe670b677 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -63,6 +63,7 @@
 import org.apache.ignite.internal.GridTaskResultCacheSelfTest;
 import org.apache.ignite.internal.GridTaskTimeoutSelfTest;
 import org.apache.ignite.internal.IgniteComputeEmptyClusterGroupTest;
+import org.apache.ignite.internal.IgniteComputeJobOneThreadTest;
 import org.apache.ignite.internal.IgniteComputeTopologyExceptionTest;
 import org.apache.ignite.internal.IgniteExecutorServiceTest;
 import org.apache.ignite.internal.IgniteExplicitImplicitDeploymentSelfTest;
@@ -152,6 +153,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TaskNodeRestartTest.class);
         suite.addTestSuite(IgniteRoundRobinErrorAfterClientReconnectTest.class);
         suite.addTestSuite(PublicThreadpoolStarvationTest.class);
+        suite.addTestSuite(IgniteComputeJobOneThreadTest.class);
 
         return suite;
     }

From f3adb9559b42698771b0b9b5116dd535446d2bef Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgain.com>
Date: Wed, 26 Jul 2017 15:40:19 +0700
Subject: [PATCH 365/446] IGNITE-5781 Visor throws ClassCastException if cache
 store implementation is other than CacheJdbcPojoStore.

---
 .../ignite/internal/visor/cache/VisorCacheTypeMetadata.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
index f17e5889663fc..a9ab8e061d3cb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
@@ -108,7 +108,7 @@ public static Collection<VisorCacheTypeMetadata> list(Collection<QueryEntity> qr
             }
 
         // Add JDBC types.
-        if (factory != null && factory instanceof CacheJdbcPojoStoreFactory) {
+        if (factory instanceof CacheJdbcPojoStoreFactory) {
              CacheJdbcPojoStoreFactory jdbcFactory = (CacheJdbcPojoStoreFactory) factory;
 
             JdbcType[] jdbcTypes = jdbcFactory.getTypes();

From a58688f6cc6c5b114dcdd1b2fde43b7e1e5e0732 Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgain.com>
Date: Wed, 26 Jul 2017 15:40:19 +0700
Subject: [PATCH 366/446] IGNITE-5781 Visor throws ClassCastException if cache
 store implementation is other than CacheJdbcPojoStore. (cherry picked from
 commit f3adb95)

---
 .../ignite/internal/visor/cache/VisorCacheTypeMetadata.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
index c87ad05d824e6..46d72c3e4e7ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
@@ -109,7 +109,7 @@ public static Collection<VisorCacheTypeMetadata> list(Collection<QueryEntity> qr
             }
 
         // Add JDBC types.
-        if (factory != null && factory instanceof CacheJdbcPojoStoreFactory) {
+        if (factory instanceof CacheJdbcPojoStoreFactory) {
              CacheJdbcPojoStoreFactory jdbcFactory = (CacheJdbcPojoStoreFactory) factory;
 
             JdbcType[] jdbcTypes = jdbcFactory.getTypes();

From b7d1fb25ceba20b82631bb2e926a0ad52bf19e9d Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 20 Jul 2017 17:43:17 +0300
Subject: [PATCH 367/446] Do not process partition exchange messages in striped
 pool.

(cherry picked from commit 3a33706)
---
 .../dht/preloader/GridDhtPartitionsAbstractMessage.java      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java
index 6e69161cb6ff2..34521e5cd2714 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java
@@ -65,6 +65,11 @@ protected GridDhtPartitionsAbstractMessage() {
         this.lastVer = lastVer;
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return Integer.MIN_VALUE;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean addDeploymentInfo() {
         return false;

From c338bb9f5ac8f34dccbac1f7058765c5ce4549a4 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 20 Jul 2017 17:49:55 +0300
Subject: [PATCH 368/446] Removed unnecessary discoCache.updateAlives.

(cherry picked from commit 07a0698)
---
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java          | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index d3e3701936786..ba47ffd435e06 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -460,8 +460,6 @@ public void init() throws IgniteInterruptedCheckedException {
         assert !dummy && !forcePreload : this;
 
         try {
-            discoCache.updateAlives(cctx.discovery());
-
             srvNodes = new ArrayList<>(discoCache.serverNodes());
 
             remaining.addAll(F.nodeIds(F.view(srvNodes, F.remoteNodes(cctx.localNodeId()))));

From 8c992fb8ba33a0c0ac5c0fb741ee8ffd515c0f31 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Tue, 1 Aug 2017 17:46:27 +0300
Subject: [PATCH 369/446] IGNITE-5775 Fix removing jobs from activeJobs for
 jobAlwaysActivate

---
 .../ignite/internal/processors/job/GridJobProcessor.java     | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index f88ea07d069bc..7889a5579c7bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -1769,6 +1769,11 @@ private class JobEventListener implements GridJobEventListener {
                 maxFinishedJobsTime.setIfGreater(execTime);
 
                 if (jobAlwaysActivate) {
+                    if (!activeJobs.remove(worker.getJobId(), worker))
+                        cancelledJobs.remove(worker.getJobId(), worker);
+
+                    heldJobs.remove(worker.getJobId());
+                    
                     if (metricsUpdateFreq > -1L)
                         updateJobMetrics();
                 }

From f0f1c82c8f9877d952f639ffe56803043e53415b Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 4 Aug 2017 14:58:52 +0300
Subject: [PATCH 370/446] IGNITE-5290 Partial back port commit from master
 (42293fa sboikov <sboikov@gridgain.com> on 29.05.2017 at 16:41)

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../CacheContinuousQueryHandler.java          | 152 +++++---
 .../CacheContinuousQueryListener.java         |   5 +
 .../CacheContinuousQueryManager.java          |  50 ++-
 .../continuous/GridContinuousProcessor.java   |   2 -
 .../IgniteCacheMessageWriteTimeoutTest.java   |   2 +
 ...ousQueryConcurrentPartitionUpdateTest.java | 341 ++++++++++++++++++
 ...ntinuousQueryFailoverAbstractSelfTest.java | 116 +-----
 ...CacheContinuousQueryOrderingEventTest.java |  21 ++
 .../IgniteCacheQuerySelfTestSuite3.java       |   3 +
 9 files changed, 513 insertions(+), 179 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 17f4308c0709b..165b8b77e0ab4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -141,7 +141,7 @@ public class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler
     private transient boolean keepBinary;
 
     /** */
-    private transient ConcurrentMap<Integer, PartitionRecovery> rcvs;
+    private transient ConcurrentMap<Integer, PartitionRecovery> rcvs = new ConcurrentHashMap<>();
 
     /** */
     private transient ConcurrentMap<Integer, EntryBuffer> entryBufs;
@@ -340,8 +340,6 @@ public void keepBinary(boolean keepBinary) {
 
         ackBuf = new AcknowledgeBuffer();
 
-        rcvs = new ConcurrentHashMap<>();
-
         this.nodeId = nodeId;
 
         this.routineId = routineId;
@@ -490,6 +488,16 @@ public void keepBinary(boolean keepBinary) {
                 sendBackupAcknowledge(ackBuf.acknowledgeOnTimeout(), routineId, ctx);
             }
 
+            @Override public void checkQueueOnTimeout(GridKernalContext ctx) {
+                for (PartitionRecovery rcv : rcvs.values()) {
+                    Collection<CacheEntryEvent<? extends K, ? extends V>> evts = rcv.onTimeout(
+                        ctx.cache().safeJcache(cacheName, cacheId), ctx.cache().context().cacheContext(cacheId));
+
+                    if (!evts.isEmpty())
+                        locLsnr.onUpdated(evts);
+                }
+            }
+
             @Override public void skipUpdateEvent(CacheContinuousQueryEvent<K, V> evt,
                 AffinityTopologyVersion topVer, boolean primary) {
                 assert evt != null;
@@ -557,20 +565,6 @@ private void prepareEntry(GridCacheContext cctx, UUID nodeId, CacheContinuousQue
             entry.prepareMarshal(cctx);
     }
 
-    /**
-     * Wait topology.
-     */
-    public void waitTopologyFuture(GridKernalContext ctx) throws IgniteCheckedException {
-        GridCacheContext<K, V> cctx = cacheContext(ctx);
-
-        if (!cctx.isLocal()) {
-            cacheContext(ctx).affinity().affinityReadyFuture(initTopVer).get();
-
-            for (int partId = 0; partId < cacheContext(ctx).affinity().partitions(); partId++)
-                getOrCreatePartitionRecovery(ctx, partId);
-        }
-    }
-
     /** {@inheritDoc} */
     @Override public void unregister(UUID routineId, GridKernalContext ctx) {
         assert routineId != null;
@@ -864,31 +858,7 @@ private String taskName() {
         PartitionRecovery rec = rcvs.get(partId);
 
         if (rec == null) {
-            Long partCntr = null;
-
-            AffinityTopologyVersion initTopVer0 = initTopVer;
-
-            if (initTopVer0 != null) {
-                GridCacheContext<K, V> cctx = cacheContext(ctx);
-
-                GridCacheAffinityManager aff = cctx.affinity();
-
-                if (initUpdCntrsPerNode != null) {
-                    for (ClusterNode node : aff.nodes(partId, initTopVer)) {
-                        Map<Integer, Long> map = initUpdCntrsPerNode.get(node.id());
-
-                        if (map != null) {
-                            partCntr = map.get(partId);
-
-                            break;
-                        }
-                    }
-                }
-                else if (initUpdCntrs != null)
-                    partCntr = initUpdCntrs.get(partId);
-            }
-
-            rec = new PartitionRecovery(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY), initTopVer0, partCntr);
+            rec = new PartitionRecovery(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY));
 
             PartitionRecovery oldRec = rcvs.putIfAbsent(partId, rec);
 
@@ -936,7 +906,7 @@ private CacheContinuousQueryEntry handleEntry(CacheContinuousQueryEntry e) {
     /**
      *
      */
-    private static class PartitionRecovery {
+    private static class PartitionRecovery<K, V> {
         /** Event which means hole in sequence. */
         private static final CacheContinuousQueryEntry HOLE = new CacheContinuousQueryEntry();
 
@@ -949,6 +919,12 @@ private static class PartitionRecovery {
         /** */
         private long lastFiredEvt;
 
+        /** */
+        private long prevCheckCntr = -1;
+
+        /** */
+        private int pendingSize;
+
         /** */
         private AffinityTopologyVersion curTop = AffinityTopologyVersion.NONE;
 
@@ -957,19 +933,9 @@ private static class PartitionRecovery {
 
         /**
          * @param log Logger.
-         * @param topVer Topology version.
-         * @param initCntr Update counters.
          */
-        PartitionRecovery(IgniteLogger log, AffinityTopologyVersion topVer, @Nullable Long initCntr) {
+        PartitionRecovery(IgniteLogger log) {
             this.log = log;
-
-            if (initCntr != null) {
-                assert topVer.topologyVersion() > 0 : topVer;
-
-                this.lastFiredEvt = initCntr;
-
-                curTop = topVer;
-            }
         }
 
         /**
@@ -1067,6 +1033,17 @@ private static class PartitionRecovery {
                         }
                     }
                 }
+                else if (!entry.isBackup() &&
+                    (entry.topologyVersion() != null && entry.topologyVersion().equals(curTop))) {
+
+                    if (log.isDebugEnabled())
+                        log.debug("Processed message out of order: " + entry);
+
+                    return !entry.isFiltered() ?
+                        F.<CacheEntryEvent<? extends K, ? extends V>>
+                            asList(new CacheContinuousQueryEvent<K, V>(cache, cctx, entry)) :
+                        Collections.<CacheEntryEvent<? extends K, ? extends V>>emptyList();
+                }
                 else {
                     if (log.isDebugEnabled())
                         log.debug("Skip duplicate continuous query message: " + entry);
@@ -1139,6 +1116,73 @@ private static class PartitionRecovery {
 
             return entries;
         }
+
+        /**
+         * Check queue on stuck.
+         *
+         * @param cache Cache.
+         * @param cctx Cache context.
+         *
+         * @return Stacked events.
+         */
+        Collection<CacheEntryEvent<? extends K, ? extends V>> onTimeout(IgniteCache cache, GridCacheContext cctx) {
+            List<CacheEntryEvent<? extends K, ? extends V>> res = Collections.emptyList();
+
+            synchronized (pendingEvts) {
+                // Visited this the first time.
+                if (prevCheckCntr == -1) {
+                    prevCheckCntr = lastFiredEvt;
+                    pendingSize = pendingEvts.size();
+
+                    return Collections.emptyList();
+                }
+
+                // Pending event not found.
+                if (pendingEvts.size() == 0) {
+                    prevCheckCntr = lastFiredEvt;
+                    pendingSize = 0;
+
+                    if (log.isDebugEnabled())
+                        log.debug("No stuck events.");
+
+                    return Collections.emptyList();
+                }
+
+                if (log.isDebugEnabled())
+                    log.debug("Check stuck events [lastFiredEvt=" + lastFiredEvt + ", "
+                        + "prevCheckCntr=" + prevCheckCntr + ", "
+                        + "pendingSize=" + pendingSize + ", "
+                        + "pendingEvts=" + pendingEvts + "]");
+
+                if (pendingSize > 0 && prevCheckCntr == lastFiredEvt) {
+                    int pendingSize0 = Math.min(pendingSize, pendingEvts.size());
+
+                    assert pendingSize0 > 0;
+
+                    res = new ArrayList<>(pendingSize0);
+                    Iterator<Map.Entry<Long, CacheContinuousQueryEntry>> iter = pendingEvts.entrySet().iterator();
+
+                    for (int i = 0; i < pendingSize0; i++) {
+                        Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
+
+                        if (e.getValue() != HOLE && !e.getValue().isFiltered())
+                            res.add(new CacheContinuousQueryEvent(cache, cctx, e.getValue()));
+
+                        iter.remove();
+
+                        lastFiredEvt = e.getKey();
+                    }
+                }
+
+                prevCheckCntr = lastFiredEvt;
+                pendingSize = pendingEvts.size();
+
+                if (log.isDebugEnabled())
+                    log.debug("Process stuck events [lastFiredEvt=" + lastFiredEvt + ", " + "evts=" + res + "]");
+            }
+
+            return res;
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
index 84b22f933e848..c9b486a5d9617 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
@@ -68,6 +68,11 @@ public void onEntryUpdated(CacheContinuousQueryEvent<K, V> evt, boolean primary,
      */
     public void acknowledgeBackupOnTimeout(GridKernalContext ctx);
 
+    /**
+     * @param ctx Context.
+     */
+    public void checkQueueOnTimeout(GridKernalContext ctx);
+
     /**
      * @param evt Event
      * @param topVer Topology version.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 91c199184a2de..a5f647aa3d2b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -99,6 +99,9 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
     /** */
     private static final long BACKUP_ACK_FREQ = 5000;
 
+    /** */
+    private static final long QUEUE_CHECKER_FREQ = 3000;
+
     /** Listeners. */
     private final ConcurrentMap<UUID, CacheContinuousQueryListener> lsnrs = new ConcurrentHashMap8<>();
 
@@ -139,6 +142,8 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
 
             cctx.time().schedule(new BackupCleaner(lsnrs, cctx.kernalContext()), BACKUP_ACK_FREQ, BACKUP_ACK_FREQ);
         }
+
+        cctx.time().schedule(new QueueChecker(lsnrs, cctx.kernalContext()), QUEUE_CHECKER_FREQ, QUEUE_CHECKER_FREQ);
     }
 
     /** {@inheritDoc} */
@@ -655,18 +660,6 @@ private UUID executeQuery0(CacheEntryUpdatedListener locLsnr,
             autoUnsubscribe,
             pred).get();
 
-        try {
-            if (hnd.isQuery() && cctx.userCache() && !onStart)
-                hnd.waitTopologyFuture(cctx.kernalContext());
-        }
-        catch (IgniteCheckedException e) {
-            log.warning("Failed to start continuous query.", e);
-
-            cctx.kernalContext().continuous().stopRoutine(id);
-
-            throw new IgniteCheckedException("Failed to start continuous query.", e);
-        }
-
         if (notifyExisting) {
             final Iterator<GridCacheEntryEx> it = cctx.cache().allEntries().iterator();
 
@@ -1196,8 +1189,37 @@ public BackupCleaner(Map<UUID, CacheContinuousQueryListener> lsnrs, GridKernalCo
 
         /** {@inheritDoc} */
         @Override public void run() {
-            for (CacheContinuousQueryListener lsnr : lsnrs.values())
-                lsnr.acknowledgeBackupOnTimeout(ctx);
+            if (!lsnrs.isEmpty()) {
+                for (CacheContinuousQueryListener lsnr : lsnrs.values())
+                    lsnr.acknowledgeBackupOnTimeout(ctx);
+            }
+        }
+    }
+
+    /**
+     * Task flash backup queue.
+     */
+    private static final class QueueChecker implements Runnable {
+        /** Listeners. */
+        private final Map<UUID, CacheContinuousQueryListener> lsnrs;
+
+        /** Context. */
+        private final GridKernalContext ctx;
+
+        /**
+         * @param lsnrs Listeners.
+         */
+        QueueChecker(Map<UUID, CacheContinuousQueryListener> lsnrs, GridKernalContext ctx) {
+            this.lsnrs = lsnrs;
+            this.ctx = ctx;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            if (!lsnrs.isEmpty()) {
+                for (CacheContinuousQueryListener lsnr : lsnrs.values())
+                    lsnr.checkQueueOnTimeout(ctx);
+            }
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index f0429bc3a3c91..50d29d1f8de6e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -240,8 +240,6 @@ public GridContinuousProcessor(GridKernalContext ctx) {
 
                                 if (cctx != null && cntrsPerNode != null && !cctx.isLocal() && cctx.affinityNode())
                                     cntrsPerNode.put(ctx.localNodeId(), cctx.topology().updateCounters(false));
-
-                                routine.handler().updateCounters(topVer, cntrsPerNode, cntrs);
                             }
 
                             fut.onRemoteRegistered();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index 6256225f0a54e..339e39ac4e98b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -70,6 +70,8 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testMessageQueueLimit() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-12398");
+
         startGridsMultiThreaded(3);
 
         for (int i = 0; i < 15; i++) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
new file mode 100644
index 0000000000000..c1675f328d706
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryUpdatedListener;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.query.CacheQueryEntryEvent;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static String DEFAULT_CACHE_NAME = "cache-name";
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi) cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatePartitionAtomic() throws Exception {
+        concurrentUpdatePartition(ATOMIC, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatePartitionTx() throws Exception {
+        concurrentUpdatePartition(TRANSACTIONAL, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatePartitionAtomicClient() throws Exception {
+        concurrentUpdatePartition(ATOMIC, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatePartitionTxClient() throws Exception {
+        concurrentUpdatePartition(TRANSACTIONAL, true);
+    }
+
+    /**
+     * @param atomicityMode Cache atomicity mode.
+     * @param clientMode Client mode.
+     * @throws Exception If failed.
+     */
+    private void concurrentUpdatePartition(CacheAtomicityMode atomicityMode, boolean clientMode) throws Exception {
+        Ignite srv = startGrid(0);
+
+        client = clientMode;
+
+        Ignite client = startGrid(1);
+
+        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+        ccfg.setAtomicityMode(atomicityMode);
+
+        IgniteCache clientCache = client.createCache(ccfg);
+
+        final AtomicInteger evtCnt = new AtomicInteger();
+
+        Affinity<Integer> aff = srv.affinity(DEFAULT_CACHE_NAME);
+
+        final List<Integer> keys = new ArrayList<>();
+
+        final int KEYS = 1000;
+
+        for (int i = 0; i < 10_000_000; i++) {
+            if (aff.partition(i) == 0) {
+                keys.add(i);
+
+                if (keys.size() == KEYS)
+                    break;
+            }
+        }
+
+        assertEquals(KEYS, keys.size());
+
+        final int THREADS = 10;
+        final int UPDATES = 1000;
+
+        final IgniteCache<Object, Object> cache0 = clientMode ? client.cache(DEFAULT_CACHE_NAME)
+            : srv.cache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < 10; i++) {
+            log.info("Iteration: " + i);
+
+            ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+            final ConcurrentSkipListSet<Long> sets = new ConcurrentSkipListSet<>();
+
+            qry.setLocalListener(new CacheEntryUpdatedListener<Object, Object>() {
+                @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts) {
+                    for (CacheEntryEvent evt : evts) {
+                        assertNotNull(evt.getKey());
+                        assertNotNull(evt.getValue());
+
+                        CacheQueryEntryEvent e = (CacheQueryEntryEvent)evt.unwrap(CacheQueryEntryEvent.class);
+
+                        sets.add(e.getPartitionUpdateCounter());
+
+                        evtCnt.incrementAndGet();
+                    }
+                }
+            });
+
+            QueryCursor qryCur = clientCache.query(qry);
+
+            GridTestUtils.runMultiThreaded(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                    for (int i = 0; i < UPDATES; i++)
+                        cache0.put(keys.get(rnd.nextInt(KEYS)), i);
+
+                    return null;
+                }
+            }, THREADS, "update");
+
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    log.info("Events: " + evtCnt.get());
+
+                    return evtCnt.get() >= THREADS * UPDATES;
+                }
+            }, 5000);
+
+            assertEquals(THREADS * UPDATES, evtCnt.get());
+
+            qryCur.close();
+
+            evtCnt.set(0);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatesAndQueryStartAtomic() throws Exception {
+        concurrentUpdatesAndQueryStart(ATOMIC);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatesAndQueryStartTx() throws Exception {
+        concurrentUpdatesAndQueryStart(TRANSACTIONAL);
+    }
+
+    /**
+     * @param atomicityMode Cache atomicity mode.
+     * @throws Exception If failed.
+     */
+    private void concurrentUpdatesAndQueryStart(CacheAtomicityMode atomicityMode) throws Exception {
+        Ignite srv = startGrid(0);
+
+        client = true;
+
+        Ignite client = startGrid(1);
+
+        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+        ccfg.setAtomicityMode(atomicityMode);
+
+        IgniteCache clientCache = this.client ? client.createCache(ccfg) : srv.createCache(ccfg);
+
+        Affinity<Integer> aff = srv.affinity(DEFAULT_CACHE_NAME);
+
+        final List<Integer> keys = new ArrayList<>();
+
+        final int KEYS = 1_000;
+
+        for (int i = 0; i < 10_000_000; i++) {
+            if (aff.partition(i) == 0) {
+                keys.add(i);
+
+                if (keys.size() == KEYS)
+                    break;
+            }
+        }
+
+        assertEquals(KEYS, keys.size());
+
+        final int THREADS = 10;
+        final int UPDATES = 100;
+
+        for (int i = 0; i < 5; i++) {
+            log.info("Iteration: " + i);
+
+            ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+            final AtomicInteger evtCnt = new AtomicInteger();
+
+            final ConcurrentSkipListSet<Long> sets = new ConcurrentSkipListSet<>();
+
+            qry.setLocalListener(new CacheEntryUpdatedListener<Object, Object>() {
+                @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts) {
+                    for (CacheEntryEvent evt : evts) {
+                        assertNotNull(evt.getKey());
+                        assertNotNull(evt.getValue());
+
+                        CacheQueryEntryEvent e = (CacheQueryEntryEvent)evt.unwrap(CacheQueryEntryEvent.class);
+
+                        sets.add(e.getPartitionUpdateCounter());
+
+                        if ((Integer)evt.getValue() >= 0)
+                            evtCnt.incrementAndGet();
+                    }
+                }
+            });
+
+            QueryCursor cur;
+
+            final IgniteCache<Object, Object> srvCache = srv.cache(DEFAULT_CACHE_NAME);
+
+            final AtomicBoolean stop = new AtomicBoolean();
+
+            try {
+                IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+                    @Override public Void call() throws Exception {
+                        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                        while (!stop.get())
+                            srvCache.put(keys.get(rnd.nextInt(KEYS)), rnd.nextInt(100) - 200);
+
+                        return null;
+                    }
+                }, THREADS, "update");
+
+                U.sleep(1000);
+
+                cur = clientCache.query(qry);
+
+                U.sleep(1000);
+
+                stop.set(true);
+
+                fut.get();
+            }
+            finally {
+                stop.set(true);
+            }
+
+            GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                    for (int i = 0; i < UPDATES; i++)
+                        srvCache.put(keys.get(rnd.nextInt(KEYS)), i);
+
+                    return null;
+                }
+            }, THREADS, "update");
+
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    log.info("Events: " + evtCnt.get());
+
+                    return evtCnt.get() >= THREADS * UPDATES;
+                }
+            }, 10000);
+
+            if ((THREADS * UPDATES) != evtCnt.get())
+                log.info("Entries: " + sets);
+
+            assertEquals(THREADS * UPDATES, evtCnt.get());
+
+            cur.close();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
index d2cb710edbeb5..ffb7557ff1258 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
@@ -142,7 +142,6 @@ public abstract class CacheContinuousQueryFailoverAbstractSelfTest extends GridC
         TestCommunicationSpi commSpi = new TestCommunicationSpi();
 
         commSpi.setSharedMemoryPort(-1);
-        commSpi.setIdleConnectionTimeout(100);
 
         cfg.setCommunicationSpi(commSpi);
 
@@ -1387,7 +1386,7 @@ public void testBackupQueueCleanupClientQuery() throws Exception {
             @Override public boolean apply() {
                 return backupQueue.isEmpty();
             }
-        }, 2000);
+        }, 10000);
 
         assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.size() < BACKUP_ACK_THRESHOLD);
 
@@ -1409,13 +1408,13 @@ public void testBackupQueueCleanupClientQuery() throws Exception {
             @Override public boolean apply() {
                 return backupQueue.isEmpty();
             }
-        }, ACK_FREQ + 2000);
-
-        assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.isEmpty());
+        }, ACK_FREQ + 20000);
 
         if (!latch.await(5, SECONDS))
             fail("Failed to wait for notifications [exp=" + keys.size() + ", left=" + lsnr.latch.getCount() + ']');
 
+        assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.isEmpty());
+
         cur.close();
 
         assertFalse("Unexpected error during test, see log for details.", err);
@@ -1465,9 +1464,7 @@ public void testBackupQueueEvict() throws Exception {
             @Override public boolean apply() {
                 return backupQueue.isEmpty();
             }
-        }, 2000);
-
-        assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.size() < BACKUP_ACK_THRESHOLD);
+        }, 20000);
 
         boolean wait = waitForCondition(new GridAbsPredicate() {
             @Override public boolean apply() {
@@ -1477,6 +1474,8 @@ public void testBackupQueueEvict() throws Exception {
 
         assertTrue("Entry evicted.", wait);
 
+        assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.size() < BACKUP_ACK_THRESHOLD);
+
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
             @Override public boolean apply() {
                 return backupQueue.isEmpty();
@@ -2076,107 +2075,6 @@ public void testMultiThreadedFailover() throws Exception {
         assertFalse("Unexpected error during test, see log for details.", err);
     }
 
-    /**
-     * @throws Exception If failed.
-     */
-    public void testMultiThreaded() throws Exception {
-        this.backups = 2;
-
-        final int SRV_NODES = 3;
-
-        startGridsMultiThreaded(SRV_NODES);
-
-        client = true;
-
-        Ignite qryClient = startGrid(SRV_NODES);
-
-        final IgniteCache<Object, Object> cache = qryClient.cache(null);
-
-        CacheEventListener1 lsnr = new CacheEventListener1(true);
-
-        ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
-
-        qry.setLocalListener(lsnr);
-
-        QueryCursor<?> cur = cache.query(qry);
-
-        client = false;
-
-        final int SRV_IDX = SRV_NODES - 1;
-
-        List<Integer> keys = primaryKeys(ignite(SRV_IDX).cache(null), 10);
-
-        final int THREADS = 10;
-
-        for (int i = 0; i < keys.size(); i++) {
-            log.info("Iteration: " + i);
-
-            Ignite srv = ignite(SRV_IDX);
-
-            TestCommunicationSpi spi = (TestCommunicationSpi)srv.configuration().getCommunicationSpi();
-
-            spi.sndFirstOnly = new AtomicBoolean(false);
-
-            final Integer key = keys.get(i);
-
-            final AtomicInteger val = new AtomicInteger();
-
-            CountDownLatch latch = new CountDownLatch(THREADS);
-
-            lsnr.latch = latch;
-
-            IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
-                @Override public Object call() throws Exception {
-                    Integer val0 = val.getAndIncrement();
-
-                    cache.put(key, val0);
-
-                    return null;
-                }
-            }, THREADS, "update-thread");
-
-            fut.get();
-
-            stopGrid(SRV_IDX);
-
-            if (!latch.await(5, SECONDS))
-                fail("Failed to wait for notifications [exp=" + THREADS + ", left=" + lsnr.latch.getCount() + ']');
-
-            assertEquals(THREADS, lsnr.allEvts.size());
-
-            Set<Integer> vals = new HashSet<>();
-
-            boolean err = false;
-
-            for (CacheEntryEvent<?, ?> evt : lsnr.allEvts) {
-                assertEquals(key, evt.getKey());
-                assertNotNull(evt.getValue());
-
-                if (!vals.add((Integer)evt.getValue())) {
-                    err = true;
-
-                    log.info("Extra event: " + evt);
-                }
-            }
-
-            for (int v = 0; v < THREADS; v++) {
-                if (!vals.contains(v)) {
-                    err = true;
-
-                    log.info("Event for value not received: " + v);
-                }
-            }
-
-            assertFalse("Invalid events, see log for details.", err);
-
-            lsnr.allEvts.clear();
-
-            startGrid(SRV_IDX);
-        }
-
-        cur.close();
-    }
-
     /**
      * @param logAll If {@code true} logs all unexpected values.
      * @param expEvts Expected values.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
index 7d975f2a28209..4dcdd32745ac0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.Cache;
 import javax.cache.configuration.FactoryBuilder;
 import javax.cache.event.CacheEntryEvent;
 import javax.cache.event.CacheEntryUpdatedListener;
@@ -49,6 +50,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteAsyncCallback;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
@@ -391,6 +393,25 @@ protected void doOrderingTest(
                 }
             }
 
+            IgniteCache<Object, Object> cache = grid(0).cache(ccfg.getName());
+
+            for (int i = 0; i < KEYS; i++) {
+                cache.put(new QueryTestKey(i), new QueryTestValue(-1));
+
+                cache.remove(new QueryTestValue(i));
+            }
+
+            GridTestUtils.waitForCondition(new PA() {
+                @Override public boolean apply() {
+                    return qryCntr.get() >= 4 * KEYS * LISTENER_CNT * NODES;
+                }
+            }, 3000L);
+
+            for (BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>> q : rcvdEvts)
+                q.clear();
+
+            qryCntr.set(0);
+
             IgniteInternalFuture<Long> f = GridTestUtils.runMultiThreadedAsync(new Runnable() {
                 @Override public void run() {
                     ThreadLocalRandom rnd = ThreadLocalRandom.current();
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
index 032dd3b5d8426..47de0ff842418 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchAckTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchForceServerModeAckTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFilterListenerTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryConcurrentPartitionUpdateTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryExecuteInPrimaryTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryAsyncFilterRandomOperationTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryFilterRandomOperationTest;
@@ -122,6 +123,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheKeepBinaryIterationNearEnabledTest.class);
         suite.addTestSuite(IgniteCacheContinuousQueryBackupQueueTest.class);
 
+        suite.addTestSuite(CacheContinuousQueryConcurrentPartitionUpdateTest.class);
+
         return suite;
     }
 }

From 3d64cf1a023ce71cf522b6ce843669fcf44a3145 Mon Sep 17 00:00:00 2001
From: Ivan Rakov <ivan.glukos@gmail.com>
Date: Thu, 30 Mar 2017 14:56:40 +0300
Subject: [PATCH 371/446] IGNITE-3477 - Fixed concurrent read-through

(cherry picked from commit 31ffef4)
---
 .../ignite/internal/processors/cache/GridCacheAdapter.java  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 7c5a54dfcb852..7aa48199bb28a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2014,9 +2014,13 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                         CacheObject cacheVal = ctx.toCacheObject(val);
 
                                         while (true) {
-                                            GridCacheEntryEx entry = entryEx(key);
+                                            GridCacheEntryEx entry = null;
 
                                             try {
+                                                entry = entryEx(key);
+
+                                                entry.unswap();
+
                                                 EntryGetResult verVal = entry.versionedValue(
                                                     cacheVal,
                                                     res.version(),

From 9b9eabd97494f291444f42159cb5977b407dda8c Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 9 Aug 2017 18:58:02 +0700
Subject: [PATCH 372/446] IGNITE-5987 Added -nq (visor will not quit in batch
 mode) option for Visor Cmd.

---
 .../apache/ignite/visor/commands/VisorConsole.scala  | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
index ad8c2edaaf8dd..31b4ff06b00dc 100644
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
@@ -93,6 +93,7 @@ class VisorConsole {
             println("        -cfg=<path>          - connect with specified configuration.")
             println("        -b=<path>            - batch mode with file.")
             println("        -e=cmd1;cmd2;...     - batch mode with commands.")
+            println("        -nq                  - batch mode will not quit after execution (useful for alerts monitoring).")
 
             visor.quit()
         }
@@ -104,6 +105,10 @@ class VisorConsole {
         val cfgFile = argValue("cfg", argLst)
         val batchFile = argValue("b", argLst)
         val batchCommand = argValue("e", argLst)
+        val noBatchQuit = hasArgName("nq", argLst)
+
+        if (noBatchQuit && batchFile.isEmpty && batchCommand.isEmpty)
+            visor.warn("Option \"-nq\" will be ignored because batch mode options \"-b\" or \"-e\" were not specified.")
 
         cfgFile.foreach(cfg => {
             if (cfg.trim.isEmpty) {
@@ -150,7 +155,10 @@ class VisorConsole {
             case Some(cmd) =>
                 visor.batchMode = true
 
-                new ByteArrayInputStream((cmd + "\nquit\n").getBytes("UTF-8"))
+                val script = if (noBatchQuit) cmd else cmd + "\nquit\n"
+
+                new ByteArrayInputStream(script.getBytes("UTF-8"))
+
             case None => new FileInputStream(FileDescriptor.in)
         }
 
@@ -160,7 +168,7 @@ class VisorConsole {
 
             new TerminalSupport(false) {}
         } catch {
-            case ignored: ClassNotFoundException => null
+            case _: ClassNotFoundException => null
         }
 
         val reader = new ConsoleReader(inputStream, System.out, term)

From 737260b070c3178d91ef04f7b200da87c9a1874f Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 10 Aug 2017 18:54:57 +0300
Subject: [PATCH 373/446] IGNITE-4991 Do not print out system properties when
 IGNITE_TO_STRING_INCLUDE_SENSITIVE is set to false

---
 .../main/java/org/apache/ignite/internal/IgniteKernal.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 9bada6c35ce3b..a1997b1adfed7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2213,7 +2213,7 @@ public <K, V> GridCacheAdapter<K, V> internalCache(@Nullable String name) {
     private void ackSystemProperties() {
         assert log != null;
 
-        if (log.isDebugEnabled())
+        if (log.isDebugEnabled() && S.INCLUDE_SENSITIVE)
             for (Map.Entry<Object, Object> entry : snapshot().entrySet())
                 log.debug("System property [" + entry.getKey() + '=' + entry.getValue() + ']');
     }
@@ -2384,7 +2384,7 @@ private void ackVmArguments(RuntimeMXBean rtBean) {
         assert log != null;
 
         // Ack IGNITE_HOME and VM arguments.
-        if (log.isInfoEnabled()) {
+        if (log.isInfoEnabled() && S.INCLUDE_SENSITIVE) {
             log.info("IGNITE_HOME=" + cfg.getIgniteHome());
             log.info("VM arguments: " + rtBean.getInputArguments());
         }

From d4f4a323a80193dffabff96b45d0f44e5ee22d9b Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Mon, 14 Aug 2017 16:44:51 +0300
Subject: [PATCH 374/446] Merge branch 'ignite-1.7.14' into ignite-1.8.10

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../CacheContinuousQueryConcurrentPartitionUpdateTest.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
index c1675f328d706..36d1dc44aabf4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
@@ -82,7 +82,7 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
     /**
      * @throws Exception If failed.
      */
-    public void testConcurrentUpdatePartitionAtomic() throws Exception {
+    public void _testConcurrentUpdatePartitionAtomic() throws Exception {
         concurrentUpdatePartition(ATOMIC, false);
     }
 
@@ -96,7 +96,7 @@ public void testConcurrentUpdatePartitionTx() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testConcurrentUpdatePartitionAtomicClient() throws Exception {
+    public void _testConcurrentUpdatePartitionAtomicClient() throws Exception {
         concurrentUpdatePartition(ATOMIC, true);
     }
 
@@ -205,7 +205,7 @@ private void concurrentUpdatePartition(CacheAtomicityMode atomicityMode, boolean
     /**
      * @throws Exception If failed.
      */
-    public void testConcurrentUpdatesAndQueryStartAtomic() throws Exception {
+    public void _testConcurrentUpdatesAndQueryStartAtomic() throws Exception {
         concurrentUpdatesAndQueryStart(ATOMIC);
     }
 

From b7bf1c09e8404d10eac57c13c9c6720c040d0c8e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 14 Jul 2017 20:14:47 +0300
Subject: [PATCH 375/446] IGNITE-5452: GridTimeoutProcessor can hang on stop.
 This closes #2279.

(cherry picked from commit b95c261)
---
 .../timeout/GridTimeoutProcessor.java         |  18 +-
 .../IgniteTxRemoveTimeoutObjectsTest.java     | 194 ++++++++++++++++++
 .../timeout/GridTimeoutProcessorSelfTest.java |  68 ++++--
 .../testsuites/IgniteCacheTestSuite3.java     |   3 +
 4 files changed, 265 insertions(+), 18 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index 0bbf9c376fc60..7fabd5cadb6a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -37,7 +37,7 @@
  */
 public class GridTimeoutProcessor extends GridProcessorAdapter {
     /** */
-    private final IgniteThread timeoutWorker;
+    private final TimeoutWorker timeoutWorker;
 
     /** Time-based sorted set for timeout objects. */
     private final GridConcurrentSkipListSet<GridTimeoutObject> timeoutObjs =
@@ -62,13 +62,12 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
     public GridTimeoutProcessor(GridKernalContext ctx) {
         super(ctx);
 
-        timeoutWorker = new IgniteThread(ctx.config().getGridName(), "grid-timeout-worker",
-            new TimeoutWorker());
+        timeoutWorker = new TimeoutWorker();
     }
 
     /** {@inheritDoc} */
     @Override public void start() {
-        timeoutWorker.start();
+        new IgniteThread(timeoutWorker).start();
 
         if (log.isDebugEnabled())
             log.debug("Timeout processor started.");
@@ -76,7 +75,7 @@ public GridTimeoutProcessor(GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public void stop(boolean cancel) throws IgniteCheckedException {
-        U.interrupt(timeoutWorker);
+        timeoutWorker.cancel();
         U.join(timeoutWorker);
 
         if (log.isDebugEnabled())
@@ -159,6 +158,13 @@ private class TimeoutWorker extends GridWorker {
                             timeoutObj.onTimeout();
                         }
                         catch (Throwable e) {
+                            if (isCancelled() && !(e instanceof Error)){
+                                if (log.isDebugEnabled())
+                                    log.debug("Error when executing timeout callback: " + timeoutObj);
+
+                                return;
+                            }
+
                             U.error(log, "Error when executing timeout callback: " + timeoutObj, e);
 
                             if (e instanceof Error)
@@ -170,7 +176,7 @@ private class TimeoutWorker extends GridWorker {
                 }
 
                 synchronized (mux) {
-                    while (true) {
+                    while (!isCancelled()) {
                         // Access of the first element must be inside of
                         // synchronization block, so we don't miss out
                         // on thread notification events sent from
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java
new file mode 100644
index 0000000000000..bdb73ad303448
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionTimeoutException;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
+
+/**
+ * Test correctness of rollback a transaction with timeout during the grid stop.
+ */
+public class IgniteTxRemoveTimeoutObjectsTest extends GridCacheAbstractSelfTest {
+    /** */
+    private static final int PUT_CNT = 1000;
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 3;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 60_000;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxRemoveTimeoutObjects() throws Exception {
+        IgniteCache<Integer, Integer> cache0 = grid(0).cache(null);
+        IgniteCache<Integer, Integer> cache1 = grid(1).cache(null);
+
+        // start additional grid to be closed.
+        IgniteCache<Integer, Integer> cacheAdditional = startGrid(gridCount()).cache(null);
+
+        for (int i = 0; i < PUT_CNT; i++)
+            cache0.put(i, Integer.MAX_VALUE);
+
+        logTimeoutObjectsFrequency();
+
+        info("Tx1 started");
+        try (Transaction tx = grid(gridCount()).transactions().txStart(PESSIMISTIC, SERIALIZABLE, 100, PUT_CNT)) {
+            try {
+                for (int i = 0; i < PUT_CNT; i++) {
+                    cacheAdditional.put(i, Integer.MIN_VALUE);
+
+                    if (i % 100 == 0)
+                        logTimeoutObjectsFrequency();
+                }
+
+                U.sleep(200);
+
+                tx.commit();
+
+                fail("A timeout should have happened.");
+            }
+            catch (Exception e) {
+                assertTrue(X.hasCause(e, TransactionTimeoutException.class));
+            }
+        }
+
+        assertDoesNotContainLockTimeoutObjects();
+
+        logTimeoutObjectsFrequency();
+
+        stopGrid(gridCount());
+
+        awaitPartitionMapExchange();
+
+        info("Grid2 closed.");
+
+        assertDoesNotContainLockTimeoutObjects();
+
+        logTimeoutObjectsFrequency();
+
+        // Check that the values have not changed and lock can be acquired.
+        try (Transaction tx2 = grid(1).transactions().txStart(PESSIMISTIC, SERIALIZABLE)) {
+            info("Tx2 started");
+
+            for (int i = 0; i < PUT_CNT; i++) {
+                assertEquals(cache1.get(i).intValue(), Integer.MAX_VALUE);
+                cache1.put(i, i);
+
+                if (i % (PUT_CNT / 5) == 0)
+                    logTimeoutObjectsFrequency();
+            }
+
+            tx2.commit();
+        }
+
+        info("Tx2 stopped");
+
+        // Check that that changes committed.
+        for (int i = 0; i < PUT_CNT; i++)
+            assertEquals(cache0.get(i).intValue(), i);
+    }
+
+    /**
+     * Fails if at least one grid contains LockTimeoutObjects.
+     */
+    private void assertDoesNotContainLockTimeoutObjects() {
+        for (Ignite ignite : G.allGrids()) {
+            for (GridTimeoutObject object : getTimeoutObjects((IgniteEx)ignite)) {
+                if (object.getClass().getSimpleName().equals("LockTimeoutObject"))
+                    fail("Grids contain LockTimeoutObjects.");
+            }
+        }
+    }
+
+    /**
+     * Print the number of each timeout object type on each grid to the log.
+     */
+    private void logTimeoutObjectsFrequency() {
+        StringBuilder sb = new StringBuilder("Timeout objects frequency [");
+
+        for (Ignite ignite : G.allGrids()) {
+            IgniteEx igniteEx = (IgniteEx)ignite;
+
+            Map<String, Integer> objFreqMap = new HashMap<>();
+
+            Set<GridTimeoutObject> objs = getTimeoutObjects(igniteEx);
+
+            for (GridTimeoutObject obj : objs) {
+                String clsName = obj.getClass().getSimpleName();
+
+                Integer cnt = objFreqMap.get(clsName);
+
+                if (cnt == null)
+                    objFreqMap.put(clsName, 1);
+                else
+                    objFreqMap.put(clsName, cnt + 1);
+            }
+
+            sb.append("[")
+                .append(igniteEx.name()).append(": size=")
+                .append(objs.size()).append(", ");
+
+            for (Map.Entry<String, Integer> entry : objFreqMap.entrySet()) {
+                sb.append(entry.getKey()).append("=")
+                    .append(entry.getValue())
+                    .append(", ");
+            }
+
+            sb.delete(sb.length() - 2, sb.length())
+                .append("]; ");
+        }
+
+        sb.delete(sb.length() - 2, sb.length())
+            .append("]");
+
+        info(sb.toString()
+            .replaceAll("distributed.IgniteTxRollbackOnStopTest", "Grid"));
+    }
+
+    /**
+     * @param igniteEx IgniteEx.
+     * @return Set of timeout objects that process on current IgniteEx.
+     */
+    private Set<GridTimeoutObject> getTimeoutObjects(IgniteEx igniteEx) {
+        GridTimeoutProcessor timeout = igniteEx.context().timeout();
+
+        return GridTestUtils.getFieldValue(timeout, timeout.getClass(), "timeoutObjs");
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessorSelfTest.java
index eb248cfd5dced..606b10252b19a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessorSelfTest.java
@@ -40,6 +40,11 @@ public class GridTimeoutProcessorSelfTest extends GridCommonAbstractTest {
     /** Kernal context. */
     private GridTestKernalContext ctx;
 
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 60_000;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         ctx = newContext();
@@ -84,7 +89,9 @@ public void testTimeouts() throws Exception {
                 }
 
                 /** {@inheritDoc} */
-                @Override public long endTime() { return endTime; }
+                @Override public long endTime() {
+                    return endTime;
+                }
 
                 /** {@inheritDoc} */
                 @Override public void onTimeout() {
@@ -152,10 +159,14 @@ public void testTimeoutsMultithreaded() throws Exception {
                         private final long endTime = System.currentTimeMillis() + RAND.nextInt(1000) + 500;
 
                         /** {@inheritDoc} */
-                        @Override public IgniteUuid timeoutId() { return id; }
+                        @Override public IgniteUuid timeoutId() {
+                            return id;
+                        }
 
                         /** {@inheritDoc} */
-                        @Override public long endTime() { return endTime; }
+                        @Override public long endTime() {
+                            return endTime;
+                        }
 
                         /** {@inheritDoc} */
                         @Override public void onTimeout() {
@@ -307,9 +318,8 @@ public void testTimeoutNeverCalled() throws Exception {
         assert timeObjs.size() == max;
 
         // Remove timeout objects so that they aren't able to times out (supposing the cycle takes less than 500 ms).
-        for (GridTimeoutObject obj : timeObjs) {
+        for (GridTimeoutObject obj : timeObjs)
             ctx.timeout().removeTimeoutObject(obj);
-        }
 
         Thread.sleep(1000);
 
@@ -350,7 +360,9 @@ public void testTimeoutNeverCalledMultithreaded() throws Exception {
                         }
 
                         /** {@inheritDoc} */
-                        @Override public long endTime() { return endTime; }
+                        @Override public long endTime() {
+                            return endTime;
+                        }
 
                         /** {@inheritDoc} */
                         @Override public void onTimeout() {
@@ -370,9 +382,8 @@ public void testTimeoutNeverCalledMultithreaded() throws Exception {
 
                 // Remove timeout objects so that they aren't able to times out
                 // (supposing the cycle takes less than 500 ms).
-                for (GridTimeoutObject obj : timeObjs) {
+                for (GridTimeoutObject obj : timeObjs)
                     ctx.timeout().removeTimeoutObject(obj);
-                }
             }
         }, threads, "timeout-test-worker");
 
@@ -381,6 +392,9 @@ public void testTimeoutNeverCalledMultithreaded() throws Exception {
         assert callCnt.get() == 0;
     }
 
+    /**
+     * @throws Exception If test failed.
+     */
     public void testAddRemoveInterleaving() throws Exception {
         final AtomicInteger callCnt = new AtomicInteger(0);
 
@@ -430,9 +444,8 @@ public void testAddRemoveInterleaving() throws Exception {
 
                 // Remove timeout objects so that they aren't able to times out
                 // (supposing the cycle takes less than 500 ms).
-                for (GridTimeoutObject obj : timeObjs) {
+                for (GridTimeoutObject obj : timeObjs)
                     ctx.timeout().removeTimeoutObject(obj);
-                }
             }
         }, 100, "timeout-test-worker");
 
@@ -516,10 +529,14 @@ public void testTimeoutCallOnce() throws Exception {
             private int cnt;
 
             /** {@inheritDoc} */
-            @Override public IgniteUuid timeoutId() { return id; }
+            @Override public IgniteUuid timeoutId() {
+                return id;
+            }
 
             /** {@inheritDoc} */
-            @Override public long endTime() { return endTime; }
+            @Override public long endTime() {
+                return endTime;
+            }
 
             /** {@inheritDoc} */
             @Override public void onTimeout() {
@@ -608,4 +625,31 @@ public void testTimeoutSameEndTime() throws Exception {
 
         assert latch.await(3000, MILLISECONDS);
     }
+
+    /**
+     * Test that eaten {@link InterruptedException} will not hang on the closing of the grid.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testCancelingWithClearedInterruptedFlag() throws Exception {
+        final CountDownLatch onTimeoutCalled = new CountDownLatch(1);
+
+        ctx.timeout().addTimeoutObject(new GridTimeoutObjectAdapter(10) {
+            /** {@inheritDoc} */
+            @Override public void onTimeout() {
+                try {
+                    onTimeoutCalled.countDown();
+
+                    // Wait for CacheProcessor has stopped and cause InterruptedException
+                    // which clears interrupted flag.
+                    Thread.sleep(Long.MAX_VALUE);
+                }
+                catch (InterruptedException ignore) {
+                    // No-op.
+                }
+            }
+        });
+
+        onTimeoutCalled.await();
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index 0785714c61b0d..87051c3d425ce 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.CacheAsyncOperationsTest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheMixedModeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteTxGetAfterStopTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteTxRemoveTimeoutObjectsTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDaemonNodePartitionedSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedOnlyP2PDisabledByteArrayValuesSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedOnlyP2PEnabledByteArrayValuesSelfTest;
@@ -205,6 +206,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(CacheAsyncOperationsTest.class);
 
+        suite.addTestSuite(IgniteTxRemoveTimeoutObjectsTest.class);
+
         return suite;
     }
 }

From b63c7911adedd0952edb5c2062a2dad8927976ac Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Tue, 15 Aug 2017 17:36:02 +0300
Subject: [PATCH 376/446] Muted
 IgniteCacheMessageWriteTimeoutTest#testMessageQueueLimit test.

---
 .../cache/distributed/IgniteCacheMessageWriteTimeoutTest.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index 0dd4079733081..8f3f7723efb60 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -75,7 +75,9 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testMessageQueueLimit() throws Exception {
-        for (int i = 0; i < 3; i++) {
+        fail("https://ggsystems.atlassian.net/browse/GG-12398");
+
+        for (int i = 0; i < 15; i++) {
             log.info("Iteration: " + i);
 
             startGridsMultiThreaded(3);

From 60e2de794b52cc7fe3525c5e6de831106bcb9d5f Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Wed, 16 Aug 2017 00:24:07 +0300
Subject: [PATCH 377/446] IgniteCacheNearRestartRollbackSelfTest#testRestarts
 is muted.

---
 .../distributed/IgniteCacheNearRestartRollbackSelfTest.java     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index 3f242b5726867..aea4d7782d858 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -132,6 +132,8 @@ protected CacheConfiguration<Object, Object> cacheConfiguration(String gridName)
      */
     @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
     public void testRestarts() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-12398");
+
         startGrids(4);
 
         Ignite tester = ignite(3);

From 928d445f63cf47ad52185b73ccfef559f22717e8 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Wed, 16 Aug 2017 14:06:37 +0300
Subject: [PATCH 378/446] Fixed flaky test "IgniteCacheEntryListener*"

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../query/continuous/CacheContinuousQueryEntry.java      | 9 +--------
 .../query/continuous/CacheContinuousQueryHandler.java    | 2 --
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index 366a1e05fa46c..ffbbc2c1e05ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -199,13 +199,6 @@ void markFiltered() {
         depInfo = null;
     }
 
-    /**
-     * @param topVer Topology version.
-     */
-    void topologyVersion(AffinityTopologyVersion topVer) {
-        this.topVer = topVer;
-    }
-
     /**
      * @return Size include this event and filtered.
      */
@@ -222,7 +215,7 @@ CacheContinuousQueryEntry forBackupQueue() {
             return this;
 
         CacheContinuousQueryEntry e =
-            new CacheContinuousQueryEntry(cacheId, null, null, null, null, keepBinary, part, updateCntr, null);
+            new CacheContinuousQueryEntry(cacheId, null, null, null, null, keepBinary, part, updateCntr, topVer);
 
         e.flags = flags;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 7cf7d0761db13..26c361498fbab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -481,8 +481,6 @@ public void keepBinary(boolean keepBinary) {
                             for (CacheContinuousQueryEntry e : backupQueue0) {
                                 if (!e.isFiltered())
                                     prepareEntry(cctx, nodeId, e);
-
-                                e.topologyVersion(topVer);
                             }
 
                             ctx.continuous().addBackupNotification(nodeId, routineId, backupQueue0, topic);

From aa5ca8fd154a853c37c2121d88a92e3404eead3a Mon Sep 17 00:00:00 2001
From: Nikolay Izhikov <nizhikov.dev@gmail.com>
Date: Wed, 16 Aug 2017 15:45:16 +0300
Subject: [PATCH 379/446] IGNITE-5897 Fix session init/end logic. This fixes
 tests.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../cache/store/CacheStoreManager.java        |  4 +++-
 .../store/GridCacheStoreManagerAdapter.java   | 20 +++++++++----------
 .../cache/transactions/IgniteTxAdapter.java   |  9 +++++++--
 3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
index 8d6b63dce3c8a..b096edf6a8b95 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
@@ -165,9 +165,11 @@ public boolean removeAll(@Nullable IgniteInternalTx tx, Collection<Object> keys)
     /**
      * @param tx Transaction.
      * @param commit Commit.
+     * @param last {@code True} if this is last store in transaction.
+     * @param storeSessionEnded {@code True} if session for underlying store already ended.
      * @throws IgniteCheckedException If failed.
      */
-    public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException;
+    public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last, boolean storeSessionEnded) throws IgniteCheckedException;
 
     /**
      * End session initiated by write-behind store.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 685e8f6e03181..14ec92237c3d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -777,7 +777,8 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public final void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException {
+    @Override public final void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last,
+        boolean storeSessionEnded) throws IgniteCheckedException {
         assert store != null;
 
         sessionInit0(tx);
@@ -788,7 +789,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
                     lsnr.onSessionEnd(locSes, commit);
             }
 
-            if (!sesHolder.get().ended(store))
+            if (!sesHolder.get().ended(store) && !storeSessionEnded)
                 store.sessionEnd(commit);
         }
         catch (Throwable e) {
@@ -857,7 +858,7 @@ private void sessionInit0(@Nullable IgniteInternalTx tx) throws IgniteCheckedExc
         sesHolder.set(ses);
 
         try {
-            if (sesLsnrs != null && !ses.started(this)) {
+            if (!ses.started(store) && sesLsnrs != null) {
                 for (CacheStoreSessionListener lsnr : sesLsnrs)
                     lsnr.onSessionStart(locSes);
             }
@@ -920,11 +921,8 @@ private static class SessionData {
         private Object attachment;
 
         /** */
-        private final Set<CacheStoreManager> started =
-            new GridSetWrapper<>(new IdentityHashMap<CacheStoreManager, Object>());
-
-        /** */
-        private final Set<CacheStore> ended = new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
+        private final Set<CacheStore> started =
+            new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
 
         /**
          * @param tx Current transaction.
@@ -987,8 +985,8 @@ private void cacheName(String cacheName) {
         /**
          * @return If session is started.
          */
-        private boolean started(CacheStoreManager mgr) {
-            return !started.add(mgr);
+        private boolean started(CacheStore store) {
+            return !started.add(store);
         }
 
         /**
@@ -996,7 +994,7 @@ private boolean started(CacheStoreManager mgr) {
          * @return Whether session already ended on this store instance.
          */
         private boolean ended(CacheStore store) {
-            return !ended.add(store);
+            return !started.remove(store);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 18c301124405f..cd5babe7118b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -40,6 +41,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -61,6 +63,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
+import org.apache.ignite.internal.util.GridSetWrapper;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
 import org.apache.ignite.internal.util.lang.GridTuple;
@@ -1238,13 +1241,15 @@ protected boolean isWriteToStoreFromDhtValid(Collection<CacheStoreManager> store
      * @param commit Commit flag.
      * @throws IgniteCheckedException In case of error.
      */
-    protected void sessionEnd(Collection<CacheStoreManager> stores, boolean commit) throws IgniteCheckedException {
+    protected void sessionEnd(final Collection<CacheStoreManager> stores, boolean commit) throws IgniteCheckedException {
         Iterator<CacheStoreManager> it = stores.iterator();
 
+        Set<CacheStore> visited = new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
+
         while (it.hasNext()) {
             CacheStoreManager store = it.next();
 
-            store.sessionEnd(this, commit, !it.hasNext());
+            store.sessionEnd(this, commit, !it.hasNext(), !visited.add(store.store()));
         }
     }
 

From 4ee9e7da3187245fb34d87e2475b69bb5b474b0c Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Wed, 16 Aug 2017 18:00:31 +0300
Subject: [PATCH 380/446] gg-12637 Fixed unevenly partitions distribution in
 FairAffinityFunction

---
 .../affinity/fair/FairAffinityFunction.java   | 29 ++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
index 7acb5b429e693..fe146ea7ad191 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
@@ -95,6 +95,15 @@ public class FairAffinityFunction implements AffinityFunction {
     /** Exclude neighbors warning. */
     private transient boolean exclNeighborsWarn;
 
+    /**
+     * <b>NOTE:</b> Use {@code true} value only for new clusters or in case of cluster upgrade with downtime.
+     * If nodes of cluster have affinity function with different configuration it will lead to different
+     * assignments calculation on different nodes and, therefore, broken consistent hashing functionality.
+     *
+     * Compatibility flag. Lead to better partitions distribution if value is {@code true}.
+     */
+    private boolean ceilIdealPartCnt = false;
+
     /** Logger instance. */
     @LoggerResource
     private transient IgniteLogger log;
@@ -280,6 +289,24 @@ public void setExcludeNeighbors(boolean exclNeighbors) {
         this.exclNeighbors = exclNeighbors;
     }
 
+    /**
+     * Returns value of {@link #ceilIdealPartCnt} compatibility flag.
+     *
+     * @return Value of {@link #ceilIdealPartCnt} compatibility flag.
+     */
+    public boolean isCeilIdealPartitionsCount() {
+        return ceilIdealPartCnt;
+    }
+
+    /**
+     * Sets value of {@link #ceilIdealPartCnt} compatibility flag.
+     *
+     * @param ceilIdealPartCnt Indicates that ideal partitions count should be rounded to biggest integer value.
+     */
+    public void setCeilIdealPartitionsCount(boolean ceilIdealPartCnt) {
+        this.ceilIdealPartCnt = ceilIdealPartCnt;
+    }
+
     /** {@inheritDoc} */
     @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext ctx) {
         List<ClusterNode> topSnapshot = ctx.currentTopologySnapshot();
@@ -386,7 +413,7 @@ private void assignPending(int tier,
         if (F.isEmpty(pending))
             return;
 
-        int idealPartCnt = parts / topSnapshot.size();
+        int idealPartCnt = ceilIdealPartCnt ? (int)Math.ceil((double)parts / topSnapshot.size()) : parts / topSnapshot.size();
 
         Map<UUID, PartitionSet> tierMapping = fullMap.tierMapping(tier);
 

From a76932f6f3d253a292803a0d1e954d42589bdbc6 Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Wed, 16 Aug 2017 18:00:31 +0300
Subject: [PATCH 381/446] gg-12637 Fixed unevenly partitions distribution in
 FairAffinityFunction

---
 .../affinity/fair/FairAffinityFunction.java   | 29 ++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
index cffcf108c18df..522dac2144db1 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
@@ -95,6 +95,15 @@ public class FairAffinityFunction implements AffinityFunction {
     /** Exclude neighbors warning. */
     private transient boolean exclNeighborsWarn;
 
+    /**
+     * <b>NOTE:</b> Use {@code true} value only for new clusters or in case of cluster upgrade with downtime.
+     * If nodes of cluster have affinity function with different configuration it will lead to different
+     * assignments calculation on different nodes and, therefore, broken consistent hashing functionality.
+     *
+     * Compatibility flag. Lead to better partitions distribution if value is {@code true}.
+     */
+    private boolean ceilIdealPartCnt = false;
+
     /** Logger instance. */
     @LoggerResource
     private transient IgniteLogger log;
@@ -280,6 +289,24 @@ public void setExcludeNeighbors(boolean exclNeighbors) {
         this.exclNeighbors = exclNeighbors;
     }
 
+    /**
+     * Returns value of {@link #ceilIdealPartCnt} compatibility flag.
+     *
+     * @return Value of {@link #ceilIdealPartCnt} compatibility flag.
+     */
+    public boolean isCeilIdealPartitionsCount() {
+        return ceilIdealPartCnt;
+    }
+
+    /**
+     * Sets value of {@link #ceilIdealPartCnt} compatibility flag.
+     *
+     * @param ceilIdealPartCnt Indicates that ideal partitions count should be rounded to biggest integer value.
+     */
+    public void setCeilIdealPartitionsCount(boolean ceilIdealPartCnt) {
+        this.ceilIdealPartCnt = ceilIdealPartCnt;
+    }
+
     /** {@inheritDoc} */
     @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext ctx) {
         List<ClusterNode> topSnapshot = ctx.currentTopologySnapshot();
@@ -382,7 +409,7 @@ private void assignPending(int tier,
         if (F.isEmpty(pending))
             return;
 
-        int idealPartCnt = parts / topSnapshot.size();
+        int idealPartCnt = ceilIdealPartCnt ? (int)Math.ceil((double)parts / topSnapshot.size()) : parts / topSnapshot.size();
 
         Map<UUID, PartitionSet> tierMapping = fullMap.tierMapping(tier);
 

From 533128821357c0909710069ea589894d99908474 Mon Sep 17 00:00:00 2001
From: Vyacheslav Daradur <daradurvs@gmail.com>
Date: Fri, 10 Feb 2017 16:51:37 +0300
Subject: [PATCH 382/446] GG-12647: Backport IGNITE-3196 Add support for
 BigDecimals with negative scale in BinaryMarshaller

---
 .../ignite/internal/binary/BinaryUtils.java   | 12 ++---
 .../internal/binary/BinaryWriterExImpl.java   | 14 +++---
 .../binary/BinaryMarshallerSelfTest.java      | 30 +++++++++++++
 modules/platforms/cpp/odbc/src/utility.cpp    | 17 ++++---
 .../Compute/ComputeApiTest.cs                 | 18 ++++----
 .../Impl/Binary/BinaryUtils.cs                | 44 +++++++++++--------
 6 files changed, 92 insertions(+), 43 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index fdc54c7871711..ceadd2a59585c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -65,8 +65,8 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 
 /**
  * Binary utils.
@@ -1193,13 +1193,15 @@ public static BigDecimal doReadDecimal(BinaryInputStream in) {
         int scale = in.readInt();
         byte[] mag = doReadByteArray(in);
 
-        BigInteger intVal = new BigInteger(mag);
+        boolean negative = mag[0] < 0;
 
-        if (scale < 0) {
-            scale &= 0x7FFFFFFF;
+        if (negative)
+            mag[0] &= 0x7F;
 
+        BigInteger intVal = new BigInteger(mag);
+
+        if (negative)
             intVal = intVal.negate();
-        }
 
         return new BigDecimal(intVal, scale);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index 1de0a6533e600..12ce868354afc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -364,18 +364,20 @@ public void doWriteDecimal(@Nullable BigDecimal val) {
 
             out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL);
 
+            out.unsafeWriteInt(val.scale());
+
             BigInteger intVal = val.unscaledValue();
 
-            if (intVal.signum() == -1) {
-                intVal = intVal.negate();
+            boolean negative = intVal.signum() == -1;
 
-                out.unsafeWriteInt(val.scale() | 0x80000000);
-            }
-            else
-                out.unsafeWriteInt(val.scale());
+            if (negative)
+                intVal = intVal.negate();
 
             byte[] vals = intVal.toByteArray();
 
+            if (negative)
+                vals[0] |= -0x80;
+
             out.unsafeWriteInt(vals.length);
             out.writeByteArray(vals);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index 6d07c9ba42770..bfc8bd357f292 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -29,6 +29,7 @@
 import java.lang.reflect.Proxy;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.math.RoundingMode;
 import java.net.InetSocketAddress;
 import java.sql.Timestamp;
 import java.util.AbstractQueue;
@@ -178,6 +179,35 @@ public void testDecimal() throws Exception {
         assertEquals((val = new BigDecimal(new BigInteger("-79228162514264337593543950336"))), marshalUnmarshal(val));
     }
 
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -1)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -2)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -3)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -4)), marshalUnmarshal(val));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleRoundingModeDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.ZERO.setScale(-1, RoundingMode.HALF_UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE).setScale(-3, RoundingMode.HALF_DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE).setScale(-5, RoundingMode.HALF_EVEN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MAX_VALUE).setScale(-8, RoundingMode.UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MIN_VALUE).setScale(-10, RoundingMode.DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MAX_VALUE).setScale(-12, RoundingMode.CEILING)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MIN_VALUE).setScale(-15, RoundingMode.FLOOR)), marshalUnmarshal(val));
+    }
+
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/platforms/cpp/odbc/src/utility.cpp b/modules/platforms/cpp/odbc/src/utility.cpp
index 19be799a10aae..c331332a34756 100644
--- a/modules/platforms/cpp/odbc/src/utility.cpp
+++ b/modules/platforms/cpp/odbc/src/utility.cpp
@@ -93,8 +93,14 @@ namespace ignite
 
             impl::binary::BinaryUtils::ReadInt8Array(reader.GetStream(), mag.data(), static_cast<int32_t>(mag.size()));
 
-            int32_t sign = (scale & 0x80000000) ? -1 : 1;
-            scale = scale & 0x7FFFFFFF;
+            int32_t sign = 1;
+            
+            if (mag[0] < 0)
+            {
+                mag[0] &= 0x7F;
+
+                sign = -1;
+            }
 
             common::Decimal res(mag.data(), static_cast<int32_t>(mag.size()), scale, sign);
 
@@ -107,14 +113,15 @@ namespace ignite
 
             const common::BigInteger &unscaled = decimal.GetUnscaledValue();
 
-            int32_t signFlag = unscaled.GetSign() == -1 ? 0x80000000 : 0;
-
-            writer.WriteInt32(decimal.GetScale() | signFlag);
+            writer.WriteInt32(decimal.GetScale());
 
             common::FixedSizeArray<int8_t> magnitude;
 
             unscaled.MagnitudeToBytes(magnitude);
 
+            if (unscaled.GetSign() == -1)
+                magnitude[0] |= -0x80;
+
             writer.WriteInt32(magnitude.GetSize());
 
             impl::binary::BinaryUtils::WriteInt8Array(writer.GetStream(), magnitude.GetData(), magnitude.GetSize());
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
index 1e0287fe56e8f..15d8dba7895be 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
@@ -818,16 +818,16 @@ public void TestEchoDecimal()
             Assert.AreEqual(val = decimal.Parse("-11,12"), _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { val, val.ToString() }));
 
             // Test echo with overflow.
-            try
-            {
-                _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { null, decimal.MaxValue.ToString() + 1 });
+            var ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, decimal.MaxValue.ToString() + 1}));
 
-                Assert.Fail();
-            }
-            catch (IgniteException)
-            {
-                // No-op.
-            }
+            Assert.AreEqual("Decimal magnitude overflow (must be less than 96 bits): 104", ex.Message);
+
+            // Negative scale. 1E+1 parses to "1 scale -1" on Java side.
+            ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, "1E+1"}));
+
+            Assert.AreEqual("Decimal value scale overflow (must be between 0 and 28): -1", ex.Message);
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
index cc5d8a109ce86..5ce77d0ee4347 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
@@ -893,7 +893,9 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
             // Write scale and negative flag.
             int scale = (vals[3] & 0x00FF0000) >> 16; 
 
-            stream.WriteInt(((vals[3] & 0x80000000) == 0x80000000) ? (int)((uint)scale | 0x80000000) : scale);
+            stream.WriteInt(scale);
+
+            Boolean neg = vals[3] < 0;
 
             if (idx == -1)
             {
@@ -923,13 +925,15 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             if ((part24 & 0x80) == 0x80)
                             {
                                 stream.WriteInt(len + 1);
+                                
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
 
-                                stream.WriteByte(ByteZero);
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len);
 
-                            stream.WriteByte((byte)part24);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part24 | -0x80) : part24));
                             stream.WriteByte((byte)part16);
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
@@ -940,12 +944,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 1);
-
-                            stream.WriteByte((byte)part16);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part16 | -0x80) : part16));
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
                         }
@@ -955,12 +961,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 1);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 2);
-
-                            stream.WriteByte((byte)part8);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part8 | -0x80) : part8));
                             stream.WriteByte((byte)part0);
                         }
                         else
@@ -969,12 +977,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 2);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 3);
 
-                            stream.WriteByte((byte)part0);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part0 | -0x80) : part0));
                         }
                     }
                     else
@@ -997,18 +1007,16 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
         {
             int scale = stream.ReadInt();
 
-            bool neg;
+            bool neg = false;
+
+            byte[] mag = ReadByteArray(stream);
 
-            if (scale < 0)
+            if ((sbyte)mag[0] < 0)
             {
-                scale = scale & 0x7FFFFFFF;
+                mag[0] &= 0x7F;
 
                 neg = true;
             }
-            else
-                neg = false;
-
-            byte[] mag = ReadByteArray(stream);
 
             if (scale < 0 || scale > 28)
                 throw new BinaryObjectException("Decimal value scale overflow (must be between 0 and 28): " + scale);

From 558d755a2f5a8499f0dd690834fd97137245fc63 Mon Sep 17 00:00:00 2001
From: Vyacheslav Daradur <daradurvs@gmail.com>
Date: Fri, 10 Feb 2017 16:51:37 +0300
Subject: [PATCH 383/446] GG-12647: Backport IGNITE-3196 Add support for
 BigDecimals with negative scale in BinaryMarshaller

---
 .../ignite/internal/binary/BinaryUtils.java   | 12 ++---
 .../internal/binary/BinaryWriterExImpl.java   | 14 +++---
 .../binary/BinaryMarshallerSelfTest.java      | 30 +++++++++++++
 modules/platforms/cpp/odbc/src/utility.cpp    | 17 ++++---
 .../Compute/ComputeApiTest.cs                 | 18 ++++----
 .../Impl/Binary/BinaryUtils.cs                | 44 +++++++++++--------
 6 files changed, 92 insertions(+), 43 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 1153d155c0400..6831ef9bb0d66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -65,8 +65,8 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 
 /**
  * Binary utils.
@@ -1193,13 +1193,15 @@ public static BigDecimal doReadDecimal(BinaryInputStream in) {
         int scale = in.readInt();
         byte[] mag = doReadByteArray(in);
 
-        BigInteger intVal = new BigInteger(mag);
+        boolean negative = mag[0] < 0;
 
-        if (scale < 0) {
-            scale &= 0x7FFFFFFF;
+        if (negative)
+            mag[0] &= 0x7F;
 
+        BigInteger intVal = new BigInteger(mag);
+
+        if (negative)
             intVal = intVal.negate();
-        }
 
         return new BigDecimal(intVal, scale);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index adaacdda40676..3289780e3fd0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -403,18 +403,20 @@ public void doWriteDecimal(@Nullable BigDecimal val) {
 
             out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL);
 
+            out.unsafeWriteInt(val.scale());
+
             BigInteger intVal = val.unscaledValue();
 
-            if (intVal.signum() == -1) {
-                intVal = intVal.negate();
+            boolean negative = intVal.signum() == -1;
 
-                out.unsafeWriteInt(val.scale() | 0x80000000);
-            }
-            else
-                out.unsafeWriteInt(val.scale());
+            if (negative)
+                intVal = intVal.negate();
 
             byte[] vals = intVal.toByteArray();
 
+            if (negative)
+                vals[0] |= -0x80;
+
             out.unsafeWriteInt(vals.length);
             out.writeByteArray(vals);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index cd8a487d6d876..31d6f31eef961 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -29,6 +29,7 @@
 import java.lang.reflect.Proxy;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.math.RoundingMode;
 import java.net.InetSocketAddress;
 import java.sql.Timestamp;
 import java.util.AbstractQueue;
@@ -178,6 +179,35 @@ public void testDecimal() throws Exception {
         assertEquals((val = new BigDecimal(new BigInteger("-79228162514264337593543950336"))), marshalUnmarshal(val));
     }
 
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -1)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -2)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -3)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -4)), marshalUnmarshal(val));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleRoundingModeDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.ZERO.setScale(-1, RoundingMode.HALF_UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE).setScale(-3, RoundingMode.HALF_DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE).setScale(-5, RoundingMode.HALF_EVEN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MAX_VALUE).setScale(-8, RoundingMode.UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MIN_VALUE).setScale(-10, RoundingMode.DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MAX_VALUE).setScale(-12, RoundingMode.CEILING)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MIN_VALUE).setScale(-15, RoundingMode.FLOOR)), marshalUnmarshal(val));
+    }
+
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/platforms/cpp/odbc/src/utility.cpp b/modules/platforms/cpp/odbc/src/utility.cpp
index 22191eb8b045e..2c8f9f3a3cdab 100644
--- a/modules/platforms/cpp/odbc/src/utility.cpp
+++ b/modules/platforms/cpp/odbc/src/utility.cpp
@@ -87,8 +87,14 @@ namespace ignite
 
             impl::binary::BinaryUtils::ReadInt8Array(reader.GetStream(), mag.data(), static_cast<int32_t>(mag.size()));
 
-            int32_t sign = (scale & 0x80000000) ? -1 : 1;
-            scale = scale & 0x7FFFFFFF;
+            int32_t sign = 1;
+            
+            if (mag[0] < 0)
+            {
+                mag[0] &= 0x7F;
+
+                sign = -1;
+            }
 
             common::Decimal res(mag.data(), static_cast<int32_t>(mag.size()), scale, sign);
 
@@ -101,14 +107,15 @@ namespace ignite
 
             const common::BigInteger &unscaled = decimal.GetUnscaledValue();
 
-            int32_t signFlag = unscaled.GetSign() == -1 ? 0x80000000 : 0;
-
-            writer.WriteInt32(decimal.GetScale() | signFlag);
+            writer.WriteInt32(decimal.GetScale());
 
             common::FixedSizeArray<int8_t> magnitude;
 
             unscaled.MagnitudeToBytes(magnitude);
 
+            if (unscaled.GetSign() == -1)
+                magnitude[0] |= -0x80;
+
             writer.WriteInt32(magnitude.GetSize());
 
             impl::binary::BinaryUtils::WriteInt8Array(writer.GetStream(), magnitude.GetData(), magnitude.GetSize());
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
index 71a4718130475..3a3100a6b3131 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
@@ -817,16 +817,16 @@ public void TestEchoDecimal()
             Assert.AreEqual(val = decimal.Parse("-11,12"), _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { val, val.ToString() }));
 
             // Test echo with overflow.
-            try
-            {
-                _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { null, decimal.MaxValue.ToString() + 1 });
+            var ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, decimal.MaxValue.ToString() + 1}));
 
-                Assert.Fail();
-            }
-            catch (IgniteException)
-            {
-                // No-op.
-            }
+            Assert.AreEqual("Decimal magnitude overflow (must be less than 96 bits): 104", ex.Message);
+
+            // Negative scale. 1E+1 parses to "1 scale -1" on Java side.
+            ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, "1E+1"}));
+
+            Assert.AreEqual("Decimal value scale overflow (must be between 0 and 28): -1", ex.Message);
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
index cc5d8a109ce86..5ce77d0ee4347 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
@@ -893,7 +893,9 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
             // Write scale and negative flag.
             int scale = (vals[3] & 0x00FF0000) >> 16; 
 
-            stream.WriteInt(((vals[3] & 0x80000000) == 0x80000000) ? (int)((uint)scale | 0x80000000) : scale);
+            stream.WriteInt(scale);
+
+            Boolean neg = vals[3] < 0;
 
             if (idx == -1)
             {
@@ -923,13 +925,15 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             if ((part24 & 0x80) == 0x80)
                             {
                                 stream.WriteInt(len + 1);
+                                
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
 
-                                stream.WriteByte(ByteZero);
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len);
 
-                            stream.WriteByte((byte)part24);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part24 | -0x80) : part24));
                             stream.WriteByte((byte)part16);
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
@@ -940,12 +944,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 1);
-
-                            stream.WriteByte((byte)part16);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part16 | -0x80) : part16));
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
                         }
@@ -955,12 +961,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 1);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 2);
-
-                            stream.WriteByte((byte)part8);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part8 | -0x80) : part8));
                             stream.WriteByte((byte)part0);
                         }
                         else
@@ -969,12 +977,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 2);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 3);
 
-                            stream.WriteByte((byte)part0);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part0 | -0x80) : part0));
                         }
                     }
                     else
@@ -997,18 +1007,16 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
         {
             int scale = stream.ReadInt();
 
-            bool neg;
+            bool neg = false;
+
+            byte[] mag = ReadByteArray(stream);
 
-            if (scale < 0)
+            if ((sbyte)mag[0] < 0)
             {
-                scale = scale & 0x7FFFFFFF;
+                mag[0] &= 0x7F;
 
                 neg = true;
             }
-            else
-                neg = false;
-
-            byte[] mag = ReadByteArray(stream);
 
             if (scale < 0 || scale > 28)
                 throw new BinaryObjectException("Decimal value scale overflow (must be between 0 and 28): " + scale);

From 0700f8f473fd1e5507e3782f4c4bb9e74ec17947 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Mon, 21 Aug 2017 17:54:12 +0300
Subject: [PATCH 384/446] GG-12588 Backport IGNITE-5943 Communication. Server
 node may reject client connection during massive clients join.

---
 .../tcp/TcpCommunicationSpi.java              |  61 ++++++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  24 ++-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |  10 +
 .../tcp/IgniteClientConnectTest.java          | 172 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   6 +
 5 files changed, 266 insertions(+), 7 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 7a5f7c1f16f3a..0743ea47c3e67 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -65,6 +65,7 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.GridConcurrentFactory;
 import org.apache.ignite.internal.util.GridSpinReadWriteLock;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -129,6 +130,8 @@
 import org.apache.ignite.spi.IgniteSpiTimeoutObject;
 import org.apache.ignite.spi.communication.CommunicationListener;
 import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentLinkedDeque8;
@@ -137,6 +140,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
+import static org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.RecoveryLastReceivedMessage.NEED_WAIT;
 
 /**
  * <tt>TcpCommunicationSpi</tt> is default communication SPI which uses
@@ -301,6 +305,13 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     /** Connection index meta for session. */
     private static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
 
+    /**
+     * Version when client is ready to wait to connect to server (could be needed when client tries to open connection
+     * before it starts being visible for server)
+     */
+    private static final IgniteProductVersion VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT =
+        IgniteProductVersion.fromString("1.8.10");
+
     /** Message tracker meta for session. */
     private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
 
@@ -435,7 +446,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
              * @param ses Session.
              * @param msg Message.
              */
-            private void onFirstMessage(GridNioSession ses, Message msg) {
+            private void onFirstMessage(final GridNioSession ses, Message msg) {
                 UUID sndId;
 
                 ConnectionKey connKey;
@@ -459,10 +470,35 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                 final ClusterNode rmtNode = getSpiContext().node(sndId);
 
                 if (rmtNode == null) {
-                    U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId +
-                        ", ses=" + ses + ']');
+                    DiscoverySpi discoverySpi = ignite.configuration().getDiscoverySpi();
+
+                    assert discoverySpi instanceof TcpDiscoverySpi;
+
+                    TcpDiscoverySpi tcpDiscoverySpi = (TcpDiscoverySpi) discoverySpi;
+
+                    ClusterNode node0 = tcpDiscoverySpi.getNode0(sndId);
+
+                    boolean unknownNode = true;
 
-                    ses.close();
+                    if (node0 != null) {
+                        assert node0.isClient() : node0;
+
+                        if (node0.version().compareTo(VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT) >= 0)
+                            unknownNode = false;
+                    }
+
+                    if (unknownNode) {
+                        U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId + ", ses=" + ses + ']');
+
+                        ses.close();
+                    }
+                    else {
+                        ses.send(new RecoveryLastReceivedMessage(NEED_WAIT)).listen(new CI1<IgniteInternalFuture<?>>() {
+                            @Override public void apply(IgniteInternalFuture<?> fut) {
+                                ses.close();
+                            }
+                        });
+                    }
 
                     return;
                 }
@@ -2858,6 +2894,8 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
             IgniteSpiOperationTimeoutHelper timeoutHelper = new IgniteSpiOperationTimeoutHelper(this);
 
+            int lastWaitingTimeout = 1;
+
             while (!conn) { // Reconnection on handshake timeout.
                 try {
                     SocketChannel ch = SocketChannel.open();
@@ -2920,6 +2958,18 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
                         if (rcvCnt == -1)
                             return null;
+                        else if (rcvCnt == NEED_WAIT) {
+                            recoveryDesc.release();
+
+                            U.closeQuiet(ch);
+
+                            if (lastWaitingTimeout < 60000)
+                                lastWaitingTimeout *= 2;
+
+                            U.sleep(lastWaitingTimeout);
+
+                            continue;
+                        }
                     }
                     finally {
                         if (recoveryDesc != null && rcvCnt == -1)
@@ -4294,6 +4344,9 @@ public static class RecoveryLastReceivedMessage implements Message {
         /** */
         private static final long serialVersionUID = 0L;
 
+        /** Need wait. */
+        static final long NEED_WAIT = -3;
+
         /** */
         private long rcvCnt;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 1c8988acb2ba1..cfa36f1599590 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1785,9 +1785,27 @@ private static void removeMetrics(TcpDiscoveryHeartbeatMessage msg, UUID nodeId)
     }
 
     /**
-     * Thread that cleans IP finder and keeps it in the correct state, unregistering addresses of the nodes that has
-     * left the topology. <p> This thread should run only on coordinator node and will clean IP finder if and only if
-     * {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()} is {@code true}.
+     * Trying get node in any state (visible or not)
+     * @param nodeId Node id.
+     */
+    ClusterNode getNode0(UUID nodeId) {
+        assert nodeId != null;
+
+        UUID locNodeId0 = getLocalNodeId();
+
+        if (locNodeId0 != null && locNodeId0.equals(nodeId))
+            // Return local node directly.
+            return locNode;
+
+        return ring.node(nodeId);
+    }
+
+    /**
+     * Thread that cleans IP finder and keeps it in the correct state, unregistering
+     * addresses of the nodes that has left the topology.
+     * <p>
+     * This thread should run only on coordinator node and will clean IP finder
+     * if and only if {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()} is {@code true}.
      */
     private class IpFinderCleaner extends IgniteSpiThread {
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 3cc4ee53242b2..b505ae373f210 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -437,6 +437,16 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, T
         return impl.getNode(nodeId);
     }
 
+    /**
+     * @param id Id.
+     */
+    public ClusterNode getNode0(UUID id) {
+        if (impl instanceof ServerImpl)
+            return ((ServerImpl)impl).getNode0(id);
+
+        return getNode(id);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean pingNode(UUID nodeId) {
         return impl.pingNode(nodeId);
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java
new file mode 100644
index 0000000000000..b1c4dbf25434d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+
+/**
+ * We emulate that client receive message about joining to topology earlier than some server nodes in topology.
+ * And make this client connect to such servers.
+ * To emulate this we connect client to second node in topology and pause sending message about joining finishing to
+ * third node.
+ */
+public class IgniteClientConnectTest extends GridCommonAbstractTest {
+
+    /** Custom cache name. */
+    private static final String DEFAULT_CACHE_NAME = "default-cache";
+
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** Latch to stop message sending. */
+    private final CountDownLatch latch = new CountDownLatch(1);
+
+    /** Start client flag. */
+    private final AtomicBoolean clientJustStarted = new AtomicBoolean(false);
+
+    /** Instance name. */
+    private String igniteInstanceName;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        this.igniteInstanceName = igniteInstanceName;
+
+        TestTcpDiscoverySpi disco = new TestTcpDiscoverySpi();
+
+        if (igniteInstanceName.equals("client")) {
+            TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
+
+            ipFinder.registerAddresses(Collections.singleton(new InetSocketAddress(InetAddress.getLoopbackAddress(), 47501)));
+
+            disco.setIpFinder(ipFinder);
+        }
+        else
+            disco.setIpFinder(ipFinder);
+
+        disco.setJoinTimeout(2 * 60_000);
+        disco.setSocketTimeout(1000);
+        disco.setNetworkTimeout(2000);
+
+        cfg.setDiscoverySpi(disco);
+
+        CacheConfiguration cacheConfiguration = new CacheConfiguration()
+                .setName(DEFAULT_CACHE_NAME)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setAffinity(new RendezvousAffinityFunction(false, 8))
+                .setBackups(0);
+
+        cfg.setCacheConfiguration(cacheConfiguration);
+
+        return cfg;
+    }
+
+    /**
+     *
+     * @throws Exception If failed.
+     */
+    public void testClientConnectToBigTopology() throws Exception {
+        Ignite ignite = startGrids(3);
+
+        IgniteCache<Object, Object> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        Set<Integer> keys = new HashSet<>();
+
+        for (int i = 0; i < 80; i++) {
+            cache.put(i, i);
+
+            keys.add(i);
+        }
+
+        TcpDiscoveryImpl discovery = ((TestTcpDiscoverySpi) ignite.configuration().getDiscoverySpi()).discovery();
+
+        assertTrue(discovery instanceof ServerImpl);
+
+        IgniteConfiguration clientCfg = getConfiguration("client");
+
+        clientCfg.setClientMode(true);
+
+        clientJustStarted.set(true);
+
+        Ignite client = startGrid(igniteInstanceName, clientCfg);
+
+        latch.countDown();
+
+        System.err.println("GET ALL");
+        client.cache(DEFAULT_CACHE_NAME).getAll(keys);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     *
+     */
+    class TestTcpDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg, long timeout) throws IOException,
+                IgniteCheckedException {
+            if (msg instanceof TcpDiscoveryNodeAddFinishedMessage) {
+                if (msg.senderNodeId() != null && clientJustStarted.get())
+                    try {
+                        latch.await();
+
+                        Thread.sleep(3000);
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    }
+
+                super.writeToSocket(sock, out, msg, timeout);
+            }
+            else
+                super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /**
+         *
+         */
+        TcpDiscoveryImpl discovery() {
+            return impl;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index e6b39f740a0dd..c2913ba6c6a5b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -19,6 +19,8 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.spi.GridTcpSpiForwardingSelfTest;
+import org.apache.ignite.spi.discovery.tcp.IgniteClientConnectTest;
+import org.apache.ignite.spi.discovery.tcp.IgniteClientReconnectMassiveShutdownTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest;
@@ -88,6 +90,10 @@ public static TestSuite suite() throws Exception {
 
         suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
 
+        //Client connect
+        suite.addTest(new TestSuite(IgniteClientConnectTest.class));
+        suite.addTest(new TestSuite(IgniteClientReconnectMassiveShutdownTest.class));
+
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));

From 890dcd128b3ae0829dd1fb96e3b1ab913f70d541 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Wed, 16 Aug 2017 18:06:36 +0300
Subject: [PATCH 385/446] IGNITE-1094: wip

---
 .../apache/ignite/internal/IgniteKernal.java  |   4 +
 .../ignite/internal/IgniteNodeAttributes.java |   3 +
 .../discovery/GridDiscoveryManager.java       |  39 +-
 .../cache/CacheAffinitySharedManager.java     |  71 +++
 .../DynamicCacheChangeFailureMessage.java     | 129 ++++++
 .../GridCachePartitionExchangeManager.java    |  13 +-
 .../processors/cache/GridCacheProcessor.java  |  42 +-
 .../GridDhtPartitionsExchangeFuture.java      | 189 +++++++-
 .../GridDhtPartitionsSingleMessage.java       |  59 ++-
 .../IgniteDynamicCacheStartFailSelfTest.java  | 421 ++++++++++++++++++
 10 files changed, 938 insertions(+), 32 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 28ac0436eab9d..e58811baa6fac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -201,6 +201,7 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CONSISTENCY_CHECK_SKIPPED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DAEMON;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_EXCHANGE_ROLLBACK_SUPPORTED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_GRID_NAME;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IPS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JIT_NAME;
@@ -1493,6 +1494,9 @@ private void fillNodeAttributes(boolean notifyEnabled) throws IgniteCheckedExcep
         if (cfg.getConnectorConfiguration() != null)
             add(ATTR_REST_PORT_RANGE, cfg.getConnectorConfiguration().getPortRange());
 
+        // Exchange rollback is supported
+        add(ATTR_EXCHANGE_ROLLBACK_SUPPORTED, Boolean.TRUE);
+
         // Stick in SPI versions and classes attributes.
         addSpiAttributes(cfg.getCollisionSpi());
         addSpiAttributes(cfg.getSwapSpaceSpi());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
index 436792459af4a..5002c4c31867c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
@@ -166,6 +166,9 @@ public final class IgniteNodeAttributes {
     /** Ignite security compatibility mode. */
     public static final String ATTR_SECURITY_COMPATIBILITY_MODE = ATTR_PREFIX + ".security.compatibility.enabled";
 
+    /** Internal attribute name constant. */
+    public static final String ATTR_EXCHANGE_ROLLBACK_SUPPORTED = ATTR_PREFIX + ".exchange.rollback.supported";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 71b7217bd6afc..1d67869bb478e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -254,7 +254,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
         new ConcurrentHashMap8<>();
 
     /** Map of dynamic cache filters. */
-    private Map<String, CachePredicate> registeredCaches = new HashMap<>();
+    private Map<String, CachePredicate> registeredCachesPreds = new HashMap<>();
 
     /** */
     private final GridSpinBusyLock busyLock = new GridSpinBusyLock();
@@ -312,19 +312,20 @@ public void setCacheFilter(
         boolean nearEnabled,
         CacheMode cacheMode
     ) {
-        if (!registeredCaches.containsKey(cacheName))
-            registeredCaches.put(cacheName, new CachePredicate(filter, nearEnabled, cacheMode));
+        if (!registeredCachesPreds.containsKey(cacheName))
+            registeredCachesPreds.put(cacheName, new CachePredicate(filter, nearEnabled, cacheMode));
     }
 
     /**
      * Removes dynamic cache filter.
      *
      * @param cacheName Cache name.
+     * @param force Removes dynamic cache filter without check that it was previously created.
      */
-    public void removeCacheFilter(String cacheName) {
-        CachePredicate p = registeredCaches.remove(cacheName);
+    public void removeCacheFilter(String cacheName, boolean force) {
+        CachePredicate p = registeredCachesPreds.remove(cacheName);
 
-        assert p != null : cacheName;
+        assert force || p != null : cacheName;
     }
 
     /**
@@ -336,7 +337,7 @@ public void removeCacheFilter(String cacheName) {
      * @return {@code True} if new node ID was added.
      */
     public boolean addClientNode(String cacheName, UUID clientNodeId, boolean nearEnabled) {
-        CachePredicate p = registeredCaches.get(cacheName);
+        CachePredicate p = registeredCachesPreds.get(cacheName);
 
         assert p != null : cacheName;
 
@@ -351,7 +352,7 @@ public boolean addClientNode(String cacheName, UUID clientNodeId, boolean nearEn
      * @return {@code True} if existing node ID was removed.
      */
     public boolean onClientCacheClose(String cacheName, UUID clientNodeId) {
-        CachePredicate p = registeredCaches.get(cacheName);
+        CachePredicate p = registeredCachesPreds.get(cacheName);
 
         assert p != null : cacheName;
 
@@ -364,12 +365,12 @@ public boolean onClientCacheClose(String cacheName, UUID clientNodeId) {
     public Map<String, Map<UUID, Boolean>> clientNodesMap() {
         Map<String, Map<UUID, Boolean>> res = null;
 
-        for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
+        for (Map.Entry<String, CachePredicate> entry : registeredCachesPreds.entrySet()) {
             CachePredicate pred = entry.getValue();
 
             if (!F.isEmpty(pred.clientNodes)) {
                 if (res == null)
-                    res = U.newHashMap(registeredCaches.size());
+                    res = U.newHashMap(registeredCachesPreds.size());
 
                 res.put(entry.getKey(), new HashMap<>(pred.clientNodes));
             }
@@ -382,7 +383,7 @@ public Map<String, Map<UUID, Boolean>> clientNodesMap() {
      * @param leftNodeId Left node ID.
      */
     private void updateClientNodes(UUID leftNodeId) {
-        for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
+        for (Map.Entry<String, CachePredicate> entry : registeredCachesPreds.entrySet()) {
             CachePredicate pred = entry.getValue();
 
             pred.onNodeLeft(leftNodeId);
@@ -602,7 +603,7 @@ else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
 
                     locJoin = new GridFutureAdapter<>();
 
-                    registeredCaches.clear();
+                    registeredCachesPreds.clear();
 
                     for (AffinityTopologyVersion histVer : discoCacheHist.keySet()) {
                         Object rmvd = discoCacheHist.remove(histVer);
@@ -1718,7 +1719,7 @@ public Collection<ClusterNode> cacheAffinityNodes(int cacheId, AffinityTopologyV
      * @return {@code True} if node is a cache data node.
      */
     public boolean cacheAffinityNode(ClusterNode node, String cacheName) {
-        CachePredicate pred = registeredCaches.get(cacheName);
+        CachePredicate pred = registeredCachesPreds.get(cacheName);
 
         return pred != null && pred.dataNode(node);
     }
@@ -1729,7 +1730,7 @@ public boolean cacheAffinityNode(ClusterNode node, String cacheName) {
      * @return {@code True} if node has near cache enabled.
      */
     public boolean cacheNearNode(ClusterNode node, String cacheName) {
-        CachePredicate pred = registeredCaches.get(cacheName);
+        CachePredicate pred = registeredCachesPreds.get(cacheName);
 
         return pred != null && pred.nearNode(node);
     }
@@ -1740,7 +1741,7 @@ public boolean cacheNearNode(ClusterNode node, String cacheName) {
      * @return {@code True} if node has client cache (without near cache).
      */
     public boolean cacheClientNode(ClusterNode node, String cacheName) {
-        CachePredicate pred = registeredCaches.get(cacheName);
+        CachePredicate pred = registeredCachesPreds.get(cacheName);
 
         return pred != null && pred.clientNode(node);
     }
@@ -1751,7 +1752,7 @@ public boolean cacheClientNode(ClusterNode node, String cacheName) {
      * @return If cache with the given name is accessible on the given node.
      */
     public boolean cacheNode(ClusterNode node, String cacheName) {
-        CachePredicate pred = registeredCaches.get(cacheName);
+        CachePredicate pred = registeredCachesPreds.get(cacheName);
 
         return pred != null && pred.cacheNode(node);
     }
@@ -1761,9 +1762,9 @@ public boolean cacheNode(ClusterNode node, String cacheName) {
      * @return Public cache names accessible on the given node.
      */
     public Map<String, CacheMode> nodeCaches(ClusterNode node) {
-        Map<String, CacheMode> caches = U.newHashMap(registeredCaches.size());
+        Map<String, CacheMode> caches = U.newHashMap(registeredCachesPreds.size());
 
-        for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
+        for (Map.Entry<String, CachePredicate> entry : registeredCachesPreds.entrySet()) {
             String cacheName = entry.getKey();
 
             CachePredicate pred = entry.getValue();
@@ -2012,7 +2013,7 @@ public void reconnect() {
             assert node.order() != 0 : "Invalid node order [locNode=" + loc + ", node=" + node + ']';
             assert !node.isDaemon();
 
-            for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
+            for (Map.Entry<String, CachePredicate> entry : registeredCachesPreds.entrySet()) {
                 String cacheName = entry.getKey();
                 CachePredicate filter = entry.getValue();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 84372febb2c91..fc9e02ce294ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -322,6 +322,77 @@ public void onCacheCreated(GridCacheContext cctx) {
         }
     }
 
+    /**
+     * Called during exchange rollback in order to stop the given cache(s)
+     * even if it's not fully initialized (e.g. fail on cache init stage).
+     *
+     * @param fut Exchange future.
+     * @param crd Coordinator flag.
+     * @param reqs Cache change requests.
+     */
+    public void forceCloseCache(final GridDhtPartitionsExchangeFuture fut, boolean crd,
+        Collection<DynamicCacheChangeRequest> reqs) {
+
+        assert !F.isEmpty(reqs) : fut;
+
+        for (DynamicCacheChangeRequest req : reqs) {
+            assert req.stop() : req;
+
+            Integer cacheId = CU.cacheId(req.cacheName());
+
+            registeredCaches.remove(cacheId);
+        }
+
+        Set<Integer> stoppedCaches = null;
+
+        for (DynamicCacheChangeRequest req : reqs) {
+            Integer cacheId = CU.cacheId(req.cacheName());
+
+            cctx.cache().blockGateway(req);
+
+            if (crd) {
+                CacheHolder cache = caches.remove(cacheId);
+
+                if (cache != null) {
+                    if (stoppedCaches == null)
+                        stoppedCaches = new HashSet<>();
+
+                    stoppedCaches.add(cache.cacheId());
+
+                    cctx.io().removeHandler(cacheId, GridDhtAffinityAssignmentResponse.class);
+                }
+            }
+        }
+
+        if (stoppedCaches != null) {
+            boolean notify = false;
+
+            synchronized (mux) {
+                if (waitInfo != null) {
+                    for (Integer cacheId : stoppedCaches) {
+                        boolean rmv =  waitInfo.waitCaches.remove(cacheId) != null;
+
+                        if (rmv) {
+                            notify = true;
+
+                            waitInfo.assignments.remove(cacheId);
+                        }
+                    }
+                }
+            }
+
+            if (notify) {
+                final AffinityTopologyVersion topVer = affCalcVer;
+
+                cctx.kernalContext().closure().runLocalSafe(new Runnable() {
+                    @Override public void run() {
+                        onCacheStopped(topVer);
+                    }
+                });
+            }
+        }
+    }
+
     /**
      * Called on exchange initiated for cache start/stop request.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
new file mode 100644
index 0000000000000..162c67351cb32
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * This class represents discovery message that is used to provide information about dynamic cache start failure.
+ */
+public class DynamicCacheChangeFailureMessage implements DiscoveryCustomMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Change requests. */
+    @GridToStringInclude
+    private Collection<DynamicCacheChangeRequest> reqs;
+
+    /** Custom message ID. */
+    private IgniteUuid id;
+
+    /** */
+    private GridDhtPartitionExchangeId exchId;
+
+    /** */
+    private GridCacheVersion lastVer;
+
+    @GridToStringInclude
+    private IgniteCheckedException cause;
+
+    /**
+     * Creates new DynamicCacheChangeFailureMessage instance.
+     *
+     * @param exchId Exchange Id.
+     * @param lastVer Last version.
+     * @param cause
+     * @param reqs Cache change requests.
+     */
+    public DynamicCacheChangeFailureMessage(
+        ClusterNode localNode,
+        GridDhtPartitionExchangeId exchId,
+        @Nullable GridCacheVersion lastVer,
+        IgniteCheckedException cause,
+        Collection<DynamicCacheChangeRequest> reqs)
+    {
+        this.id = IgniteUuid.fromUuid(localNode.id());
+        this.exchId = exchId;
+        this.lastVer = lastVer;
+        this.cause = cause;
+        this.reqs = reqs;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteUuid id() {
+        return id;
+    }
+
+    /**
+     * @param id Message ID.
+     */
+    public void id(IgniteUuid id) {
+        this.id = id;
+    }
+
+    /**
+     * @return Collection of change requests.
+     */
+    public Collection<DynamicCacheChangeRequest> requests() {
+        return reqs;
+    }
+
+    public IgniteCheckedException getError() {
+        return cause;
+    }
+
+    /**
+     * @return Last used version among all nodes.
+     */
+    @Nullable public GridCacheVersion lastVersion() {
+        return lastVer;
+    }
+
+    /**
+     * @return Exchange version.
+     */
+    @Nullable public GridDhtPartitionExchangeId exchangeId() {
+        return exchId;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public DiscoveryCustomMessage ackMessage() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMutable() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DynamicCacheChangeFailureMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index c62ffd2aec79c..a994bd46f1f71 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -269,8 +269,17 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                                 exchFut = exchangeFuture(exchId, evt, cache, null, msg);
                             }
                         }
-                        else if (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
-                            exchangeFuture(msg.exchangeId(), null, null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
+                        else if (msg.exchangeId().topologyVersion().topologyVersion() >=
+                            affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                            exchangeFuture(msg.exchangeId(), null, null, null, null)
+                                .onAffinityChangeMessage(customEvt.eventNode(), msg);
+                    }
+                    else if (customEvt.customMessage() instanceof DynamicCacheChangeFailureMessage) {
+                        DynamicCacheChangeFailureMessage msg =
+                            (DynamicCacheChangeFailureMessage)customEvt.customMessage();
+
+                        exchangeFuture(msg.exchangeId(), null, null, null, null)
+                            .onDynamicCacheChangeFail(customEvt.eventNode(), msg);
                     }
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 35074e9ba7cc7..2fd181dba75b8 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1747,6 +1747,20 @@ public void blockGateway(DynamicCacheChangeRequest req) {
         }
     }
 
+    /**
+     * Called during exchange rollback in order to stop the given cache
+     * even if it's not fully initialized (e.g. fail on cache init stage).
+     */
+     public void forceCloseCache(DynamicCacheChangeRequest req) {
+        assert req.stop() : req;
+
+        registeredCaches.remove(maskNull(req.cacheName()));
+
+        stopGateway(req);
+
+        prepareCacheStop(req, true);
+    }
+
     /**
      * @param req Request.
      */
@@ -1850,7 +1864,17 @@ else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId()) || forc
                         }
                     }
                 }
+            }
+        }
+    }
 
+    /**
+     * @param reqs Collection of requests to complete future for.
+     * @param err Error to be passed to futures.
+     */
+    public void completeStartFutures(Collection<DynamicCacheChangeRequest> reqs, @Nullable Throwable err) {
+        if (!F.isEmpty(reqs)) {
+            for (DynamicCacheChangeRequest req : reqs) {
                 completeStartFuture(req, err);
             }
         }
@@ -2597,10 +2621,26 @@ public boolean onCustomEvent(DiscoveryCustomMessage msg,
         AffinityTopologyVersion topVer) {
         if (msg instanceof CacheAffinityChangeMessage)
             return sharedCtx.affinity().onCustomEvent(((CacheAffinityChangeMessage)msg));
+        else if (msg instanceof DynamicCacheChangeFailureMessage)
+            return onCacheChangeRequested((DynamicCacheChangeFailureMessage)msg);
 
         return msg instanceof DynamicCacheChangeBatch && onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer);
     }
 
+    /**
+     * @param failMsg Dynamic change change request fail message.
+     * @return {@code True} if minor topology version should be increased.
+     */
+    private boolean onCacheChangeRequested(DynamicCacheChangeFailureMessage failMsg) {
+        for (DynamicCacheChangeRequest req : failMsg.requests()) {
+            registeredCaches.remove(maskNull(req.cacheName()));
+
+            ctx.discovery().removeCacheFilter(req.cacheName(), true);
+        }
+
+        return false;
+    }
+
     /**
      * @param batch Change request batch.
      * @param topVer Current topology version.
@@ -2738,7 +2778,7 @@ private boolean onCacheChangeRequested(DynamicCacheChangeBatch batch,
 
                         assert old != null : "Dynamic cache map was concurrently modified [req=" + req + ']';
 
-                        ctx.discovery().removeCacheFilter(req.cacheName());
+                        ctx.discovery().removeCacheFilter(req.cacheName(), false);
 
                         needExchange = true;
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index d3e3701936786..6a7145442f49e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -51,6 +51,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
+import org.apache.ignite.internal.processors.cache.DynamicCacheChangeFailureMessage;
 import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -74,6 +75,7 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteRunnable;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -82,6 +84,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_EXCHANGE_ROLLBACK_SUPPORTED;
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
 
@@ -161,10 +164,10 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
      * Messages received on non-coordinator are stored in case if this node
      * becomes coordinator.
      */
-    private final Map<ClusterNode, GridDhtPartitionsSingleMessage> singleMsgs = new ConcurrentHashMap8<>();
+    private Map<ClusterNode, GridDhtPartitionsSingleMessage> singleMsgs = new ConcurrentHashMap8<>();
 
     /** Messages received from new coordinator. */
-    private final Map<ClusterNode, GridDhtPartitionsFullMessage> fullMsgs = new ConcurrentHashMap8<>();
+    private Map<ClusterNode, GridDhtPartitionsFullMessage> fullMsgs = new ConcurrentHashMap8<>();
 
     /** */
     @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"})
@@ -198,6 +201,12 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** */
     private boolean centralizedAff;
 
+    /** Exception that was thrown during init phase on local node. */
+    private IgniteCheckedException exchangeLocalException;
+
+    /** Exchange exceptions from all participating nodes. */
+    private Map<UUID, Exception> exchangeGlobalExceptions = new ConcurrentHashMap8<>();
+
     /** Forced Rebalance future. */
     private GridCompoundFuture<Boolean, Boolean> forcedRebFut;
 
@@ -536,11 +545,46 @@ public void init() throws IgniteInterruptedCheckedException {
         catch (IgniteNeedReconnectException e) {
             onDone(e);
         }
+        catch (Exception e) {
+            if (reconnectOnError(e)) {
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+
+                return;
+            }
+
+            U.error(log, "Failed to reinitialize local partitions (rebalancing will be stopped): " + exchId, e);
+
+            if (cctx.kernalContext().clientNode() || !isRollbackSupported()) {
+                onDone(e);
+
+                return;
+            }
+
+            exchangeLocalException = new IgniteCheckedException(
+                "Failed to initialize exchange locally [locNodeId=" + cctx.localNodeId() + "]", e);
+
+            exchangeGlobalExceptions.put(cctx.localNodeId(), exchangeLocalException);
+
+            if (crd.isLocal()) {
+                boolean isRemainingEmpty = false;
+
+                synchronized (mux) {
+                    isRemainingEmpty = remaining.isEmpty();
+                }
+
+                if (isRemainingEmpty)
+                    onAllReceived(false);
+            }
+            else
+                sendPartitions(crd);
+
+            initDone();
+        }
         catch (Throwable e) {
             if (reconnectOnError(e))
                 onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
             else {
-                U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
+                U.error(log, "Failed to reinitialize local partitions (rebalancing will be stopped): " + exchId, e);
 
                 onDone(e);
             }
@@ -977,6 +1021,9 @@ private void sendLocalPartitions(ClusterNode node)
             clientOnlyExchange,
             true);
 
+        if (exchangeLocalException != null)
+            m.setError(exchangeLocalException);
+
         if (log.isDebugEnabled())
             log.debug("Sending local partitions [nodeId=" + node.id() + ", exchId=" + exchId + ", msg=" + m + ']');
 
@@ -1084,6 +1131,8 @@ private void sendPartitions(ClusterNode oldestNode) {
 
         cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err, false);
 
+        cctx.cache().completeStartFutures(reqs, err);
+
         if (super.onDone(res, err) && realExchange) {
             if (log.isDebugEnabled())
                 log.debug("Completed partition exchange [localNode=" + cctx.localNodeId() + ", exchange= " + this +
@@ -1130,10 +1179,12 @@ private void sendPartitions(ClusterNode oldestNode) {
      * Cleans up resources to avoid excessive memory usage.
      */
     public void cleanUp() {
-        singleMsgs.clear();
-        fullMsgs.clear();
+        singleMsgs = null;
+        fullMsgs = null;
         crd = null;
         partReleaseFut = null;
+        exchangeLocalException = null;
+        exchangeGlobalExceptions = null;
     }
 
     /**
@@ -1210,6 +1261,9 @@ private void processMessage(ClusterNode node, GridDhtPartitionsSingleMessage msg
 
                     pendingSingleUpdates++;
 
+                    if (msg.getError() != null)
+                        exchangeGlobalExceptions.put(node.id(), msg.getError());
+
                     allReceived = remaining.isEmpty();
                 }
             }
@@ -1278,6 +1332,82 @@ private void onAffinityInitialized(IgniteInternalFuture<Map<Integer, Map<Integer
         }
     }
 
+    /**
+     * Creates an IgniteCheckedException that is used as root cause of the exchange initialization failure.
+     * This method aggregates all the exceptions provided from all participating nodes.
+     *
+     * @param globalExceptions collection exceptions from all participating nodes.
+     * @return exception that represents a cause of the exchange initialization failure.
+     */
+    private IgniteCheckedException createExchangeException(Map<UUID, Exception> globalExceptions) {
+        IgniteCheckedException ex;
+
+        if (exchangeLocalException != null)
+            ex = exchangeLocalException;
+        else
+            ex = new IgniteCheckedException("Failed to complete exchange process (will try to rollback).");
+
+        for (Map.Entry<UUID, Exception> entry : globalExceptions.entrySet()) {
+            // avoid self-suppression
+            if (ex != entry.getValue())
+                ex.addSuppressed(entry.getValue());
+        }
+
+        return ex;
+    }
+
+    /**
+     * Returns {@code true} if the given {@code discoEvt} supports the rollback procedure.
+     *
+     * @return {@code true} if the given {@code discoEvt} supports the rollback procedure.
+     */
+    private boolean isRollbackSupported() {
+        boolean rollbackSupported = false;
+
+        for (ClusterNode node : discoCache.allNodes()) {
+            Boolean exchangeSupported = node.attribute(ATTR_EXCHANGE_ROLLBACK_SUPPORTED);
+            if (exchangeSupported == null || !exchangeSupported)
+                return false;
+        }
+
+        // Currently the rollback process is supported for dynamically started caches.
+        if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && !F.isEmpty(reqs)) {
+            for (DynamicCacheChangeRequest req : reqs) {
+                if (req.start()) {
+                    rollbackSupported = true;
+
+                    break;
+                }
+            }
+        }
+
+        return rollbackSupported;
+    }
+
+    /**
+     * Tries to revert all the changes that were done during initialization phase
+     * in case of the given {@code discoEvt} supports the rollback procedure.
+     */
+    private void rollbackExchange() {
+        if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && !F.isEmpty(reqs)) {
+            for (DynamicCacheChangeRequest req : reqs) {
+                if (req.start()) {
+                    DynamicCacheChangeRequest stopReq =
+                        new DynamicCacheChangeRequest(req.cacheName(), cctx.localNodeId());
+
+                    stopReq.stop(true);
+                    stopReq.deploymentId(req.deploymentId());
+
+                    // cleanup GridCacheProcessor
+                    cctx.cache().forceCloseCache(stopReq);
+
+                    // cleanup CacheAffinitySharedManager
+                    cctx.affinity().forceCloseCache(this, crd.isLocal(), Collections.singletonList(stopReq));
+                }
+            }
+        }
+    }
+
     /**
      * @param discoThread If {@code true} completes future from another thread (to do not block discovery thread).
      */
@@ -1285,6 +1415,24 @@ private void onAllReceived(boolean discoThread) {
         try {
             assert crd.isLocal();
 
+            if (!F.isEmpty(exchangeGlobalExceptions) && isRollbackSupported()) {
+                updateLastVersion(cctx.versions().last());
+
+                cctx.versions().onExchange(lastVer.get().order());
+
+                IgniteCheckedException err = createExchangeException(exchangeGlobalExceptions);
+
+                DynamicCacheChangeFailureMessage msg = new DynamicCacheChangeFailureMessage(
+                    cctx.localNode(), exchId, lastVer.get(), err, reqs);
+
+                if (log.isDebugEnabled())
+                    log.debug("Dynamic cache change failed. Send message to all participating nodes: " + msg);
+
+                cctx.discovery().sendCustomEvent(msg);
+
+                return;
+            }
+
             if (!crd.equals(discoCache.serverNodes().get(0))) {
                 for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
                     if (!cacheCtx.isLocal())
@@ -1488,6 +1636,37 @@ private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
         }
     }
 
+    /**
+     * Cache change failure message callback, processed from the discovery thread.
+     *
+     * @param node Message sender node.
+     * @param msg Message.
+     */
+    public void onDynamicCacheChangeFail(final ClusterNode node, final DynamicCacheChangeFailureMessage msg) {
+        assert exchId.equals(msg.exchangeId()) : msg;
+        assert msg.lastVersion() != null : msg;
+
+        onDiscoveryEvent(new IgniteRunnable() {
+            @Override public void run() {
+                if (isDone() || !enterBusy())
+                    return;
+
+                try {
+                    if (isRollbackSupported())
+                        rollbackExchange();
+
+                    if (!crd.isLocal())
+                        cctx.versions().onExchange(msg.lastVersion().order());
+
+                    onDone(exchId.topologyVersion(), msg.getError());
+                }
+                finally {
+                    leaveBusy();
+                }
+            }
+        });
+    }
+
     /**
      * Affinity change message callback, processed from the same thread as {@link #onNodeLeft}.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index bf08f0abf8832..abc05c0afb83e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -64,6 +64,14 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
     /** Serialized partitions counters. */
     private byte[] partCntrsBytes;
 
+    /** Exception. */
+    @GridToStringInclude
+    @GridDirectTransient
+    private Exception err;
+
+    /** */
+    private byte[] errBytes;
+
     /** */
     private boolean client;
 
@@ -149,6 +157,20 @@ public Map<Integer, Long> partitionUpdateCounters(int cacheId) {
         return Collections.emptyMap();
     }
 
+    /**
+     * @param ex Exception.
+     */
+    public void setError(Exception ex) {
+        this.err = ex;
+    }
+
+    /**
+     * @return Not null exception if exchange processing failed.
+     */
+    @Nullable public Exception getError() {
+        return err;
+    }
+
     /**
      * @return Local partitions.
      */
@@ -156,8 +178,7 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
         return parts;
     }
 
-    /** {@inheritDoc}
-     * @param ctx*/
+    /** {@inheritDoc} */
     @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
         super.prepareMarshal(ctx);
 
@@ -166,6 +187,7 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
         if (marshal) {
             byte[] partsBytes0 = null;
             byte[] partCntrsBytes0 = null;
+            byte[] errBytes0 = null;
 
             if (parts != null && partsBytes == null)
                 partsBytes0 = U.marshal(ctx, parts);
@@ -173,15 +195,20 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
             if (partCntrs != null && partCntrsBytes == null)
                 partCntrsBytes0 = U.marshal(ctx, partCntrs);
 
+            if (err != null && errBytes == null)
+                errBytes0 = U.marshal(ctx, err);
+
             if (compress) {
                 assert !compressed();
 
                 try {
                     byte[] partsBytesZip = U.zip(partsBytes0);
                     byte[] partCntrsBytesZip = U.zip(partCntrsBytes0);
+                    byte[] exBytesZip = U.zip(errBytes0);
 
                     partsBytes0 = partsBytesZip;
                     partCntrsBytes0 = partCntrsBytesZip;
+                    errBytes0 = exBytesZip;
 
                     compressed(true);
                 }
@@ -192,6 +219,7 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
 
             partsBytes = partsBytes0;
             partCntrsBytes = partCntrsBytes0;
+            errBytes = errBytes0;
         }
     }
 
@@ -213,6 +241,13 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
                 partCntrs = U.unmarshal(ctx, partCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
         }
 
+        if (errBytes != null && err == null) {
+            if (compressed())
+                err = U.unmarshalZip(ctx.marshaller(), errBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+            else
+                err = U.unmarshal(ctx, errBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+        }
+
         if (dupPartsData != null) {
             assert parts != null;
 
@@ -262,12 +297,18 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
                 writer.incrementState();
 
             case 8:
-                if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
+                if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
             case 9:
+                if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
+                    return false;
+
+                writer.incrementState();
+
+            case 10:
                 if (!writer.writeByteArray("partsBytes", partsBytes))
                     return false;
 
@@ -306,7 +347,7 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
                 reader.incrementState();
 
             case 8:
-                partCntrsBytes = reader.readByteArray("partCntrsBytes");
+                errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
                     return false;
@@ -314,6 +355,14 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
                 reader.incrementState();
 
             case 9:
+                partCntrsBytes = reader.readByteArray("partCntrsBytes");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 10:
                 partsBytes = reader.readByteArray("partsBytes");
 
                 if (!reader.isLastRead())
@@ -333,7 +382,7 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 10;
+        return 11;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java
new file mode 100644
index 0000000000000..fba54aa80ed92
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import javax.cache.CacheException;
+import javax.cache.configuration.Factory;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.AffinityFunctionContext;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+@SuppressWarnings("unchecked")
+public class IgniteDynamicCacheStartFailSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final String DYNAMIC_CACHE_NAME = "TestDynamicCache";
+
+    /** Coordinator node. */
+    private Ignite crd;
+
+    /** Coordinator node index. */
+    private int crdIdx;
+
+    /**
+     * @return Number of nodes for this test.
+     */
+    public int nodeCount() {
+        return 3;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        crdIdx = 0;
+        crd = startGrid(crdIdx);
+
+        for (int i = 1; i < nodeCount(); ++i)
+            startGrid(i);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    public void testBrokenAffinityFunctionOnAllNodes() {
+        final boolean failOnAllNodes = true;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenAffinityFunctionOnInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 1;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenAffinityFunctionOnNonInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 2;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenAffinityFunctionOnCoordinatorDiffInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = (crdIdx + 1) % nodeCount();
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenAffinityFunctionOnCoordinator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = crdIdx;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenAffinityFunctionWithNodeFilter() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 0;
+        final int numberOfCaches = 1;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, true),
+            initiator);
+    }
+
+    public void testBrokenCacheStoreOnAllNodes() {
+        final boolean failOnAllNodes = true;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenCacheStoreOnInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 1;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenCacheStoreOnNonInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 2;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenCacheStoreOnCoordinatorDiffInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = (crdIdx + 1) % nodeCount();
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenCacheStoreFunctionOnCoordinator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = crdIdx;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testCreateCacheMultipleTimes() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 0;
+        final int numOfAttempts = 100;
+
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFunction(
+            failOnAllNodes, unluckyNode, unluckyCfg, 1, false).get(0);
+
+        for (int i = 0; i < numOfAttempts; ++i) {
+            try {
+                IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+
+                fail("Expected exception was not thrown");
+            }
+            catch (CacheException e) {
+            }
+        }
+    }
+
+    private List<CacheConfiguration> createCacheConfigsWithBrokenAffinityFunction(
+        boolean failOnAllNodes,
+        int unluckyNode,
+        final int unluckyCfg,
+        int cacheNum,
+        boolean useFilter
+        ) {
+        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
+
+        final UUID uuid = ignite(unluckyNode).cluster().localNode().id();
+
+        List<CacheConfiguration> cfgs = new ArrayList<>();
+
+        for (int i = 0; i < cacheNum; ++i) {
+            CacheConfiguration cfg = new CacheConfiguration();
+
+            cfg.setName(DYNAMIC_CACHE_NAME + "-" + i);
+
+            if (i == unluckyCfg)
+                cfg.setAffinity(new BrokenAffinityFunction(failOnAllNodes, getTestGridName(unluckyNode)));
+
+            if (useFilter)
+                cfg.setNodeFilter(new NodeFilter(uuid));
+
+            cfgs.add(cfg);
+        }
+
+        return cfgs;
+    }
+
+    private Collection<CacheConfiguration> createCacheConfigsWithBrokenCacheStore(
+        boolean failOnAllNodes,
+        int unluckyNode,
+        int unluckyCfg,
+        int cacheNum,
+        boolean useFilter
+    ) {
+        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
+
+        final UUID uuid = ignite(unluckyNode).cluster().localNode().id();
+
+        List<CacheConfiguration> cfgs = new ArrayList<>();
+
+        for (int i = 0; i < cacheNum; ++i) {
+            CacheConfiguration cfg = new CacheConfiguration();
+
+            cfg.setName(DYNAMIC_CACHE_NAME + "-" + i);
+
+            if (i == unluckyCfg)
+                cfg.setCacheStoreFactory(new BrokenStoreFactory(failOnAllNodes, getTestGridName(unluckyNode)));
+
+            if (useFilter)
+                cfg.setNodeFilter(new NodeFilter(uuid));
+
+            cfgs.add(cfg);
+        }
+
+        return cfgs;
+    }
+
+    private void testDynamicCacheStart(final Collection<CacheConfiguration> cfgs, final int initiatorId) {
+        assert initiatorId < nodeCount();
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                grid(initiatorId).getOrCreateCaches(cfgs);
+                return null;
+            }
+        }, CacheException.class, null);
+
+        for (CacheConfiguration cfg : cfgs)
+            assertNull("initiatorId=" + initiatorId, grid(initiatorId).cache(cfg.getName()));
+    }
+
+    private static class NodeFilter implements IgnitePredicate<ClusterNode> {
+        /** Cache should be created node with certain UUID. */
+        public UUID uuid;
+
+        public NodeFilter() {
+        }
+
+        public NodeFilter(UUID uuid) {
+            this.uuid = uuid;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode clusterNode) {
+            return clusterNode.id().equals(uuid);
+        }
+    }
+
+    private static class BrokenAffinityFunction extends RendezvousAffinityFunction {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Exception should arise on all nodes. */
+        private boolean exceptionOnAllNodes = false;
+
+        /** Exception should arise on node with certain name. */
+        private String gridName;
+
+        public BrokenAffinityFunction() {
+        }
+
+        public BrokenAffinityFunction(boolean exceptionOnAllNodes, String gridName) {
+            this.exceptionOnAllNodes = exceptionOnAllNodes;
+            this.gridName = gridName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext affCtx) {
+            if (exceptionOnAllNodes || ignite.name().equals(gridName))
+                throw new IllegalStateException("Simulated exception [locNodeId="
+                    + ignite.cluster().localNode().id() + "]");
+            else
+                return super.assignPartitions(affCtx);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(ObjectOutput out) throws IOException {
+            super.writeExternal(out);
+            out.writeBoolean(exceptionOnAllNodes);
+            out.writeObject(gridName);
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("unchecked")
+        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+            super.readExternal(in);
+            exceptionOnAllNodes = in.readBoolean();
+            gridName = (String)in.readObject();
+        }
+    }
+
+    private static class BrokenStoreFactory implements Factory<CacheStore<Integer, String>> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Exception should arise on all nodes. */
+        boolean exceptionOnAllNodes = true;
+
+        /** Exception should arise on node with certain name. */
+        public static String gridName;
+
+        public BrokenStoreFactory() {
+        }
+
+        public BrokenStoreFactory(boolean exceptionOnAllNodes, String gridName) {
+            this.exceptionOnAllNodes = exceptionOnAllNodes;
+            this.gridName = gridName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public CacheStore<Integer, String> create() {
+            if (exceptionOnAllNodes || ignite.name().equals(gridName))
+                throw new IllegalStateException("Simulated exception [locNodeId="
+                    + ignite.cluster().localNode().id() + "]");
+            else
+                return null;
+        }
+    }
+}

From 07e211df9589e037e690c4d0b67034287cfd1317 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 24 Aug 2017 12:12:22 +0300
Subject: [PATCH 386/446] IGNITE-1094: final review

---
 .../managers/discovery/DiscoCache.java        |  19 ++
 .../discovery/GridDiscoveryManager.java       |   5 +-
 .../cache/CacheAffinitySharedManager.java     |  93 ++----
 .../DynamicCacheChangeFailureMessage.java     |  51 ++-
 .../GridCachePartitionExchangeManager.java    |  11 +-
 .../processors/cache/GridCacheProcessor.java  |  77 ++---
 .../GridDhtPartitionsExchangeFuture.java      | 126 +++++---
 .../GridDhtPartitionsSingleMessage.java       |   4 +-
 .../IgniteDynamicCacheStartFailSelfTest.java  | 291 ++++++++++++++----
 .../testsuites/IgniteCacheTestSuite4.java     |   2 +
 10 files changed, 420 insertions(+), 259 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index 5247ac1b28091..30d2b7d9a5c07 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -295,6 +295,25 @@ public void updateAlives(GridDiscoveryManager discovery) {
         }
     }
 
+    /**
+     * Returns {@code True} if all nodes has the given attribute and its value equals to {@code expVal}.
+     *
+     * @param <T> Attribute Type.
+     * @param name Attribute name.
+     * @param expVal Expected value.
+     * @return {@code True} if all the given nodes has the given attribute and its value equals to {@code expVal}.
+     */
+    public <T> boolean checkAttribute(String name, T expVal) {
+        for (ClusterNode node : allNodes) {
+            T attr = node.attribute(name);
+
+            if (attr == null || !expVal.equals(attr))
+                return false;
+        }
+
+        return true;
+    }
+
     /**
      * @param nodes Cluster nodes.
      * @return Empty collection if nodes list is {@code null}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 1d67869bb478e..40dea984ee56d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -320,12 +320,11 @@ public void setCacheFilter(
      * Removes dynamic cache filter.
      *
      * @param cacheName Cache name.
-     * @param force Removes dynamic cache filter without check that it was previously created.
      */
-    public void removeCacheFilter(String cacheName, boolean force) {
+    public void removeCacheFilter(String cacheName) {
         CachePredicate p = registeredCachesPreds.remove(cacheName);
 
-        assert force || p != null : cacheName;
+        assert p != null : cacheName;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index fc9e02ce294ca..7cbd333907058 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -322,6 +322,16 @@ public void onCacheCreated(GridCacheContext cctx) {
         }
     }
 
+    /**
+     * @param cacheId Cache ID.
+     * @return {@code True} if cache is in wait list.
+     */
+    private boolean waitCache(int cacheId) {
+        synchronized (mux) {
+            return waitInfo != null && waitInfo.waitCaches.containsKey(cacheId);
+        }
+    }
+
     /**
      * Called during exchange rollback in order to stop the given cache(s)
      * even if it's not fully initialized (e.g. fail on cache init stage).
@@ -332,7 +342,6 @@ public void onCacheCreated(GridCacheContext cctx) {
      */
     public void forceCloseCache(final GridDhtPartitionsExchangeFuture fut, boolean crd,
         Collection<DynamicCacheChangeRequest> reqs) {
-
         assert !F.isEmpty(reqs) : fut;
 
         for (DynamicCacheChangeRequest req : reqs) {
@@ -341,6 +350,8 @@ public void forceCloseCache(final GridDhtPartitionsExchangeFuture fut, boolean c
             Integer cacheId = CU.cacheId(req.cacheName());
 
             registeredCaches.remove(cacheId);
+
+            assert !waitCache(cacheId);
         }
 
         Set<Integer> stoppedCaches = null;
@@ -363,34 +374,6 @@ public void forceCloseCache(final GridDhtPartitionsExchangeFuture fut, boolean c
                 }
             }
         }
-
-        if (stoppedCaches != null) {
-            boolean notify = false;
-
-            synchronized (mux) {
-                if (waitInfo != null) {
-                    for (Integer cacheId : stoppedCaches) {
-                        boolean rmv =  waitInfo.waitCaches.remove(cacheId) != null;
-
-                        if (rmv) {
-                            notify = true;
-
-                            waitInfo.assignments.remove(cacheId);
-                        }
-                    }
-                }
-            }
-
-            if (notify) {
-                final AffinityTopologyVersion topVer = affCalcVer;
-
-                cctx.kernalContext().closure().runLocalSafe(new Runnable() {
-                    @Override public void run() {
-                        onCacheStopped(topVer);
-                    }
-                });
-            }
-        }
     }
 
     /**
@@ -450,48 +433,38 @@ else if (req.start() && !req.clientStartOnly()) {
             Integer cacheId = CU.cacheId(req.cacheName());
 
             if (req.start()) {
-                try {
-                    cctx.cache().prepareCacheStart(req, fut.topologyVersion());
+                cctx.cache().prepareCacheStart(req, fut.topologyVersion());
 
-                    if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
-                        if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
-                            U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
-                    }
+                if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
+                    if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
+                        U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+                }
 
-                    if (!crd || !lateAffAssign) {
-                        GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+                if (!crd || !lateAffAssign) {
+                    GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
-                        if (cacheCtx != null && !cacheCtx.isLocal()) {
-                            boolean clientCacheStarted =
-                                req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
+                    if (cacheCtx != null && !cacheCtx.isLocal()) {
+                        boolean clientCacheStarted =
+                            req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
 
-                            if (clientCacheStarted)
-                                initAffinity(cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
-                            else if (!req.clientStartOnly()) {
-                                assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
+                        if (clientCacheStarted)
+                            initAffinity(cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
+                        else if (!req.clientStartOnly()) {
+                            assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
 
-                                GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
+                            GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
 
-                                assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
+                            assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
 
-                                List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
-                                    fut.discoveryEvent(), fut.discoCache());
+                            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
+                                fut.discoveryEvent(), fut.discoCache());
 
-                                aff.initialize(fut.topologyVersion(), assignment);
-                            }
+                            aff.initialize(fut.topologyVersion(), assignment);
                         }
                     }
-                    else
-                        initStartedCacheOnCoordinator(fut, cacheId);
-                }
-                catch (IgniteCheckedException | RuntimeException e) {
-                    U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
-                        "[cacheName=" + req.cacheName() + ']', e);
-
-                    cctx.cache().forceCloseCache(fut.topologyVersion(), req, e);
-
-                    throw e;
                 }
+                else
+                    initStartedCacheOnCoordinator(fut, cacheId);
             }
             else if (req.stop() || req.close()) {
                 cctx.cache().blockGateway(req);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
index 162c67351cb32..c755bb1b31f85 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
@@ -18,14 +18,12 @@
 package org.apache.ignite.internal.processors.cache;
 
 import java.util.Collection;
-import java.util.Map;
-import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
@@ -37,9 +35,9 @@ public class DynamicCacheChangeFailureMessage implements DiscoveryCustomMessage
     /** */
     private static final long serialVersionUID = 0L;
 
-    /** Change requests. */
+    /** Cache names. */
     @GridToStringInclude
-    private Collection<DynamicCacheChangeRequest> reqs;
+    private Collection<String> cacheNames;
 
     /** Custom message ID. */
     private IgniteUuid id;
@@ -48,31 +46,31 @@ public class DynamicCacheChangeFailureMessage implements DiscoveryCustomMessage
     private GridDhtPartitionExchangeId exchId;
 
     /** */
-    private GridCacheVersion lastVer;
-
     @GridToStringInclude
     private IgniteCheckedException cause;
 
     /**
      * Creates new DynamicCacheChangeFailureMessage instance.
      *
+     * @param locNode Local node.
      * @param exchId Exchange Id.
-     * @param lastVer Last version.
-     * @param cause
-     * @param reqs Cache change requests.
+     * @param cause Cache start error.
+     * @param cacheNames Cache names.
      */
     public DynamicCacheChangeFailureMessage(
-        ClusterNode localNode,
+        ClusterNode locNode,
         GridDhtPartitionExchangeId exchId,
-        @Nullable GridCacheVersion lastVer,
         IgniteCheckedException cause,
-        Collection<DynamicCacheChangeRequest> reqs)
+        Collection<String> cacheNames)
     {
-        this.id = IgniteUuid.fromUuid(localNode.id());
+        assert exchId != null;
+        assert cause != null;
+        assert !F.isEmpty(cacheNames) : cacheNames;
+
+        this.id = IgniteUuid.fromUuid(locNode.id());
         this.exchId = exchId;
-        this.lastVer = lastVer;
         this.cause = cause;
-        this.reqs = reqs;
+        this.cacheNames = cacheNames;
     }
 
     /** {@inheritDoc} */
@@ -81,30 +79,19 @@ public DynamicCacheChangeFailureMessage(
     }
 
     /**
-     * @param id Message ID.
+     * @return Collection of failed caches.
      */
-    public void id(IgniteUuid id) {
-        this.id = id;
+    public Collection<String> cacheNames() {
+        return cacheNames;
     }
 
     /**
-     * @return Collection of change requests.
+     * @return Cache start error.
      */
-    public Collection<DynamicCacheChangeRequest> requests() {
-        return reqs;
-    }
-
-    public IgniteCheckedException getError() {
+    public IgniteCheckedException error() {
         return cause;
     }
 
-    /**
-     * @return Last used version among all nodes.
-     */
-    @Nullable public GridCacheVersion lastVersion() {
-        return lastVer;
-    }
-
     /**
      * @return Exchange version.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index a994bd46f1f71..4809fd2c0df73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -278,8 +278,10 @@ else if (customEvt.customMessage() instanceof DynamicCacheChangeFailureMessage)
                         DynamicCacheChangeFailureMessage msg =
                             (DynamicCacheChangeFailureMessage)customEvt.customMessage();
 
-                        exchangeFuture(msg.exchangeId(), null, null, null, null)
-                            .onDynamicCacheChangeFail(customEvt.eventNode(), msg);
+                        if (msg.exchangeId().topologyVersion().topologyVersion() >=
+                            affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                            exchangeFuture(msg.exchangeId(), null, null, null, null)
+                                .onDynamicCacheChangeFail(customEvt.eventNode(), msg);
                     }
                 }
 
@@ -1279,6 +1281,9 @@ private void processSinglePartitionUpdate(final ClusterNode node, final GridDhtP
                     log.debug("Received local partition update [nodeId=" + node.id() + ", parts=" +
                         msg + ']');
 
+                if (msg.partitions() == null)
+                    return;
+
                 boolean updated = false;
 
                 for (Map.Entry<Integer, GridDhtPartitionMap2> entry : msg.partitions().entrySet()) {
@@ -1593,7 +1598,7 @@ private void dumpPendingObjects(@Nullable AffinityTopologyVersion exchTopVer) {
      * @return {@code True} if can use compression for partition map messages.
      */
     @SuppressWarnings("SimplifiableIfStatement")
-    private boolean canUsePartitionMapCompression(ClusterNode node) {
+    public boolean canUsePartitionMapCompression(ClusterNode node) {
         IgniteProductVersion ver = node.version();
 
         if (ver.compareToIgnoreTimestamp(GridDhtPartitionsAbstractMessage.PART_MAP_COMPRESS_SINCE) >= 0) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 2fd181dba75b8..673294184a365 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1750,12 +1750,12 @@ public void blockGateway(DynamicCacheChangeRequest req) {
     /**
      * Called during exchange rollback in order to stop the given cache
      * even if it's not fully initialized (e.g. fail on cache init stage).
+     *
+     * @param req Stop request.
      */
      public void forceCloseCache(DynamicCacheChangeRequest req) {
         assert req.stop() : req;
 
-        registeredCaches.remove(maskNull(req.cacheName()));
-
         stopGateway(req);
 
         prepareCacheStop(req, true);
@@ -1795,35 +1795,18 @@ private void prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose)
         }
     }
 
-    /**
-     * Closes cache even if it's not fully initialized (e.g. fail on cache init stage).
-     *
-     * @param topVer Completed topology version.
-     * @param req Change request.
-     * @param err Error.
-     */
-    void forceCloseCache(
-        AffinityTopologyVersion topVer,
-        DynamicCacheChangeRequest req,
-        Throwable err
-    ) {
-        onExchangeDone(topVer, Collections.singleton(req), err, true);
-    }
-
     /**
      * Callback invoked when first exchange future for dynamic cache is completed.
      *
      * @param topVer Completed topology version.
      * @param reqs Change requests.
      * @param err Error.
-     * @param forceClose Close cache despite flags in requests.
      */
     @SuppressWarnings("unchecked")
     public void onExchangeDone(
         AffinityTopologyVersion topVer,
         Collection<DynamicCacheChangeRequest> reqs,
-        Throwable err,
-        boolean forceClose
+        Throwable err
     ) {
         for (GridCacheAdapter<?, ?> cache : caches.values()) {
             GridCacheContext<?, ?> cacheCtx = cache.context();
@@ -1838,43 +1821,35 @@ public void onExchangeDone(
             }
         }
 
-        if (!F.isEmpty(reqs) && (err == null || forceClose)) {
+        if (!F.isEmpty(reqs)) {
             for (DynamicCacheChangeRequest req : reqs) {
-                String masked = maskNull(req.cacheName());
+                if (err == null) {
+                    String masked = maskNull(req.cacheName());
 
-                if (req.stop()) {
-                    stopGateway(req);
+                    if (req.stop()) {
+                        stopGateway(req);
 
-                    prepareCacheStop(req, forceClose);
-                }
-                else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId()) || forceClose) {
-                    IgniteCacheProxy<?, ?> proxy = jCacheProxies.remove(masked);
+                        prepareCacheStop(req, false);
+                    }
+                    else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId())) {
+                        IgniteCacheProxy<?, ?> proxy = jCacheProxies.remove(masked);
 
-                    if (proxy != null) {
-                        if (proxy.context().affinityNode()) {
-                            GridCacheAdapter<?, ?> cache = caches.get(masked);
+                        if (proxy != null) {
+                            if (proxy.context().affinityNode()) {
+                                GridCacheAdapter<?, ?> cache = caches.get(masked);
 
-                            if (cache != null)
-                                jCacheProxies.put(masked, new IgniteCacheProxy(cache.context(), cache, null, false));
-                        }
-                        else {
-                            proxy.context().gate().onStopped();
+                                if (cache != null)
+                                    jCacheProxies.put(masked, new IgniteCacheProxy(cache.context(), cache, null, false));
+                            }
+                            else {
+                                proxy.context().gate().onStopped();
 
-                            prepareCacheStop(req, forceClose);
+                                prepareCacheStop(req, false);
+                            }
                         }
                     }
                 }
-            }
-        }
-    }
 
-    /**
-     * @param reqs Collection of requests to complete future for.
-     * @param err Error to be passed to futures.
-     */
-    public void completeStartFutures(Collection<DynamicCacheChangeRequest> reqs, @Nullable Throwable err) {
-        if (!F.isEmpty(reqs)) {
-            for (DynamicCacheChangeRequest req : reqs) {
                 completeStartFuture(req, err);
             }
         }
@@ -2632,10 +2607,10 @@ else if (msg instanceof DynamicCacheChangeFailureMessage)
      * @return {@code True} if minor topology version should be increased.
      */
     private boolean onCacheChangeRequested(DynamicCacheChangeFailureMessage failMsg) {
-        for (DynamicCacheChangeRequest req : failMsg.requests()) {
-            registeredCaches.remove(maskNull(req.cacheName()));
+        for (String cacheName : failMsg.cacheNames()) {
+            registeredCaches.remove(maskNull(cacheName));
 
-            ctx.discovery().removeCacheFilter(req.cacheName(), true);
+            ctx.discovery().removeCacheFilter(cacheName);
         }
 
         return false;
@@ -2778,7 +2753,7 @@ private boolean onCacheChangeRequested(DynamicCacheChangeBatch batch,
 
                         assert old != null : "Dynamic cache map was concurrently modified [req=" + req + ']';
 
-                        ctx.discovery().removeCacheFilter(req.cacheName(), false);
+                        ctx.discovery().removeCacheFilter(req.cacheName());
 
                         needExchange = true;
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 6a7145442f49e..b4d02ec2bdebd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -75,7 +75,6 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
-import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteRunnable;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -164,10 +163,10 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
      * Messages received on non-coordinator are stored in case if this node
      * becomes coordinator.
      */
-    private Map<ClusterNode, GridDhtPartitionsSingleMessage> singleMsgs = new ConcurrentHashMap8<>();
+    private final Map<ClusterNode, GridDhtPartitionsSingleMessage> singleMsgs = new ConcurrentHashMap8<>();
 
     /** Messages received from new coordinator. */
-    private Map<ClusterNode, GridDhtPartitionsFullMessage> fullMsgs = new ConcurrentHashMap8<>();
+    private final Map<ClusterNode, GridDhtPartitionsFullMessage> fullMsgs = new ConcurrentHashMap8<>();
 
     /** */
     @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"})
@@ -202,10 +201,10 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     private boolean centralizedAff;
 
     /** Exception that was thrown during init phase on local node. */
-    private IgniteCheckedException exchangeLocalException;
+    private IgniteCheckedException exchangeLocE;
 
     /** Exchange exceptions from all participating nodes. */
-    private Map<UUID, Exception> exchangeGlobalExceptions = new ConcurrentHashMap8<>();
+    private final Map<UUID, Exception> exchangeGlobalExceptions = new ConcurrentHashMap8<>();
 
     /** Forced Rebalance future. */
     private GridCompoundFuture<Boolean, Boolean> forcedRebFut;
@@ -552,31 +551,39 @@ public void init() throws IgniteInterruptedCheckedException {
                 return;
             }
 
-            U.error(log, "Failed to reinitialize local partitions (rebalancing will be stopped): " + exchId, e);
+            U.error(log, "Failed to initialize cache (will try to rollback). " + exchId, e);
 
-            if (cctx.kernalContext().clientNode() || !isRollbackSupported()) {
+            if (!isRollbackSupported()) {
                 onDone(e);
 
                 return;
             }
 
-            exchangeLocalException = new IgniteCheckedException(
+            exchangeLocE = new IgniteCheckedException(
                 "Failed to initialize exchange locally [locNodeId=" + cctx.localNodeId() + "]", e);
 
-            exchangeGlobalExceptions.put(cctx.localNodeId(), exchangeLocalException);
+            exchangeGlobalExceptions.put(cctx.localNodeId(), exchangeLocE);
 
-            if (crd.isLocal()) {
-                boolean isRemainingEmpty = false;
+            if (crd != null) {
+                if (crd.isLocal()) {
+                    boolean allRcvd;
 
-                synchronized (mux) {
-                    isRemainingEmpty = remaining.isEmpty();
+                    synchronized (mux) {
+                        allRcvd = remaining.isEmpty();
+                    }
+
+                    if (allRcvd)
+                        onAllReceived(false);
                 }
+                else {
+                    clientOnlyExchange = cctx.kernalContext().clientNode();
 
-                if (isRemainingEmpty)
-                    onAllReceived(false);
+                    if (!centralizedAff)
+                        sendPartitions(crd);
+                }
             }
             else
-                sendPartitions(crd);
+                onDone(e);
 
             initDone();
         }
@@ -1016,13 +1023,21 @@ public boolean stopping(int cacheId) {
      */
     private void sendLocalPartitions(ClusterNode node)
         throws IgniteCheckedException {
-        GridDhtPartitionsSingleMessage m = cctx.exchange().createPartitionsSingleMessage(node,
-            exchangeId(),
-            clientOnlyExchange,
-            true);
+        GridDhtPartitionsSingleMessage m;
 
-        if (exchangeLocalException != null)
-            m.setError(exchangeLocalException);
+        if (exchangeLocE == null)
+            m = cctx.exchange().createPartitionsSingleMessage(node,
+                exchangeId(),
+                clientOnlyExchange,
+                true);
+        else {
+            m = new GridDhtPartitionsSingleMessage(exchangeId(),
+                clientOnlyExchange,
+                cctx.versions().last(),
+                cctx.exchange().canUsePartitionMapCompression(node));
+
+            m.setError(exchangeLocE);
+        }
 
         if (log.isDebugEnabled())
             log.debug("Sending local partitions [nodeId=" + node.id() + ", exchId=" + exchId + ", msg=" + m + ']');
@@ -1129,9 +1144,7 @@ private void sendPartitions(ClusterNode oldestNode) {
 
         cctx.exchange().onExchangeDone(this, err);
 
-        cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err, false);
-
-        cctx.cache().completeStartFutures(reqs, err);
+        cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err);
 
         if (super.onDone(res, err) && realExchange) {
             if (log.isDebugEnabled())
@@ -1179,12 +1192,12 @@ private void sendPartitions(ClusterNode oldestNode) {
      * Cleans up resources to avoid excessive memory usage.
      */
     public void cleanUp() {
-        singleMsgs = null;
-        fullMsgs = null;
+        singleMsgs.clear();
+        fullMsgs.clear();
         crd = null;
         partReleaseFut = null;
-        exchangeLocalException = null;
-        exchangeGlobalExceptions = null;
+        exchangeLocE = null;
+        exchangeGlobalExceptions.clear();
     }
 
     /**
@@ -1222,10 +1235,17 @@ public void onReceive(final ClusterNode node, final GridDhtPartitionsSingleMessa
                 log.debug("Received message for finished future (will reply only to sender) [msg=" + msg +
                     ", fut=" + this + ']');
 
+            // Custom message (DynamicCacheChangeFailureMessage) was sent.
+            // Do not need sendAllPartitions.
+            if (!exchangeGlobalExceptions.isEmpty())
+                return;
+
             if (!centralizedAff)
                 sendAllPartitions(node.id(), cctx.gridConfig().getNetworkSendRetryCount());
         }
         else {
+            assert !msg.client();
+
             initFut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
                 @Override public void apply(IgniteInternalFuture<Boolean> f) {
                     try {
@@ -1342,13 +1362,13 @@ private void onAffinityInitialized(IgniteInternalFuture<Map<Integer, Map<Integer
     private IgniteCheckedException createExchangeException(Map<UUID, Exception> globalExceptions) {
         IgniteCheckedException ex;
 
-        if (exchangeLocalException != null)
-            ex = exchangeLocalException;
+        if (exchangeLocE != null)
+            ex = exchangeLocE;
         else
             ex = new IgniteCheckedException("Failed to complete exchange process (will try to rollback).");
 
         for (Map.Entry<UUID, Exception> entry : globalExceptions.entrySet()) {
-            // avoid self-suppression
+            // Avoid self-suppression.
             if (ex != entry.getValue())
                 ex.addSuppressed(entry.getValue());
         }
@@ -1364,11 +1384,8 @@ private IgniteCheckedException createExchangeException(Map<UUID, Exception> glob
     private boolean isRollbackSupported() {
         boolean rollbackSupported = false;
 
-        for (ClusterNode node : discoCache.allNodes()) {
-            Boolean exchangeSupported = node.attribute(ATTR_EXCHANGE_ROLLBACK_SUPPORTED);
-            if (exchangeSupported == null || !exchangeSupported)
-                return false;
-        }
+        if (!discoCache.checkAttribute(ATTR_EXCHANGE_ROLLBACK_SUPPORTED, Boolean.TRUE))
+            return false;
 
         // Currently the rollback process is supported for dynamically started caches.
         if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && !F.isEmpty(reqs)) {
@@ -1398,10 +1415,10 @@ private void rollbackExchange() {
                     stopReq.stop(true);
                     stopReq.deploymentId(req.deploymentId());
 
-                    // cleanup GridCacheProcessor
+                    // Cleanup GridCacheProcessor.
                     cctx.cache().forceCloseCache(stopReq);
 
-                    // cleanup CacheAffinitySharedManager
+                    // Cleanup CacheAffinitySharedManager.
                     cctx.affinity().forceCloseCache(this, crd.isLocal(), Collections.singletonList(stopReq));
                 }
             }
@@ -1416,14 +1433,15 @@ private void onAllReceived(boolean discoThread) {
             assert crd.isLocal();
 
             if (!F.isEmpty(exchangeGlobalExceptions) && isRollbackSupported()) {
-                updateLastVersion(cctx.versions().last());
+                IgniteCheckedException err = createExchangeException(exchangeGlobalExceptions);
 
-                cctx.versions().onExchange(lastVer.get().order());
+                List<String> cacheNames = new ArrayList<>(reqs.size());
 
-                IgniteCheckedException err = createExchangeException(exchangeGlobalExceptions);
+                for (DynamicCacheChangeRequest req : reqs)
+                    cacheNames.add(req.cacheName());
 
                 DynamicCacheChangeFailureMessage msg = new DynamicCacheChangeFailureMessage(
-                    cctx.localNode(), exchId, lastVer.get(), err, reqs);
+                    cctx.localNode(), exchId, err, cacheNames);
 
                 if (log.isDebugEnabled())
                     log.debug("Dynamic cache change failed. Send message to all participating nodes: " + msg);
@@ -1572,6 +1590,8 @@ private void processMessage(ClusterNode node, GridDhtPartitionsFullMessage msg)
         assert msg.exchangeId().equals(exchId) : msg;
         assert msg.lastVersion() != null : msg;
 
+        boolean isRollbackNedeed = false;
+
         synchronized (mux) {
             if (crd == null)
                 return;
@@ -1586,6 +1606,17 @@ private void processMessage(ClusterNode node, GridDhtPartitionsFullMessage msg)
 
                 return;
             }
+
+            if (exchangeLocE != null && isRollbackSupported())
+                isRollbackNedeed = true;
+        }
+
+        if (isRollbackNedeed) {
+            rollbackExchange();
+
+            onDone(exchId.topologyVersion(), exchangeLocE);
+
+            return;
         }
 
         updatePartitionFullMap(msg);
@@ -1625,6 +1656,9 @@ private void updatePartitionFullMap(GridDhtPartitionsFullMessage msg) {
      * @param msg Partitions single message.
      */
     private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
+        if (msg.partitions() == null)
+            return;
+
         for (Map.Entry<Integer, GridDhtPartitionMap2> entry : msg.partitions().entrySet()) {
             Integer cacheId = entry.getKey();
             GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
@@ -1644,7 +1678,6 @@ private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
      */
     public void onDynamicCacheChangeFail(final ClusterNode node, final DynamicCacheChangeFailureMessage msg) {
         assert exchId.equals(msg.exchangeId()) : msg;
-        assert msg.lastVersion() != null : msg;
 
         onDiscoveryEvent(new IgniteRunnable() {
             @Override public void run() {
@@ -1655,10 +1688,7 @@ public void onDynamicCacheChangeFail(final ClusterNode node, final DynamicCacheC
                     if (isRollbackSupported())
                         rollbackExchange();
 
-                    if (!crd.isLocal())
-                        cctx.versions().onExchange(msg.lastVersion().order());
-
-                    onDone(exchId.topologyVersion(), msg.getError());
+                    onDone(exchId.topologyVersion(), msg.error());
                 }
                 finally {
                     leaveBusy();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index abc05c0afb83e..f2664d50b8f8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -182,7 +182,9 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
     @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
         super.prepareMarshal(ctx);
 
-        boolean marshal = (parts != null && partsBytes == null) || (partCntrs != null && partCntrsBytes == null);
+        boolean marshal = (parts != null && partsBytes == null) ||
+            (partCntrs != null && partCntrsBytes == null) ||
+            (err != null && errBytes == null);
 
         if (marshal) {
             byte[] partsBytes0 = null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java
index fba54aa80ed92..2ca40249c29d3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java
@@ -35,8 +35,6 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -45,6 +43,9 @@
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+/**
+ *
+ */
 @SuppressWarnings("unchecked")
 public class IgniteDynamicCacheStartFailSelfTest extends GridCommonAbstractTest {
     /** */
@@ -53,8 +54,8 @@ public class IgniteDynamicCacheStartFailSelfTest extends GridCommonAbstractTest
     /** */
     private static final String DYNAMIC_CACHE_NAME = "TestDynamicCache";
 
-    /** Coordinator node. */
-    private Ignite crd;
+    /** */
+    private static final String CLIENT_GRID_NAME = "client";
 
     /** Coordinator node index. */
     private int crdIdx;
@@ -78,7 +79,8 @@ public int nodeCount() {
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         crdIdx = 0;
-        crd = startGrid(crdIdx);
+
+        startGrid(crdIdx);
 
         for (int i = 1; i < nodeCount(); ++i)
             startGrid(i);
@@ -89,156 +91,307 @@ public int nodeCount() {
         stopAllGrids();
     }
 
-    public void testBrokenAffinityFunctionOnAllNodes() {
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnServerFailedOnClient() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnClient";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-server-1");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, clientName));
+
+        try {
+            IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+        }
+        catch (CacheException e) {
+            fail("Exception should not be thrown.");
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnServerFailedOnServer() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnServer";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-server-2");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, getTestGridName(0)));
+
+        try {
+            IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+
+            fail("Expected exception was not thrown.");
+        }
+        catch (CacheException e) {
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnClientFailOnClient() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnClientFailOnClient";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-client-1");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, clientName));
+
+        try {
+            IgniteCache cache = client.getOrCreateCache(cfg);
+
+            fail("Expected exception was not thrown.");
+        }
+        catch (CacheException e) {
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnClientFailOnServer() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnClientFailOnServer";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-client-2");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, getTestGridName(0)));
+
+        try {
+            IgniteCache cache = client.getOrCreateCache(cfg);
+
+            fail("Expected exception was not thrown.");
+        }
+        catch (CacheException e) {
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on all nodes.
+     */
+    public void testBrokenAffinityFunOnAllNodes() {
         final boolean failOnAllNodes = true;
         final int unluckyNode = 0;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 0;
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenAffinityFunctionOnInitiator() {
+    /**
+     * Test cache start with broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunOnInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 1;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 1;
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenAffinityFunctionOnNonInitiator() {
+    /**
+     * Test cache start with broken affinity function that throws an exception on non-initiator node.
+     */
+    public void testBrokenAffinityFunOnNonInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 1;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 2;
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenAffinityFunctionOnCoordinatorDiffInitiator() {
+    /**
+     * Test cache start with broken affinity function that throws an exception on coordinator node.
+     */
+    public void testBrokenAffinityFunOnCoordinatorDiffInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = crdIdx;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = (crdIdx + 1) % nodeCount();
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenAffinityFunctionOnCoordinator() {
+    /**
+     * Test cache start with broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunOnCoordinator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = crdIdx;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = crdIdx;
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenAffinityFunctionWithNodeFilter() {
+    /**
+     * Tests cache start with node filter and broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunWithNodeFilter() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 0;
         final int unluckyCfg = 0;
-        final int numberOfCaches = 1;
+        final int numOfCaches = 1;
         final int initiator = 0;
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, true),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, true),
             initiator);
     }
 
+    /**
+     * Tests cache start with broken cache store that throws an exception on all nodes.
+     */
     public void testBrokenCacheStoreOnAllNodes() {
         final boolean failOnAllNodes = true;
         final int unluckyNode = 0;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 0;
 
         testDynamicCacheStart(
             createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
+    /**
+     * Tests cache start with broken cache store that throws an exception on initiator node.
+     */
     public void testBrokenCacheStoreOnInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 1;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 1;
 
         testDynamicCacheStart(
             createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
+    /**
+     * Tests cache start with broken cache store that throws an exception on non-initiator node.
+     */
     public void testBrokenCacheStoreOnNonInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 1;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 2;
 
         testDynamicCacheStart(
             createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
+    /**
+     *  Tests cache start with broken cache store that throws an exception on initiator node.
+     */
     public void testBrokenCacheStoreOnCoordinatorDiffInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = crdIdx;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = (crdIdx + 1) % nodeCount();
 
         testDynamicCacheStart(
             createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenCacheStoreFunctionOnCoordinator() {
+    /**
+     *  Tests cache start with broken cache store that throws an exception on coordinator node.
+     */
+    public void testBrokenCacheStoreFunOnCoordinator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = crdIdx;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = crdIdx;
 
         testDynamicCacheStart(
             createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
+    /**
+     *  Tests multiple creation of cache with broken affinity function.
+     */
     public void testCreateCacheMultipleTimes() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 1;
         final int unluckyCfg = 0;
         final int numOfAttempts = 100;
 
-        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFunction(
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
             failOnAllNodes, unluckyNode, unluckyCfg, 1, false).get(0);
 
         for (int i = 0; i < numOfAttempts; ++i) {
@@ -252,7 +405,7 @@ public void testCreateCacheMultipleTimes() {
         }
     }
 
-    private List<CacheConfiguration> createCacheConfigsWithBrokenAffinityFunction(
+    private List<CacheConfiguration> createCacheConfigsWithBrokenAffinityFun(
         boolean failOnAllNodes,
         int unluckyNode,
         final int unluckyCfg,
@@ -321,18 +474,18 @@ private void testDynamicCacheStart(final Collection<CacheConfiguration> cfgs, fi
                 return null;
             }
         }, CacheException.class, null);
-
-        for (CacheConfiguration cfg : cfgs)
-            assertNull("initiatorId=" + initiatorId, grid(initiatorId).cache(cfg.getName()));
     }
 
+    /**
+     * Filter specifying on which node the cache should be started.
+     */
     private static class NodeFilter implements IgnitePredicate<ClusterNode> {
         /** Cache should be created node with certain UUID. */
         public UUID uuid;
 
-        public NodeFilter() {
-        }
-
+        /**
+         * @param uuid node ID.
+         */
         public NodeFilter(UUID uuid) {
             this.uuid = uuid;
         }
@@ -343,6 +496,9 @@ public NodeFilter(UUID uuid) {
         }
     }
 
+    /**
+     * Factory that throws an exception is got created.
+     */
     private static class BrokenAffinityFunction extends RendezvousAffinityFunction {
         /** */
         private static final long serialVersionUID = 0L;
@@ -352,22 +508,30 @@ private static class BrokenAffinityFunction extends RendezvousAffinityFunction {
         private Ignite ignite;
 
         /** Exception should arise on all nodes. */
-        private boolean exceptionOnAllNodes = false;
+        private boolean eOnAllNodes = false;
 
         /** Exception should arise on node with certain name. */
         private String gridName;
 
+        /**
+         * Default constructor.
+         */
         public BrokenAffinityFunction() {
+            // No-op.
         }
 
-        public BrokenAffinityFunction(boolean exceptionOnAllNodes, String gridName) {
-            this.exceptionOnAllNodes = exceptionOnAllNodes;
+        /**
+         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
+         * @param gridName Exception should arise on node with certain name.
+         */
+        public BrokenAffinityFunction(boolean eOnAllNodes, String gridName) {
+            this.eOnAllNodes = eOnAllNodes;
             this.gridName = gridName;
         }
 
         /** {@inheritDoc} */
         @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext affCtx) {
-            if (exceptionOnAllNodes || ignite.name().equals(gridName))
+            if (eOnAllNodes || ignite.name().equals(gridName))
                 throw new IllegalStateException("Simulated exception [locNodeId="
                     + ignite.cluster().localNode().id() + "]");
             else
@@ -377,7 +541,7 @@ public BrokenAffinityFunction(boolean exceptionOnAllNodes, String gridName) {
         /** {@inheritDoc} */
         @Override public void writeExternal(ObjectOutput out) throws IOException {
             super.writeExternal(out);
-            out.writeBoolean(exceptionOnAllNodes);
+            out.writeBoolean(eOnAllNodes);
             out.writeObject(gridName);
         }
 
@@ -385,33 +549,38 @@ public BrokenAffinityFunction(boolean exceptionOnAllNodes, String gridName) {
         @SuppressWarnings("unchecked")
         @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
             super.readExternal(in);
-            exceptionOnAllNodes = in.readBoolean();
+            eOnAllNodes = in.readBoolean();
             gridName = (String)in.readObject();
         }
     }
 
+    /**
+     * Factory that throws an exception is got created.
+     */
     private static class BrokenStoreFactory implements Factory<CacheStore<Integer, String>> {
         /** */
         @IgniteInstanceResource
         private Ignite ignite;
 
         /** Exception should arise on all nodes. */
-        boolean exceptionOnAllNodes = true;
+        boolean eOnAllNodes = true;
 
         /** Exception should arise on node with certain name. */
         public static String gridName;
 
-        public BrokenStoreFactory() {
-        }
+        /**
+         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
+         * @param gridName Exception should arise on node with certain name.
+         */
+        public BrokenStoreFactory(boolean eOnAllNodes, String gridName) {
+            this.eOnAllNodes = eOnAllNodes;
 
-        public BrokenStoreFactory(boolean exceptionOnAllNodes, String gridName) {
-            this.exceptionOnAllNodes = exceptionOnAllNodes;
-            this.gridName = gridName;
+            BrokenStoreFactory.gridName = gridName;
         }
 
         /** {@inheritDoc} */
         @Override public CacheStore<Integer, String> create() {
-            if (exceptionOnAllNodes || ignite.name().equals(gridName))
+            if (eOnAllNodes || ignite.name().equals(gridName))
                 throw new IllegalStateException("Simulated exception [locNodeId="
                     + ignite.cluster().localNode().id() + "]");
             else
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 791ed6bf9d4bc..a4ec552f562ed 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -85,6 +85,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxStoreValueTest;
 import org.apache.ignite.internal.processors.cache.IgniteClientCacheInitializationFailTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheFilterTest;
+import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartNoExchangeTimeoutTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartStopConcurrentTest;
@@ -228,6 +229,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheTxPreloadNoWriteTest.class);
 
         suite.addTestSuite(IgniteDynamicCacheStartSelfTest.class);
+        suite.addTestSuite(IgniteDynamicCacheStartFailSelfTest.class);
         suite.addTestSuite(IgniteDynamicCacheWithConfigStartSelfTest.class);
         suite.addTestSuite(IgniteCacheDynamicStopSelfTest.class);
         suite.addTestSuite(IgniteDynamicCacheStartStopConcurrentTest.class);

From e47a4c03bc65a55349981ad79eb6a9f7fa657a2a Mon Sep 17 00:00:00 2001
From: sk0x50 <slava.koptilin@gmail.com>
Date: Fri, 25 Aug 2017 10:48:33 +0300
Subject: [PATCH 387/446] IGNITE-1094: added cacheChangeFailureMsgSent flag

---
 .../preloader/GridDhtPartitionsExchangeFuture.java   | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index b4d02ec2bdebd..524412bf55f5f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -206,6 +206,9 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** Exchange exceptions from all participating nodes. */
     private final Map<UUID, Exception> exchangeGlobalExceptions = new ConcurrentHashMap8<>();
 
+    /** Used to track the fact that {@code DynamicCacheChangeFailureMessage} was sent. */
+    private volatile boolean cacheChangeFailureMsgSent;
+
     /** Forced Rebalance future. */
     private GridCompoundFuture<Boolean, Boolean> forcedRebFut;
 
@@ -1235,9 +1238,8 @@ public void onReceive(final ClusterNode node, final GridDhtPartitionsSingleMessa
                 log.debug("Received message for finished future (will reply only to sender) [msg=" + msg +
                     ", fut=" + this + ']');
 
-            // Custom message (DynamicCacheChangeFailureMessage) was sent.
-            // Do not need sendAllPartitions.
-            if (!exchangeGlobalExceptions.isEmpty())
+            // Custom message (DynamicCacheChangeFailureMessage) was sent. Do not need sendAllPartitions.
+            if (cacheChangeFailureMsgSent)
                 return;
 
             if (!centralizedAff)
@@ -1444,10 +1446,12 @@ private void onAllReceived(boolean discoThread) {
                     cctx.localNode(), exchId, err, cacheNames);
 
                 if (log.isDebugEnabled())
-                    log.debug("Dynamic cache change failed. Send message to all participating nodes: " + msg);
+                    log.debug("Dynamic cache change failed (send message to all participating nodes): " + msg);
 
                 cctx.discovery().sendCustomEvent(msg);
 
+                cacheChangeFailureMsgSent = true;
+
                 return;
             }
 

From d90a9207756b035294945f696fa0c00c69bdd330 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 25 Aug 2017 12:28:37 +0300
Subject: [PATCH 388/446] ignite-1094 review

---
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java   | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 524412bf55f5f..66527298efd37 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1448,10 +1448,10 @@ private void onAllReceived(boolean discoThread) {
                 if (log.isDebugEnabled())
                     log.debug("Dynamic cache change failed (send message to all participating nodes): " + msg);
 
-                cctx.discovery().sendCustomEvent(msg);
-
                 cacheChangeFailureMsgSent = true;
 
+                cctx.discovery().sendCustomEvent(msg);
+
                 return;
             }
 
@@ -1660,8 +1660,11 @@ private void updatePartitionFullMap(GridDhtPartitionsFullMessage msg) {
      * @param msg Partitions single message.
      */
     private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
-        if (msg.partitions() == null)
+        if (msg.partitions() == null) {
+            assert msg.getError() != null : msg;
+
             return;
+        }
 
         for (Map.Entry<Integer, GridDhtPartitionMap2> entry : msg.partitions().entrySet()) {
             Integer cacheId = entry.getKey();

From a7310e49948de5780845acbaada3e6ccce288265 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 25 Aug 2017 19:09:51 +0300
Subject: [PATCH 389/446] IGNITE-1094: review

---
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java       | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 66527298efd37..91ec8d250f692 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1660,11 +1660,8 @@ private void updatePartitionFullMap(GridDhtPartitionsFullMessage msg) {
      * @param msg Partitions single message.
      */
     private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
-        if (msg.partitions() == null) {
-            assert msg.getError() != null : msg;
-
+        if (msg.partitions() == null)
             return;
-        }
 
         for (Map.Entry<Integer, GridDhtPartitionMap2> entry : msg.partitions().entrySet()) {
             Integer cacheId = entry.getKey();

From 75febb824b3e261ec1c15224a024e1d4160f2f42 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Tue, 13 Jun 2017 19:41:55 +0300
Subject: [PATCH 390/446] ignite-2.1.1 Extract Ignite updates checker to
 separate class. Fixed GridUpdateNotifier test.

---
 .../cluster/GridUpdateNotifier.java           | 56 ++++++++--------
 .../cluster/HttpIgniteUpdatesChecker.java     | 65 +++++++++++++++++++
 .../cluster/GridUpdateNotifierSelfTest.java   | 10 ++-
 3 files changed, 100 insertions(+), 31 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
index 592fdd10bde73..83588c26227ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
@@ -64,8 +64,8 @@ class GridUpdateNotifier {
     /** Sleep milliseconds time for worker thread. */
     private static final int WORKER_THREAD_SLEEP_TIME = 5000;
 
-    /** Url for request version. */
-    private final static String UPDATE_NOTIFIER_URL = "https://ignite.run/update_status_ignite-plain-text.php";
+    /** Default url for request Ignite updates. */
+    private final static String DEFAULT_IGNITE_UPDATES_URL = "https://ignite.run/update_status_ignite-plain-text.php";
 
     /** Grid version. */
     private final String ver;
@@ -103,22 +103,27 @@ class GridUpdateNotifier {
     /** Worker thread to process http request. */
     private final Thread workerThread;
 
+    /** Http client for getting Ignite updates */
+    private final HttpIgniteUpdatesChecker updatesChecker;
+
     /**
      * Creates new notifier with default values.
      *
-     * @param gridName gridName
+     * @param igniteInstanceName Ignite instance name.
      * @param ver Compound Ignite version.
      * @param gw Kernal gateway.
      * @param pluginProviders Kernal gateway.
      * @param reportOnlyNew Whether or not to report only new version.
+     * @param updatesChecker Service for getting Ignite updates
      * @throws IgniteCheckedException If failed.
      */
-    GridUpdateNotifier(String gridName, String ver, GridKernalGateway gw, Collection<PluginProvider> pluginProviders,
-        boolean reportOnlyNew) throws IgniteCheckedException {
+    GridUpdateNotifier(String igniteInstanceName, String ver, GridKernalGateway gw, Collection<PluginProvider> pluginProviders,
+                       boolean reportOnlyNew, HttpIgniteUpdatesChecker updatesChecker) throws IgniteCheckedException {
         try {
             this.ver = ver;
-            this.gridName = gridName == null ? "null" : gridName;
+            this.gridName = igniteInstanceName == null ? "null" : igniteInstanceName;
             this.gw = gw;
+            this.updatesChecker = updatesChecker;
 
             SB pluginsBuilder = new SB();
 
@@ -159,6 +164,14 @@ class GridUpdateNotifier {
         }
     }
 
+    /**
+     * Creates new notifier with default Ignite updates URL
+     */
+    GridUpdateNotifier(String igniteInstanceName, String ver, GridKernalGateway gw, Collection<PluginProvider> pluginProviders,
+                       boolean reportOnlyNew) throws IgniteCheckedException {
+        this(igniteInstanceName, ver, gw, pluginProviders, reportOnlyNew, new HttpIgniteUpdatesChecker(DEFAULT_IGNITE_UPDATES_URL, CHARSET));
+    }
+
     /**
      * Gets system properties.
      *
@@ -313,34 +326,17 @@ private class UpdateChecker extends GridWorker {
                     (!F.isEmpty(vmProps) ? "&vmProps=" + encode(vmProps, CHARSET) : "") +
                         pluginsVers;
 
-                URLConnection conn = new URL(UPDATE_NOTIFIER_URL).openConnection();
-
                 if (!isCancelled()) {
-                    conn.setDoOutput(true);
-                    conn.setRequestProperty("Accept-Charset", CHARSET);
-                    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=" + CHARSET);
-
-                    conn.setConnectTimeout(3000);
-                    conn.setReadTimeout(3000);
-
                     try {
-                        try (OutputStream os = conn.getOutputStream()) {
-                            os.write(postParams.getBytes(CHARSET));
-                        }
-
-                        try (InputStream in = conn.getInputStream()) {
-                            if (in == null)
-                                return;
-
-                            BufferedReader reader = new BufferedReader(new InputStreamReader(in, CHARSET));
+                        String updatesResponse = updatesChecker.getUpdates(postParams);
 
-                            for (String line; (line = reader.readLine()) != null; ) {
-                                if (line.contains("version"))
-                                    latestVer = obtainVersionFrom(line);
-                                else if (line.contains("downloadUrl"))
-                                    downloadUrl = obtainDownloadUrlFrom(line);
-                            }
+                        String[] lines = updatesResponse.split("\n");
 
+                        for (String line : lines) {
+                            if (line.contains("version"))
+                                latestVer = obtainVersionFrom(line);
+                            else if (line.contains("downloadUrl"))
+                                downloadUrl = obtainDownloadUrlFrom(line);
                         }
                     }
                     catch (IOException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
new file mode 100644
index 0000000000000..c052c0997fe2a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
@@ -0,0 +1,65 @@
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.URL;
+import java.net.URLConnection;
+
+/**
+ * This class is responsible for getting Ignite updates information via HTTP
+ */
+public class HttpIgniteUpdatesChecker {
+    /** Url for request updates. */
+    private final String url;
+
+    /** Charset for encoding requests/responses */
+    private final String charset;
+
+    /**
+     * Creates new HTTP Ignite updates checker with following parameters
+     * @param url URL for getting Ignite updates information
+     * @param charset Charset for encoding
+     */
+    HttpIgniteUpdatesChecker(String url, String charset) {
+        this.url = url;
+        this.charset = charset;
+    }
+
+    /**
+     * Gets information about Ignite updates via HTTP
+     * @param updateRequest HTTP Request parameters
+     * @return Information about Ignite updates separated by line endings
+     * @throws IOException If HTTP request was failed
+     */
+    public String getUpdates(String updateRequest) throws IOException {
+        URLConnection conn = new URL(url).openConnection();
+        conn.setDoOutput(true);
+        conn.setRequestProperty("Accept-Charset", charset);
+        conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=" + charset);
+
+        conn.setConnectTimeout(3000);
+        conn.setReadTimeout(3000);
+
+        try (OutputStream os = conn.getOutputStream()) {
+            os.write(updateRequest.getBytes(charset));
+        }
+
+        try (InputStream in = conn.getInputStream()) {
+            if (in == null)
+                return null;
+
+            BufferedReader reader = new BufferedReader(new InputStreamReader(in, charset));
+
+            StringBuilder response = new StringBuilder();
+
+            for (String line; (line = reader.readLine()) != null; ) {
+                response.append(line).append('\n');
+            }
+
+            return response.toString();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java
index 21b91b6b3c53e..1a20f261550ba 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java
@@ -29,6 +29,8 @@
 import org.apache.ignite.plugin.PluginProvider;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
 
 /**
  * Update notifier test.
@@ -73,8 +75,14 @@ public class GridUpdateNotifierSelfTest extends GridCommonAbstractTest {
     public void testNotifier() throws Exception {
         String nodeVer = IgniteProperties.get("ignite.version");
 
+        HttpIgniteUpdatesChecker updatesCheckerMock = Mockito.mock(HttpIgniteUpdatesChecker.class);
+
+        // Return current node version and some other info
+        Mockito.when(updatesCheckerMock.getUpdates(Matchers.anyString()))
+                .thenReturn("meta=meta" + "\n" + "version=" + nodeVer + "\n" + "downloadUrl=url");
+
         GridUpdateNotifier ntf = new GridUpdateNotifier(null, nodeVer,
-            TEST_GATEWAY, Collections.<PluginProvider>emptyList(), false);
+            TEST_GATEWAY, Collections.<PluginProvider>emptyList(), false, updatesCheckerMock);
 
         ntf.checkForNewVersion(log);
 

From f813fc036802349d81d6c313c4c1846ca5ed8fd7 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Wed, 1 Feb 2017 14:21:49 +0300
Subject: [PATCH 391/446] ignite-4557 Fixed wrong affinity manager call.

---
 .../processors/cache/GridCacheAdapter.java    | 10 ++--
 .../cache/GridCacheAffinityManager.java       | 60 +++++++------------
 .../processors/cache/GridCacheContext.java    | 17 ------
 .../cache/GridCacheEvictionManager.java       |  6 +-
 .../processors/cache/GridCacheUtils.java      | 20 -------
 .../cache/affinity/GridCacheAffinityImpl.java | 16 ++---
 .../CacheDataStructuresManager.java           |  2 +-
 .../distributed/dht/GridDhtCacheAdapter.java  |  2 +-
 .../distributed/dht/GridDhtCacheEntry.java    |  2 +-
 .../dht/GridDhtLocalPartition.java            |  4 +-
 .../dht/GridDhtPartitionTopologyImpl.java     |  4 +-
 .../distributed/dht/GridDhtTxRemote.java      |  2 +-
 .../dht/GridPartitionedGetFuture.java         |  2 +-
 .../dht/GridPartitionedSingleGetFuture.java   |  2 +-
 .../dht/atomic/GridDhtAtomicCache.java        | 13 ++--
 .../GridNearAtomicSingleUpdateFuture.java     |  2 +-
 .../atomic/GridNearAtomicUpdateFuture.java    |  4 +-
 .../dht/colocated/GridDhtColocatedCache.java  |  6 +-
 .../colocated/GridDhtColocatedLockFuture.java |  4 +-
 .../preloader/GridDhtPartitionDemander.java   |  8 +--
 .../preloader/GridDhtPartitionSupplier.java   | 12 ++--
 .../dht/preloader/GridDhtPreloader.java       |  4 +-
 .../distributed/near/GridNearAtomicCache.java |  2 +-
 .../distributed/near/GridNearCacheEntry.java  |  6 +-
 .../distributed/near/GridNearGetFuture.java   |  4 +-
 .../distributed/near/GridNearLockFuture.java  |  2 +-
 ...OptimisticSerializableTxPrepareFuture.java |  2 +-
 .../GridNearOptimisticTxPrepareFuture.java    |  2 +-
 .../GridNearPessimisticTxPrepareFuture.java   |  2 +-
 .../near/GridNearTransactionalCache.java      |  6 +-
 .../near/GridNearTxFinishFuture.java          |  2 +-
 .../cache/query/GridCacheQueryManager.java    |  9 +--
 .../CacheContinuousQueryHandler.java          |  1 -
 .../CacheContinuousQueryManager.java          |  2 +-
 .../cache/transactions/IgniteTxAdapter.java   |  4 +-
 .../transactions/IgniteTxLocalAdapter.java    |  4 +-
 .../transactions/TxDeadlockDetection.java     |  2 +-
 .../datastreamer/DataStreamerImpl.java        |  2 +-
 .../datastructures/GridCacheSetImpl.java      |  2 +-
 .../datastructures/GridSetQueryPredicate.java |  2 +-
 .../processors/job/GridJobProcessor.java      |  2 +-
 .../cache/CacheAffinityCallSelfTest.java      |  4 +-
 .../GridCacheAbstractFullApiSelfTest.java     |  6 +-
 ...gniteCacheConfigVariationsFullApiTest.java |  6 +-
 .../IgniteCachePeekModesAbstractTest.java     |  8 +--
 ...tQueueFailoverDataConsistencySelfTest.java |  2 +-
 ...teCacheClientNodeChangingTopologyTest.java |  8 +--
 .../TxOptimisticDeadlockDetectionTest.java    |  2 +-
 .../TxPessimisticDeadlockDetectionTest.java   |  2 +-
 .../processors/query/h2/IgniteH2Indexing.java |  2 +-
 .../query/h2/opt/GridH2IndexBase.java         |  2 +-
 51 files changed, 126 insertions(+), 176 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 7aa48199bb28a..9b1e0cc7721c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -778,7 +778,7 @@ public String toString() {
                 boolean nearKey;
 
                 if (!(modes.near && modes.primary && modes.backup)) {
-                    boolean keyPrimary = ctx.affinity().primary(ctx.localNode(), part, topVer);
+                    boolean keyPrimary = ctx.affinity().primaryByPartition(ctx.localNode(), part, topVer);
 
                     if (keyPrimary) {
                         if (!modes.primary)
@@ -787,7 +787,7 @@ public String toString() {
                         nearKey = false;
                     }
                     else {
-                        boolean keyBackup = ctx.affinity().belongs(ctx.localNode(), part, topVer);
+                        boolean keyBackup = ctx.affinity().partitionBelongs(ctx.localNode(), part, topVer);
 
                         if (keyBackup) {
                             if (!modes.backup)
@@ -808,7 +808,7 @@ public String toString() {
                     }
                 }
                 else {
-                    nearKey = !ctx.affinity().belongs(ctx.localNode(), part, topVer);
+                    nearKey = !ctx.affinity().partitionBelongs(ctx.localNode(), part, topVer);
 
                     if (nearKey) {
                         // Swap and offheap are disabled for near cache.
@@ -3813,8 +3813,8 @@ IgniteInternalFuture<?> globalLoadCacheAsync(@Nullable IgniteBiPredicate<K, V> p
             /** {@inheritDoc} */
             @Override public boolean apply(ClusterNode clusterNode) {
                 return clusterNode.version().compareTo(PartitionSizeLongTask.SINCE_VER) >= 0 &&
-                    ((modes.primary && aff.primary(clusterNode, part, topVer)) ||
-                        (modes.backup && aff.backup(clusterNode, part, topVer)));
+                    ((modes.primary && aff.primaryByPartition(clusterNode, part, topVer)) ||
+                        (modes.backup && aff.backupByPartition(clusterNode, part, topVer)));
             }
         }).nodes();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
index 8b7be1b3288e2..d85e76e2bbd82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
@@ -238,8 +238,8 @@ public Object affinityKey(Object key) {
      * @param topVer Topology version.
      * @return Affinity nodes.
      */
-    public List<ClusterNode> nodes(Object key, AffinityTopologyVersion topVer) {
-        return nodes(partition(key), topVer);
+    public List<ClusterNode> nodesByKey(Object key, AffinityTopologyVersion topVer) {
+        return nodesByPartition(partition(key), topVer);
     }
 
     /**
@@ -247,7 +247,7 @@ public List<ClusterNode> nodes(Object key, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Affinity nodes.
      */
-    public List<ClusterNode> nodes(int part, AffinityTopologyVersion topVer) {
+    public List<ClusterNode> nodesByPartition(int part, AffinityTopologyVersion topVer) {
         if (cctx.isLocal())
             topVer = LOC_CACHE_TOP_VER;
 
@@ -282,8 +282,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Primary node for given key.
      */
-    @Nullable public ClusterNode primary(Object key, AffinityTopologyVersion topVer) {
-        return primary(partition(key), topVer);
+    @Nullable public ClusterNode primaryByKey(Object key, AffinityTopologyVersion topVer) {
+        return primaryByPartition(partition(key), topVer);
     }
 
     /**
@@ -291,8 +291,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Primary node for given key.
      */
-    @Nullable public ClusterNode primary(int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    @Nullable public ClusterNode primaryByPartition(int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         if (nodes.isEmpty())
             return null;
@@ -306,8 +306,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code True} if checked node is primary for given key.
      */
-    public boolean primary(ClusterNode n, Object key, AffinityTopologyVersion topVer) {
-        return F.eq(primary(key, topVer), n);
+    public boolean primaryByKey(ClusterNode n, Object key, AffinityTopologyVersion topVer) {
+        return F.eq(primaryByKey(key, topVer), n);
     }
 
     /**
@@ -316,8 +316,8 @@ public boolean primary(ClusterNode n, Object key, AffinityTopologyVersion topVer
      * @param topVer Topology version.
      * @return {@code True} if checked node is primary for given partition.
      */
-    public boolean primary(ClusterNode n, int part, AffinityTopologyVersion topVer) {
-        return F.eq(primary(part, topVer), n);
+    public boolean primaryByPartition(ClusterNode n, int part, AffinityTopologyVersion topVer) {
+        return F.eq(primaryByPartition(part, topVer), n);
     }
 
     /**
@@ -325,8 +325,8 @@ public boolean primary(ClusterNode n, int part, AffinityTopologyVersion topVer)
      * @param topVer Topology version.
      * @return Backup nodes.
      */
-    public Collection<ClusterNode> backups(Object key, AffinityTopologyVersion topVer) {
-        return backups(partition(key), topVer);
+    public Collection<ClusterNode> backupsByKey(Object key, AffinityTopologyVersion topVer) {
+        return backupsByPartition(partition(key), topVer);
     }
 
     /**
@@ -334,8 +334,8 @@ public Collection<ClusterNode> backups(Object key, AffinityTopologyVersion topVe
      * @param topVer Topology version.
      * @return Backup nodes.
      */
-    public Collection<ClusterNode> backups(int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    private Collection<ClusterNode> backupsByPartition(int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         assert !F.isEmpty(nodes);
 
@@ -351,35 +351,21 @@ public Collection<ClusterNode> backups(int part, AffinityTopologyVersion topVer)
      * @param topVer Topology version.
      * @return {@code True} if checked node is a backup node for given partition.
      */
-    public boolean backup(ClusterNode n, int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    public boolean backupByPartition(ClusterNode n, int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         assert !F.isEmpty(nodes);
 
         return nodes.indexOf(n) > 0;
     }
 
-    /**
-     * @param keys keys.
-     * @param topVer Topology version.
-     * @return Nodes for the keys.
-     */
-    public Collection<ClusterNode> remoteNodes(Iterable keys, AffinityTopologyVersion topVer) {
-        Collection<Collection<ClusterNode>> colcol = new GridLeanSet<>();
-
-        for (Object key : keys)
-            colcol.add(nodes(key, topVer));
-
-        return F.view(F.flatCollections(colcol), F.remoteNodes(cctx.localNodeId()));
-    }
-
     /**
      * @param key Key to check.
      * @param topVer Topology version.
      * @return {@code true} if given key belongs to local node.
      */
-    public boolean localNode(Object key, AffinityTopologyVersion topVer) {
-        return localNode(partition(key), topVer);
+    public boolean keyLocalNode(Object key, AffinityTopologyVersion topVer) {
+        return partitionLocalNode(partition(key), topVer);
     }
 
     /**
@@ -387,10 +373,10 @@ public boolean localNode(Object key, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code true} if given partition belongs to local node.
      */
-    public boolean localNode(int part, AffinityTopologyVersion topVer) {
+    public boolean partitionLocalNode(int part, AffinityTopologyVersion topVer) {
         assert part >= 0 : "Invalid partition: " + part;
 
-        return nodes(part, topVer).contains(cctx.localNode());
+        return nodesByPartition(part, topVer).contains(cctx.localNode());
     }
 
     /**
@@ -399,11 +385,11 @@ public boolean localNode(int part, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code true} if given partition belongs to specified node.
      */
-    public boolean belongs(ClusterNode node, int part, AffinityTopologyVersion topVer) {
+    public boolean partitionBelongs(ClusterNode node, int part, AffinityTopologyVersion topVer) {
         assert node != null;
         assert part >= 0 : "Invalid partition: " + part;
 
-        return nodes(part, topVer).contains(node);
+        return nodesByPartition(part, topVer).contains(node);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 6322f9f03550a..3b44b5096f0b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -1589,23 +1589,6 @@ private void map(GridDhtCacheEntry entry, Iterable<ClusterNode> nodes,
         }
     }
 
-    /**
-     * Checks if at least one of the given keys belongs to one of the given partitions.
-     *
-     * @param keys Collection of keys to check.
-     * @param movingParts Collection of partitions to check against.
-     * @return {@code True} if there exist a key in collection {@code keys} that belongs
-     *      to one of partitions in {@code movingParts}
-     */
-    public boolean hasKey(Iterable<? extends K> keys, Collection<Integer> movingParts) {
-        for (K key : keys) {
-            if (movingParts.contains(affinity().partition(key)))
-                return true;
-        }
-
-        return false;
-    }
-
     /**
      * Check whether conflict resolution is required.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
index 134e743d4fd6e..f8722d6a4f338 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
@@ -808,7 +808,7 @@ public void touch(GridCacheEntryEx e, AffinityTopologyVersion topVer) {
             return;
 
         // Don't track non-primary entries if evicts are synchronized.
-        if (!cctx.isNear() && evictSync && !cctx.affinity().primary(cctx.localNode(), e.partition(), topVer))
+        if (!cctx.isNear() && evictSync && !cctx.affinity().primaryByPartition(cctx.localNode(), e.partition(), topVer))
             return;
 
         if (!busyLock.enterBusy())
@@ -910,7 +910,7 @@ public boolean evict(@Nullable GridCacheEntryEx entry, @Nullable GridCacheVersio
         if (evictSyncAgr) {
             assert !cctx.isNear(); // Make sure cache is not NEAR.
 
-            if (cctx.affinity().backups(
+            if (cctx.affinity().backupsByKey(
                     entry.key(),
                     cctx.topology().topologyVersion()).contains(cctx.localNode()) &&
                 evictSync)
@@ -1498,7 +1498,7 @@ void addEvent(DiscoveryEvent evt) {
                         if (!evts.isEmpty())
                             break;
 
-                        if (!cctx.affinity().primary(loc, it.next(), topVer))
+                        if (!cctx.affinity().primaryByPartition(loc, it.next(), topVer))
                             it.remove();
                     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 8ee77e36c3a77..5c21d7e4a8e98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -916,26 +916,6 @@ public static <K, V> void unwindEvicts(GridCacheSharedContext<K, V> ctx) {
             unwindEvicts(cacheCtx);
     }
 
-    /**
-     * Gets primary node on which given key is cached.
-     *
-     * @param ctx Cache.
-     * @param key Key to find primary node for.
-     * @return Primary node for the key.
-     */
-    @SuppressWarnings( {"unchecked"})
-    @Nullable public static ClusterNode primaryNode(GridCacheContext ctx, Object key) {
-        assert ctx != null;
-        assert key != null;
-
-        CacheConfiguration cfg = ctx.cache().configuration();
-
-        if (cfg.getCacheMode() != PARTITIONED)
-            return ctx.localNode();
-
-        return ctx.affinity().primary(key, ctx.affinity().affinityTopologyVersion());
-    }
-
     /**
      * @param asc {@code True} for ascending.
      * @return Descending order comparator.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
index 9e85bad610432..11361a27d384a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
@@ -82,21 +82,21 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public boolean isPrimary(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().primary(n, key, topologyVersion());
+        return cctx.affinity().primaryByKey(n, key, topologyVersion());
     }
 
     /** {@inheritDoc} */
     @Override public boolean isBackup(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().backups(key, topologyVersion()).contains(n);
+        return cctx.affinity().backupsByKey(key, topologyVersion()).contains(n);
     }
 
     /** {@inheritDoc} */
     @Override public boolean isPrimaryOrBackup(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().belongs(n, cctx.affinity().partition(key), topologyVersion());
+        return cctx.affinity().partitionBelongs(n, cctx.affinity().partition(key), topologyVersion());
     }
 
     /** {@inheritDoc} */
@@ -126,7 +126,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
         AffinityTopologyVersion topVer = topologyVersion();
 
         for (int partsCnt = partitions(), part = 0; part < partsCnt; part++) {
-            for (ClusterNode affNode : cctx.affinity().nodes(part, topVer)) {
+            for (ClusterNode affNode : cctx.affinity().nodesByPartition(part, topVer)) {
                 if (n.id().equals(affNode.id())) {
                     parts.add(part);
 
@@ -142,7 +142,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public ClusterNode mapPartitionToNode(int part) {
         A.ensure(part >= 0 && part < partitions(), "part >= 0 && part < total partitions");
 
-        return F.first(cctx.affinity().nodes(part, topologyVersion()));
+        return F.first(cctx.affinity().nodesByPartition(part, topologyVersion()));
     }
 
     /** {@inheritDoc} */
@@ -204,7 +204,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
         Map<ClusterNode, Collection<K>> res = new HashMap<>(nodesCnt, 1.0f);
 
         for (K key : keys) {
-            ClusterNode primary = cctx.affinity().primary(key, topVer);
+            ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
             if (primary == null)
                 throw new IgniteException("Failed to get primary node [topVer=" + topVer + ", key=" + key + ']');
@@ -227,14 +227,14 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public Collection<ClusterNode> mapKeyToPrimaryAndBackups(K key) {
         A.notNull(key, "key");
 
-        return cctx.affinity().nodes(partition(key), topologyVersion());
+        return cctx.affinity().nodesByPartition(partition(key), topologyVersion());
     }
 
     /** {@inheritDoc} */
     @Override public Collection<ClusterNode> mapPartitionToPrimaryAndBackups(int part) {
         A.ensure(part >= 0 && part < partitions(), "part >= 0 && part < total partitions");
 
-        return cctx.affinity().nodes(part, topologyVersion());
+        return cctx.affinity().nodesByPartition(part, topologyVersion());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index 366a4a920b9ee..2b3080981ec36 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -455,7 +455,7 @@ private void removeSetData(IgniteUuid setId, AffinityTopologyVersion topVer) thr
         Collection<SetItemKey> keys = new ArrayList<>(BATCH_SIZE);
 
         for (SetItemKey key : set) {
-            if (!loc && !aff.primary(cctx.localNode(), key, topVer))
+            if (!loc && !aff.primaryByKey(cctx.localNode(), key, topVer))
                 continue;
 
             keys.add(key);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index be7fa5525deba..2fa934b79db71 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -933,7 +933,7 @@ public void sendTtlUpdateRequest(@Nullable final IgniteCacheExpiryPolicy expiryP
                     AffinityTopologyVersion topVer = ctx.shared().exchange().readyAffinityVersion();
 
                     for (Map.Entry<KeyCacheObject, GridCacheVersion> e : entries.entrySet()) {
-                        List<ClusterNode> nodes = ctx.affinity().nodes(e.getKey(), topVer);
+                        List<ClusterNode> nodes = ctx.affinity().nodesByKey(e.getKey(), topVer);
 
                         for (int i = 0; i < nodes.size(); i++) {
                             ClusterNode node = nodes.get(i);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
index cf4085ba02335..39571ff4720e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
@@ -402,7 +402,7 @@ public Collection<UUID> readers() throws GridCacheEntryRemovedException {
         }
 
         // If remote node is (primary?) or back up, don't add it as a reader.
-        if (cctx.affinity().belongs(node, partition(), topVer)) {
+        if (cctx.affinity().partitionBelongs(node, partition(), topVer)) {
             if (log.isDebugEnabled())
                 log.debug("Ignoring near reader because remote node is affinity node [locNodeId=" + cctx.localNodeId()
                     + ", rmtNodeId=" + nodeId + ", key=" + key + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 668a1cdc006d9..b21463843bbcf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -614,7 +614,7 @@ void onUnlock() {
      * @return {@code True} if local node is primary for this partition.
      */
     public boolean primary(AffinityTopologyVersion topVer) {
-        return cctx.affinity().primary(cctx.localNode(), id, topVer);
+        return cctx.affinity().primaryByPartition(cctx.localNode(), id, topVer);
     }
 
     /**
@@ -622,7 +622,7 @@ public boolean primary(AffinityTopologyVersion topVer) {
      * @return {@code True} if local node is backup for this partition.
      */
     public boolean backup(AffinityTopologyVersion topVer) {
-        return cctx.affinity().backup(cctx.localNode(), id, topVer);
+        return cctx.affinity().backupByPartition(cctx.localNode(), id, topVer);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 1ad0ff0538fa5..23485f985767d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -568,7 +568,7 @@ else if (!node2part.nodeId().equals(loc.id())) {
             for (int p = 0; p < num; p++) {
                 GridDhtLocalPartition locPart = localPartition(p, topVer, false, false);
 
-                if (cctx.affinity().localNode(p, topVer)) {
+                if (cctx.affinity().partitionLocalNode(p, topVer)) {
                     // This partition will be created during next topology event,
                     // which obviously has not happened at this point.
                     if (locPart == null) {
@@ -691,7 +691,7 @@ private GridDhtLocalPartition localPartition(int p,
         try {
             loc = locParts.get(p);
 
-            boolean belongs = cctx.affinity().localNode(p, topVer);
+            boolean belongs = cctx.affinity().partitionLocalNode(p, topVer);
 
             if (loc != null && loc.state() == EVICTED) {
                 locParts.set(p, loc = null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
index 8942ef9d9178c..399736e8f4278 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
@@ -286,7 +286,7 @@ IgniteUuid remoteFutureId() {
             return true;
 
         // Check if we are on the backup node.
-        return !cacheCtx.affinity().backups(key, topVer).contains(cctx.localNode());
+        return !cacheCtx.affinity().backupsByKey(key, topVer).contains(cctx.localNode());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index c41711c245028..519239aecb532 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -380,7 +380,7 @@ private boolean map(
     ) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index 2b5624b489b6f..a3f6b72376e8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -325,7 +325,7 @@ private void map(AffinityTopologyVersion topVer) {
     @Nullable private ClusterNode mapKeyToNode(AffinityTopologyVersion topVer) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index dc6d3dd8271dd..16e51ee0c2815 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -2469,7 +2469,7 @@ private UpdateSingleResult updateSingle(
 
                 assert !(newConflictVer instanceof GridCacheVersionEx) : newConflictVer;
 
-                boolean primary = !req.fastMap() || ctx.affinity().primary(ctx.localNode(), entry.partition(),
+                boolean primary = !req.fastMap() || ctx.affinity().primaryByPartition(ctx.localNode(), entry.partition(),
                     req.topologyVersion());
 
                 Object writeVal = op == TRANSFORM ? req.entryProcessor(i) : req.writeValue(i);
@@ -2559,7 +2559,7 @@ else if (conflictCtx.isMerge())
 
                 if (hasNear) {
                     if (primary && updRes.sendToDht()) {
-                        if (!ctx.affinity().belongs(node, entry.partition(), topVer)) {
+                        if (!ctx.affinity().partitionBelongs(node, entry.partition(), topVer)) {
                             // If put the same value as in request then do not need to send it back.
                             if (op == TRANSFORM || writeVal != updRes.newValue()) {
                                 res.addNearValue(i,
@@ -2690,7 +2690,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
                 Map<KeyCacheObject, CacheObject> storeMap = req.fastMap() ?
                     F.view(putMap, new P1<CacheObject>() {
                         @Override public boolean apply(CacheObject key) {
-                            return ctx.affinity().primary(ctx.localNode(), key, req.topologyVersion());
+                            return ctx.affinity().primaryByKey(ctx.localNode(), key, req.topologyVersion());
                         }
                     }) :
                     putMap;
@@ -2713,7 +2713,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
                 Collection<KeyCacheObject> storeKeys = req.fastMap() ?
                     F.view(rmvKeys, new P1<Object>() {
                         @Override public boolean apply(Object key) {
-                            return ctx.affinity().primary(ctx.localNode(), key, req.topologyVersion());
+                            return ctx.affinity().primaryByKey(ctx.localNode(), key, req.topologyVersion());
                         }
                     }) :
                     rmvKeys;
@@ -2752,7 +2752,8 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
 
                     assert writeVal != null || op == DELETE : "null write value found.";
 
-                    boolean primary = !req.fastMap() || ctx.affinity().primary(ctx.localNode(), entry.key(),
+                    boolean primary = !req.fastMap() || ctx.affinity().primaryByPartition(ctx.localNode(),
+                        entry.partition(),
                         req.topologyVersion());
 
                     Collection<UUID> readers = null;
@@ -2848,7 +2849,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
 
                     if (hasNear) {
                         if (primary) {
-                            if (!ctx.affinity().belongs(node, entry.partition(), topVer)) {
+                            if (!ctx.affinity().partitionBelongs(node, entry.partition(), topVer)) {
                                 int idx = firstEntryIdx + i;
 
                                 if (req.operation() == TRANSFORM) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
index 7376affca4455..891a20c2884a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
@@ -543,7 +543,7 @@ private GridNearAtomicAbstractUpdateRequest mapSingleUpdate(AffinityTopologyVers
         else
             val = EntryProcessorResourceInjectorProxy.wrap(cctx.kernalContext(), (EntryProcessor)val);
 
-        ClusterNode primary = cctx.affinity().primary(cacheKey, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(cacheKey, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to map keys for cache (all partition nodes " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index a252d9ae30c2b..9bdd1becb7600 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -937,7 +937,7 @@ else if (conflictRmvVals != null) {
         else
             val = EntryProcessorResourceInjectorProxy.wrap(cctx.kernalContext(), (EntryProcessor)val);
 
-        ClusterNode primary = cctx.affinity().primary(cacheKey.partition(), topVer);
+        ClusterNode primary = cctx.affinity().primaryByPartition(cacheKey.partition(), topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to map keys for cache (all partition nodes " +
@@ -988,7 +988,7 @@ private List<ClusterNode> mapKey(KeyCacheObject key, AffinityTopologyVersion top
 
         // If we can send updates in parallel - do it.
         return fastMap ? cctx.topology().nodes(affMgr.partition(key), topVer) :
-            Collections.singletonList(affMgr.primary(key, topVer));
+            Collections.singletonList(affMgr.primaryByKey(key, topVer));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index c9fc983d4850c..e1e0ec2bf087e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -174,7 +174,7 @@ public GridDistributedCacheEntry entryExx(
         AffinityTopologyVersion topVer,
         boolean allowDetached
     ) {
-        return allowDetached && !ctx.affinity().primary(ctx.localNode(), key, topVer) ?
+        return allowDetached && !ctx.affinity().primaryByKey(ctx.localNode(), key, topVer) ?
             createEntry(key) : entryExx(key, topVer);
     }
 
@@ -670,7 +670,7 @@ else if (!skipVals && ctx.config().isStatisticsEnabled())
                     assert topVer.compareTo(AffinityTopologyVersion.ZERO) > 0;
 
                     // Send request to remove from remote nodes.
-                    ClusterNode primary = ctx.affinity().primary(key, topVer);
+                    ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                     if (primary == null) {
                         if (log.isDebugEnabled())
@@ -790,7 +790,7 @@ public void removeLocks(long threadId, GridCacheVersion ver, Collection<KeyCache
                         map = U.newHashMap(affNodes.size());
                     }
 
-                    ClusterNode primary = ctx.affinity().primary(key, topVer);
+                    ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                     if (primary == null) {
                         if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index 40e87ee2cfd7c..b373818d1f32f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -1201,7 +1201,7 @@ private boolean mapAsPrimary(Collection<KeyCacheObject> keys, AffinityTopologyVe
         boolean explicit = false;
 
         for (KeyCacheObject key : keys) {
-            if (!cctx.affinity().primary(cctx.localNode(), key, topVer)) {
+            if (!cctx.affinity().primaryByKey(cctx.localNode(), key, topVer)) {
                 // Remove explicit locks added so far.
                 for (KeyCacheObject k : keys)
                     cctx.mvcc().removeExplicitLock(threadId, cctx.txKey(k), lockVer);
@@ -1285,7 +1285,7 @@ private GridNearLockMapping map(
     ) throws IgniteCheckedException {
         assert mapping == null || mapping.node() != null;
 
-        ClusterNode primary = cctx.affinity().primary(key, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to lock keys " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index daae1e2aa144a..8586b69b5961d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -598,7 +598,7 @@ public void handleSupplyMessage(
             for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
                 int p = e.getKey();
 
-                if (cctx.affinity().localNode(p, topVer)) {
+                if (cctx.affinity().partitionLocalNode(p, topVer)) {
                     GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                     assert part != null;
@@ -668,7 +668,7 @@ public void handleSupplyMessage(
 
             // Only request partitions based on latest topology version.
             for (Integer miss : supply.missed()) {
-                if (cctx.affinity().localNode(miss, topVer))
+                if (cctx.affinity().partitionLocalNode(miss, topVer))
                     fut.partitionMissed(id, miss);
             }
 
@@ -1359,7 +1359,7 @@ private void demandFromNode(
                         for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
                             int p = e.getKey();
 
-                            if (cctx.affinity().localNode(p, topVer)) {
+                            if (cctx.affinity().partitionLocalNode(p, topVer)) {
                                 GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                                 assert part != null;
@@ -1436,7 +1436,7 @@ private void demandFromNode(
 
                         // Only request partitions based on latest topology version.
                         for (Integer miss : s.supply().missed()) {
-                            if (cctx.affinity().localNode(miss, topVer))
+                            if (cctx.affinity().partitionLocalNode(miss, topVer))
                                 fut.partitionMissed(node.id(), miss);
                         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index b082c4736cb9c..994242354c05d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -302,7 +302,7 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
                             (Iterator<GridCacheMapEntry>)sctx.entryIt : loc.allEntries().iterator();
 
                         while (entIt.hasNext()) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition, so we send '-1' partition and move on.
                                 s.missed(part);
 
@@ -387,7 +387,7 @@ else if (log.isDebugEnabled())
                             boolean prepared = false;
 
                             while (iter.hasNext()) {
-                                if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                                if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                     // Demander no longer needs this partition,
                                     // so we send '-1' partition and move on.
                                     s.missed(part);
@@ -510,7 +510,7 @@ else if (log.isDebugEnabled())
                             (Iterator<GridCacheEntryInfo>)sctx.entryIt : entries.iterator();
 
                         while (lsnrIt.hasNext()) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition,
                                 // so we send '-1' partition and move on.
                                 s.missed(part);
@@ -808,7 +808,7 @@ private void processOldDemandMessage(GridDhtPartitionDemandMessage d, UUID id) {
                     boolean partMissing = false;
 
                     for (GridCacheEntryEx e : loc.allEntries()) {
-                        if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                        if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                             // Demander no longer needs this partition, so we send '-1' partition and move on.
                             s.missed(part);
 
@@ -859,7 +859,7 @@ else if (log.isDebugEnabled())
                                 boolean prepared = false;
 
                                 for (Map.Entry<byte[], GridCacheSwapEntry> e : iter) {
-                                    if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                                    if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                         // Demander no longer needs this partition,
                                         // so we send '-1' partition and move on.
                                         s.missed(part);
@@ -947,7 +947,7 @@ else if (log.isDebugEnabled())
                         swapLsnr = null;
 
                         for (GridCacheEntryInfo info : entries) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition,
                                 // so we send '-1' partition and move on.
                                 s.missed(part);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 692e7c0de45a6..c012e452220ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -289,7 +289,7 @@ private IgniteCheckedException stopError() {
             }
 
             // If partition belongs to local node.
-            if (cctx.affinity().localNode(p, topVer)) {
+            if (cctx.affinity().partitionLocalNode(p, topVer)) {
                 GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                 assert part != null;
@@ -349,7 +349,7 @@ private IgniteCheckedException stopError() {
      * @return Picked owners.
      */
     private Collection<ClusterNode> pickedOwners(int p, AffinityTopologyVersion topVer) {
-        Collection<ClusterNode> affNodes = cctx.affinity().nodes(p, topVer);
+        Collection<ClusterNode> affNodes = cctx.affinity().nodesByPartition(p, topVer);
 
         int affCnt = affNodes.size();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index b843e4e6d3db1..41632ef338b58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -161,7 +161,7 @@ public void processNearAtomicUpdateResponse(
             if (F.contains(failed, key))
                 continue;
 
-            if (ctx.affinity().belongs(ctx.localNode(), ctx.affinity().partition(key), req.topologyVersion())) { // Reader became backup.
+            if (ctx.affinity().partitionBelongs(ctx.localNode(), ctx.affinity().partition(key), req.topologyVersion())) { // Reader became backup.
                 GridCacheEntryEx entry = peekEx(key);
 
                 if (entry != null && entry.markObsolete(ver))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index 30fc213fc63e2..d022805a76bf2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -112,7 +112,7 @@ public GridNearCacheEntry(
                 return false;
             }
 
-            if (cctx.affinity().backup(cctx.localNode(), part, topVer)) {
+            if (cctx.affinity().backupByPartition(cctx.localNode(), part, topVer)) {
                 this.topVer = AffinityTopologyVersion.NONE;
 
                 return false;
@@ -162,7 +162,7 @@ public void initializeFromDht(AffinityTopologyVersion topVer) throws GridCacheEn
                                 }
                             }
 
-                            ClusterNode primaryNode = cctx.affinity().primary(key, topVer);
+                            ClusterNode primaryNode = cctx.affinity().primaryByKey(key, topVer);
 
                             if (primaryNode == null)
                                 this.topVer = AffinityTopologyVersion.NONE;
@@ -686,7 +686,7 @@ private void primaryNode(UUID nodeId, AffinityTopologyVersion topVer) {
         ClusterNode primary = null;
 
         try {
-            primary = cctx.affinity().primary(part, topVer);
+            primary = cctx.affinity().primaryByPartition(part, topVer);
         }
         catch (IllegalStateException ignore) {
             // Do not have affinity history.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index cb47498be0167..fb2843c02eec9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -413,7 +413,7 @@ private Map<KeyCacheObject, GridNearCacheEntry> map(
     ) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
@@ -726,7 +726,7 @@ private Map<K, V> loadEntries(
                     info.unmarshalValue(cctx, cctx.deploy().globalLoader());
 
                     // Entries available locally in DHT should not be loaded into near cache for reading.
-                    if (!cctx.affinity().localNode(info.key(), cctx.affinity().affinityTopologyVersion())) {
+                    if (!cctx.affinity().keyLocalNode(info.key(), cctx.affinity().affinityTopologyVersion())) {
                         GridNearCacheEntry entry = savedEntries.get(info.key());
 
                         if (entry == null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index 491b4ece0ea13..8035655237635 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -1371,7 +1371,7 @@ private GridNearLockMapping map(
     ) throws IgniteCheckedException {
         assert mapping == null || mapping.node() != null;
 
-        ClusterNode primary = cctx.affinity().primary(key, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to lock keys " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index 4cbfb27d34857..f6ca77a1ef96e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -527,7 +527,7 @@ private void map(
         GridCacheContext cacheCtx = entry.context();
 
         List<ClusterNode> nodes = cacheCtx.isLocal() ?
-            cacheCtx.affinity().nodes(entry.key(), topVer) :
+            cacheCtx.affinity().nodesByKey(entry.key(), topVer) :
             cacheCtx.topology().nodes(cacheCtx.affinity().partition(entry.key()), topVer);
 
         txMapping.addMapping(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 91cfbda6e15d3..7e446f3baa0d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -601,7 +601,7 @@ private GridDistributedTxMapping map(
             nodes = cacheCtx.topology().nodes(cached0.partition(), topVer);
         else
             nodes = cacheCtx.isLocal() ?
-                cacheCtx.affinity().nodes(entry.key(), topVer) :
+                cacheCtx.affinity().nodesByKey(entry.key(), topVer) :
                 cacheCtx.topology().nodes(cacheCtx.affinity().partition(entry.key()), topVer);
 
         txMapping.addMapping(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index 5c09398b4ecb1..28cd3584362cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -194,7 +194,7 @@ private void preparePessimistic() {
             GridCacheContext cacheCtx = txEntry.context();
 
             List<ClusterNode> nodes = cacheCtx.isLocal() ?
-                cacheCtx.affinity().nodes(txEntry.key(), topVer) :
+                cacheCtx.affinity().nodesByKey(txEntry.key(), topVer) :
                 cacheCtx.topology().nodes(cacheCtx.affinity().partition(txEntry.key()), topVer);
 
             ClusterNode primary = F.first(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
index b3eb7551f05a0..940dd809779a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
@@ -476,7 +476,7 @@ private void processLockResponse(UUID nodeId, GridNearLockResponse res) {
      * @return {@code True} if entry is locally mapped as a primary or back up node.
      */
     protected boolean isNearLocallyMapped(GridCacheEntryEx e, AffinityTopologyVersion topVer) {
-        return ctx.affinity().belongs(ctx.localNode(), e.partition(), topVer);
+        return ctx.affinity().partitionBelongs(ctx.localNode(), e.partition(), topVer);
     }
 
     /**
@@ -548,7 +548,7 @@ protected boolean evictNearEntry(GridCacheEntryEx e, GridCacheVersion obsoleteVe
                             topVer = cand.topologyVersion();
 
                             // Send request to remove from remote nodes.
-                            ClusterNode primary = ctx.affinity().primary(key, topVer);
+                            ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                             if (primary == null) {
                                 if (log.isDebugEnabled())
@@ -668,7 +668,7 @@ public void removeLocks(GridCacheVersion ver, Collection<KeyCacheObject> keys) {
                                     map = U.newHashMap(affNodes.size());
                                 }
 
-                                ClusterNode primary = ctx.affinity().primary(key, cand.topologyVersion());
+                                ClusterNode primary = ctx.affinity().primaryByKey(key, cand.topologyVersion());
 
                                 if (primary == null) {
                                     if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 46604c7524527..6d3f21f095865 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -358,7 +358,7 @@ else if (tx.implicit() && tx.isSystemInvalidate()) { // Finish implicit transact
                             GridCacheContext cacheCtx = e.context();
 
                             try {
-                                if (e.op() != NOOP && !cacheCtx.affinity().localNode(e.key(), topVer)) {
+                                if (e.op() != NOOP && !cacheCtx.affinity().keyLocalNode(e.key(), topVer)) {
                                     GridCacheEntryEx entry = cacheCtx.cache().peekEx(e.key());
 
                                     if (entry != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 7efb746d4428e..47f1bed98680c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -1580,11 +1580,11 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                     // Other types are filtered in indexing manager.
                     if (!cctx.isReplicated() && qry.type() == SCAN && qry.partition() == null &&
                         cctx.config().getCacheMode() != LOCAL && !incBackups &&
-                        !cctx.affinity().primary(cctx.localNode(), key, topVer)) {
+                        !cctx.affinity().primaryByKey(cctx.localNode(), key, topVer)) {
                         if (log.isDebugEnabled())
                             log.debug("Ignoring backup element [row=" + row +
                                 ", cacheMode=" + cctx.config().getCacheMode() + ", incBackups=" + incBackups +
-                                ", primary=" + cctx.affinity().primary(cctx.localNode(), key, topVer) + ']');
+                                ", primary=" + cctx.affinity().primaryByKey(cctx.localNode(), key, topVer) + ']');
 
                         continue;
                     }
@@ -1592,7 +1592,8 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                     V val = row.getValue();
 
                     if (log.isDebugEnabled()) {
-                        ClusterNode primaryNode = CU.primaryNode(cctx, key);
+                        ClusterNode primaryNode = cctx.affinity().primaryByKey(key,
+                            cctx.affinity().affinityTopologyVersion());
 
                         log.debug(S.toString("Record",
                             "key", key, true,
@@ -2355,7 +2356,7 @@ public Collection<GridCacheSqlMetadata> sqlMetadata() throws IgniteCheckedExcept
 
                 return new IgniteBiPredicate<K, V>() {
                     @Override public boolean apply(K k, V v) {
-                        return cache.context().affinity().primary(ctx.discovery().localNode(), k, NONE);
+                        return cache.context().affinity().primaryByKey(ctx.discovery().localNode(), k, NONE);
                     }
                 };
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 165b8b77e0ab4..926c7ce58b293 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -56,7 +56,6 @@
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheDeploymentManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index a5f647aa3d2b9..12b02f0fa9386 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -372,7 +372,7 @@ public void onEntryExpired(GridCacheEntryEx e, KeyCacheObject key, CacheObject o
         if (F.isEmpty(lsnrCol))
             return;
 
-        boolean primary = cctx.affinity().primary(cctx.localNode(), e.partition(), AffinityTopologyVersion.NONE);
+        boolean primary = cctx.affinity().primaryByPartition(cctx.localNode(), e.partition(), AffinityTopologyVersion.NONE);
 
         if (cctx.isReplicated() || primary) {
             boolean recordIgniteEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 18c301124405f..b07a1175c436d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -1288,7 +1288,7 @@ protected void batchStoreCommit(Iterable<IgniteTxEntry> writeEntries) throws Ign
                         if (!skip && skipNonPrimary) {
                             skip = e.cached().isNear() ||
                                 e.cached().detached() ||
-                                !e.context().affinity().primary(e.cached().partition(), topologyVersion()).isLocal();
+                                !e.context().affinity().primaryByPartition(e.cached().partition(), topologyVersion()).isLocal();
                         }
 
                         if (!skip && !local() && // Update local store at backups only if needed.
@@ -1707,7 +1707,7 @@ protected boolean isNearLocallyMapped(IgniteTxEntry e, boolean primaryOnly) {
 
         int part = cached != null ? cached.partition() : cacheCtx.affinity().partition(e.key());
 
-        List<ClusterNode> affNodes = cacheCtx.affinity().nodes(part, topologyVersion());
+        List<ClusterNode> affNodes = cacheCtx.affinity().nodesByPartition(part, topologyVersion());
 
         e.locallyMapped(F.contains(affNodes, cctx.localNode()));
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 1a9b082b3793f..7ac439807a61f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1112,7 +1112,7 @@ assert isWriteToStoreFromDhtValid(stores) :
      * @return {@code True} if local node is current primary for given entry.
      */
     private boolean primaryLocal(GridCacheEntryEx entry) {
-        return entry.context().affinity().primary(cctx.localNode(), entry.partition(), AffinityTopologyVersion.NONE);
+        return entry.context().affinity().primaryByPartition(cctx.localNode(), entry.partition(), AffinityTopologyVersion.NONE);
     }
 
     /**
@@ -1413,7 +1413,7 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                     finally {
                         if (entry != null && readCommitted()) {
                             if (cacheCtx.isNear()) {
-                                if (cacheCtx.affinity().belongs(cacheCtx.localNode(), entry.partition(), topVer)) {
+                                if (cacheCtx.affinity().partitionBelongs(cacheCtx.localNode(), entry.partition(), topVer)) {
                                     if (entry.markObsolete(xidVer))
                                         cacheCtx.cache().removeEntry(entry);
                                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
index 70d938e144ead..67d00ea782923 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
@@ -401,7 +401,7 @@ private void mapTxKeys(@Nullable Set<IgniteTxKey> txKeys, Map<IgniteTxKey, TxLoc
         private UUID primary(IgniteTxKey txKey) {
             GridCacheContext ctx = cctx.cacheContext(txKey.cacheId());
 
-            ClusterNode node = ctx.affinity().primary(txKey.key(), topVer);
+            ClusterNode node = ctx.affinity().primaryByKey(txKey.key(), topVer);
 
             assert node != null : topVer;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index 515314eb08728..a8ae2eaaee582 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -1945,7 +1945,7 @@ else if (ttl == CU.TTL_NOT_CHANGED)
                         expiryTime = CU.toExpireTime(ttl);
                     }
 
-                    boolean primary = cctx.affinity().primary(cctx.localNode(), entry.key(), topVer);
+                    boolean primary = cctx.affinity().primaryByKey(cctx.localNode(), entry.key(), topVer);
 
                     entry.initialValue(e.getValue(),
                         ver,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
index 70232afe717fa..d5f277ebf3f16 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
@@ -496,7 +496,7 @@ private Collection<ClusterNode> dataNodes(AffinityTopologyVersion topVer) throws
         Collection<ClusterNode> nodes;
 
         if (collocated) {
-            List<ClusterNode> nodes0 = ctx.affinity().nodes(hdrPart, topVer);
+            List<ClusterNode> nodes0 = ctx.affinity().nodesByPartition(hdrPart, topVer);
 
             nodes = !nodes0.isEmpty() ?
                 Collections.singleton(nodes0.contains(ctx.localNode()) ? ctx.localNode() : F.first(nodes0)) : nodes0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
index e8b2cc7f0f4b8..bc6c1827a0774 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
@@ -91,7 +91,7 @@ public IgniteUuid setId() {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public boolean apply(K k, V v) {
-        return !filter || ctx.affinity().primary(ctx.localNode(), k, ctx.affinity().affinityTopologyVersion());
+        return !filter || ctx.affinity().primaryByKey(ctx.localNode(), k, ctx.affinity().affinityTopologyVersion());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index 7889a5579c7bb..992e02a6fc570 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -1568,7 +1568,7 @@ public PartitionsReservation(int[] cacheIds, int partId,
                         }
                     }
                     finally {
-                        if (checkPartMapping && !cctx.affinity().primary(partId, topVer).id().equals(ctx.localNodeId()))
+                        if (checkPartMapping && !cctx.affinity().primaryByPartition(partId, topVer).id().equals(ctx.localNodeId()))
                             throw new IgniteException("Failed partition reservation. " +
                                 "Partition is not primary on the node. [partition=" + partId + ", cacheName=" + cctx.name() +
                                 ", nodeId=" + ctx.localNodeId() + ", topology=" + topVer + ']');
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
index 92e2b9b2ba21d..b0337d62967e0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
@@ -214,12 +214,12 @@ public CheckCallable(Object key, AffinityTopologyVersion topVer) {
 
                 ClusterNode loc = ignite.cluster().localNode();
 
-                if (loc.equals(aff.primary(key, topVer)))
+                if (loc.equals(aff.primaryByKey(key, topVer)))
                     return true;
 
                 AffinityTopologyVersion topVer0 = new AffinityTopologyVersion(topVer.topologyVersion() + 1, 0);
 
-                assertEquals(loc, aff.primary(key, topVer0));
+                assertEquals(loc, aff.primaryByKey(key, topVer0));
             }
 
             return null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
index 1cfb330da8cd6..53b4900410033 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
@@ -5837,7 +5837,7 @@ public CheckEntriesTask(Collection<String> keys) {
             int size = 0;
 
             for (String key : keys) {
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx())) {
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx())) {
                     GridCacheEntryEx e =
                         ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key);
 
@@ -5873,7 +5873,7 @@ private static class CheckCacheSizeTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : map.keySet())
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(ctx.name()).localSize(ALL));
@@ -6116,7 +6116,7 @@ public CheckKeySizeTask(Collection<String> keys) {
             int size = 0;
 
             for (String key : keys)
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(null).localSize(ALL));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
index 6b0e1932e1f2c..d4449f9ee9228 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
@@ -5548,7 +5548,7 @@ private static class CheckEntriesTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : keys) {
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx())) {
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx())) {
                     GridCacheEntryEx e =
                         ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key);
 
@@ -5589,7 +5589,7 @@ private static class CheckCacheSizeTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : map.keySet())
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(ctx.name()).localSize(ALL));
@@ -5850,7 +5850,7 @@ public CheckKeySizeTask(Collection<String> keys, String s) {
             int size = 0;
 
             for (String key : keys)
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(cacheName).localSize(ALL));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
index 6c577c63ab693..fac24ccd00045 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
@@ -1009,9 +1009,9 @@ private T2<Integer, Integer> swapKeysCount(int nodeIdx, int part) throws IgniteC
         //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primary(ctx.localNode(), part, topVer))
+        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             primaryCnt = cnt;
-        else if (affinity.backup(ctx.localNode(), part, topVer))
+        else if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             backupCnt = cnt;
         return new T2<>(primaryCnt, backupCnt);
     }
@@ -1081,9 +1081,9 @@ private T2<Integer, Integer> offheapKeysCount(int nodeIdx, int part) throws Igni
         //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primary(ctx.localNode(), part, topVer))
+        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             primaryCnt = cnt;
-        else if (affinity.backup(ctx.localNode(), part, topVer))
+        else if (affinity.backupByPartition(ctx.localNode(), part, topVer))
             backupCnt = cnt;
         return new T2<>(primaryCnt, backupCnt);
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
index 45b4b9f299aa3..aeca2fb2279ff 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
@@ -365,7 +365,7 @@ private int primaryQueueNode(IgniteQueue queue) {
 
         for (int i = 0; i < gridCount(); i++) {
             for (GridCacheEntryEx e : ((IgniteKernal)grid(i)).context().cache().internalCache(cctx.name()).allEntries()) {
-                if (aff.primary(grid(i).localNode(), e.key(), AffinityTopologyVersion.NONE)
+                if (aff.primaryByKey(grid(i).localNode(), e.key(), AffinityTopologyVersion.NONE)
                     && e.key().value(cctx.cacheObjectContext(), false) instanceof GridCacheQueueHeaderKey)
                     return i;
             }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
index b4ef11a82c0c1..8709d05664d94 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
@@ -853,13 +853,13 @@ public void testPessimisticTx2() throws Exception {
 
         GridCacheAffinityManager aff = ignite0.context().cache().internalCache(null).context().affinity();
 
-        List<ClusterNode> nodes1 = aff.nodes(key1, topVer1);
-        List<ClusterNode> nodes2 = aff.nodes(key1, topVer2);
+        List<ClusterNode> nodes1 = aff.nodesByKey(key1, topVer1);
+        List<ClusterNode> nodes2 = aff.nodesByKey(key1, topVer2);
 
         assertEquals(nodes1, nodes2);
 
-        nodes1 = aff.nodes(key2, topVer1);
-        nodes2 = aff.nodes(key2, topVer2);
+        nodes1 = aff.nodesByKey(key2, topVer1);
+        nodes2 = aff.nodesByKey(key2, topVer2);
 
         assertFalse(nodes1.get(0).equals(nodes2.get(0)));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
index aa240aae439a9..b909de90fd42a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
@@ -286,7 +286,7 @@ private void doTestDeadlock(
                     key = keys.get(1);
 
                     ClusterNode primaryNode =
-                        ((IgniteCacheProxy)cache).context().affinity().primary(key, NONE);
+                        ((IgniteCacheProxy)cache).context().affinity().primaryByKey(key, NONE);
 
                     List<Integer> primaryKeys =
                         primaryKeys(grid(primaryNode).cache(CACHE_NAME), 5, key + (100 * threadNum));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
index 83eb908d21978..ced8b61166876 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
@@ -293,7 +293,7 @@ private void doTestDeadlock(
                     key = keys.get(1);
 
                     ClusterNode primaryNode =
-                        ((IgniteCacheProxy)cache).context().affinity().primary(key, NONE);
+                        ((IgniteCacheProxy)cache).context().affinity().primaryByKey(key, NONE);
 
                     List<Integer> primaryKeys =
                         primaryKeys(grid(primaryNode).cache(CACHE_NAME), 5, key + (100 * threadNum));
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 8b2993f263cf7..155edeeeeb229 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -2061,7 +2061,7 @@ private void createSqlFunctions(String schema, Class<?>[] clss) throws IgniteChe
 
                 return new IgniteBiPredicate<K, V>() {
                     @Override public boolean apply(K k, V v) {
-                        return aff.primary(locNode, k, topVer0);
+                        return aff.primaryByKey(locNode, k, topVer0);
                     }
                 };
             }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 22b94c7ac796b..3700774260ded 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -612,7 +612,7 @@ private ClusterNode rangeNode(GridCacheContext<?,?> cctx, GridH2QueryContext qct
             node = cctx.discovery().node(nodeId);
         }
         else // Get primary node for current topology version.
-            node = cctx.affinity().primary(affKeyObj, qctx.topologyVersion());
+            node = cctx.affinity().primaryByKey(affKeyObj, qctx.topologyVersion());
 
         if (node == null) // Node was not found, probably topology changed and we need to retry the whole query.
             throw new GridH2RetryException("Failed to find node.");

From 4b20d03c148c83bf3665d3296ecbf8f768a43e0c Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Wed, 30 Aug 2017 18:12:17 +0300
Subject: [PATCH 392/446] gg-12723 Optimistic tx recovery fixed

---
 .../cache/distributed/dht/GridDhtTxLocal.java |   5 +-
 ...iteTxRecoveryAfterStoreCommitSelfTest.java | 363 ++++++++++++++++++
 .../IgniteCacheTxRecoverySelfTestSuite.java   |  10 +-
 3 files changed, 369 insertions(+), 9 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteTxRecoveryAfterStoreCommitSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
index b659abb8ef679..9220c9ebd7e91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
@@ -437,9 +437,12 @@ public IgniteInternalFuture<GridNearTxPrepareResponse> prepareAsync(
 
         if (state() != PREPARING) {
             if (!state(PREPARING)) {
-                if (state() == PREPARED && isSystemInvalidate())
+                if (state() == PREPARED && isSystemInvalidate()) {
                     fut.complete();
 
+                    return fut;
+                }
+
                 if (setRollbackOnly()) {
                     if (timeout == -1)
                         fut.onError(new IgniteTxTimeoutCheckedException("Transaction timed out and was rolled back: " +
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteTxRecoveryAfterStoreCommitSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteTxRecoveryAfterStoreCommitSelfTest.java
new file mode 100644
index 0000000000000..fa37f23ed887f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteTxRecoveryAfterStoreCommitSelfTest.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
+import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteFutureTimeoutException;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.apache.ignite.util.TestTcpCommunicationSpi;
+import org.jsr166.ConcurrentHashMap8;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+
+/**
+ * Tests that transaction is invalidated in case of {@link IgniteTxHeuristicCheckedException}.
+ */
+public class IgniteTxRecoveryAfterStoreCommitSelfTest extends GridCacheAbstractSelfTest {
+    /** Cache name. */
+    private static final String CACHE_NAME = "cache";
+
+    /** Store map. */
+    public static final Map<Object, Object> storeMap = new ConcurrentHashMap8<>();
+
+    /** */
+    private static volatile CountDownLatch storeCommitLatch;
+
+    /** */
+    private static volatile CountDownLatch nodeFailLatch;
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 5;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        startGridsMultiThreaded(gridCount());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @return Index of node starting transaction.
+     */
+    protected int originatingNode() {
+        return 0;
+    }
+
+    @Override protected long getTestTimeout() {
+        return 300_000;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        TcpCommunicationSpi comm = new TestTcpCommunicationSpi();
+
+        comm.setSharedMemoryPort(-1);
+
+        TestTcpDiscoverySpi discoSpi = new TestTcpDiscoverySpi();
+
+        discoSpi.setIpFinder(GridCacheAbstractSelfTest.ipFinder);
+
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName)
+                .setFailureDetectionTimeout(5_000)
+                .setDiscoverySpi(discoSpi)
+                .setCommunicationSpi(comm);
+
+        if (igniteInstanceName.endsWith("0"))
+            cfg.setUserAttributes(Collections.singletonMap("ORIGINATOR", true));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
+        CacheConfiguration cfg = super.cacheConfiguration(igniteInstanceName);
+
+        cfg.setName(CACHE_NAME);
+
+        cfg.setCacheStoreFactory(new TestStoreFactory());
+
+        cfg.setReadThrough(true);
+        cfg.setWriteThrough(true);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testManyKeysCommit() throws Exception {
+        Collection<Integer> keys = new ArrayList<>(200);
+
+        for (int i = 0; i < 20; i++)
+            keys.add(i);
+
+        testTxOriginatingNodeFails(keys);
+    }
+
+    /**
+     * @param keys Keys to update.
+     * @throws Exception If failed.
+     */
+    protected void testTxOriginatingNodeFails(final Collection<Integer> keys) throws Exception {
+        assertFalse(keys.isEmpty());
+
+        final Collection<IgniteKernal> grids = new ArrayList<>();
+
+        ClusterNode txNode = grid(originatingNode()).localNode();
+
+        for (int i = 1; i < gridCount(); i++)
+            grids.add((IgniteKernal)grid(i));
+
+        final Map<Integer, String> expectedStoreState = new HashMap<>();
+
+        final String initVal = "initialValue";
+
+        for (Integer key : keys) {
+            grid(originatingNode()).cache(CACHE_NAME).put(key, initVal);
+
+            expectedStoreState.put(key, String.valueOf(key));
+        }
+
+        Map<Integer, Collection<ClusterNode>> nodeMap = new HashMap<>();
+
+        info("Node being checked: " + grid(1).localNode().id());
+
+        for (Integer key : keys) {
+            Collection<ClusterNode> nodes = new ArrayList<>();
+
+            nodes.addAll(grid(1).affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key));
+
+            nodes.remove(txNode);
+
+            nodeMap.put(key, nodes);
+        }
+
+        info("Starting tx [values=" + expectedStoreState + ", topVer=" +
+                grid(1).context().discovery().topologyVersion() + ']');
+
+        final IgniteEx originatingNodeGrid = grid(originatingNode());
+
+        storeCommitLatch = new CountDownLatch(1);
+
+        nodeFailLatch = new CountDownLatch(1);
+
+        GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                IgniteCache<Integer, String> cache = originatingNodeGrid.cache(CACHE_NAME);
+
+                assertNotNull(cache);
+
+                Transaction tx = originatingNodeGrid.transactions().txStart(OPTIMISTIC, TransactionIsolation.SERIALIZABLE);
+
+                try {
+                    cache.putAll(expectedStoreState);
+
+                    info("Before commit");
+
+                    tx.commit();
+                }
+                catch (IgniteFutureTimeoutException ignored) {
+                    info("Failed to wait for commit future completion");
+                }
+
+                return null;
+            }
+        });
+
+        nodeFailLatch.await();
+
+        for (Integer key : expectedStoreState.keySet())
+            assertEquals(expectedStoreState.get(key), storeMap.get(key));
+
+        info(">>> Stopping originating node " + txNode);
+
+        ((TestTcpDiscoverySpi)grid(originatingNode()).context().config().getDiscoverySpi()).simulateNodeFailure();
+        ((TestTcpCommunicationSpi)grid(originatingNode()).context().config().getCommunicationSpi()).simulateNodeFailure();
+
+        storeCommitLatch.countDown();
+
+        G.stop(grid(originatingNode()).name(), true);
+
+        info(">>> Stopped originating node: " + txNode.id());
+
+        boolean txFinished = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                for (IgniteKernal g : grids) {
+                    GridCacheAdapter<?, ?> cache = g.internalCache(CACHE_NAME);
+
+                    IgniteTxManager txMgr = cache.isNear() ?
+                            ((GridNearCacheAdapter)cache).dht().context().tm() :
+                            cache.context().tm();
+
+                    int txNum = txMgr.idMapSize();
+
+                    if (txNum != 0)
+                        return false;
+                }
+
+                return true;
+            }
+        }, 300_000);
+
+        assertTrue(txFinished);
+
+        info("Transactions finished.");
+
+        for (Map.Entry<Integer, Collection<ClusterNode>> e : nodeMap.entrySet()) {
+            final Integer key = e.getKey();
+
+            final String val = expectedStoreState.get(key);
+
+            assertFalse(e.getValue().isEmpty());
+
+            for (ClusterNode node : e.getValue()) {
+                final UUID checkNodeId = node.id();
+
+                compute(G.ignite(checkNodeId).cluster().forNode(node)).call(new IgniteCallable<Void>() {
+                    /** */
+                    @IgniteInstanceResource
+                    private Ignite ignite;
+
+                    @Override public Void call() throws Exception {
+                        IgniteCache<Integer, String> cache = ignite.cache(CACHE_NAME);
+
+                        assertNotNull(cache);
+
+                        assertEquals("Failed to check entry value on node: " + checkNodeId,
+                                val, cache.get(key));
+
+                        return null;
+                    }
+                });
+            }
+        }
+
+        for (Map.Entry<Integer, String> e : expectedStoreState.entrySet()) {
+            for (Ignite g : G.allGrids())
+                assertEquals(e.getValue(), g.cache(CACHE_NAME).get(e.getKey()));
+        }
+    }
+
+    /**
+     *
+     */
+    public static class TestStoreFactory implements Factory<CacheStore> {
+        @IgniteInstanceResource
+        Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public CacheStore create() {
+            return new TestStore(ignite.cluster().localNode().attribute("ORIGINATOR") != null);
+        }
+    }
+
+    /**
+     *
+     */
+    public static class TestStore extends CacheStoreAdapter<Object, Object> {
+        /** */
+        private boolean originatorNodeFlag;
+
+        /** */
+        public TestStore(boolean originatorNodeFlag) {
+
+            this.originatorNodeFlag = originatorNodeFlag;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object load(Object key) {
+            return storeMap.get(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Object, ? extends Object> entry) {
+            storeMap.put(entry.getKey(), entry.getValue());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) {
+            storeMap.remove(key);
+        }
+
+        @Override public void sessionEnd(boolean commit) {
+            if (!originatorNodeFlag)
+                return;
+
+            if (storeCommitLatch != null) {
+                try {
+                    nodeFailLatch.countDown();
+
+                    storeCommitLatch.await();
+
+                    throw new IgniteException();
+                }
+                catch (InterruptedException e) {
+                    throw new IgniteException(e);
+                }
+            }
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
index c7c8db67fe31e..e901cf5489220 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
@@ -18,14 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedTxPessimisticOriginatingNodeFailureSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledTxOriginatingNodeFailureSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedTxOriginatingNodeFailureSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheCommitDelayTxRecoveryTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedNearDisabledPrimaryNodeFailureRecoveryTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedPrimaryNodeFailureRecoveryTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedTwoBackupsPrimaryNodeFailureRecoveryTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.TxRecoveryStoreEnabledTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.*;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearTxPessimisticOriginatingNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxOriginatingNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxPessimisticOriginatingNodeFailureSelfTest;
@@ -56,6 +49,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheReplicatedTxPessimisticOriginatingNodeFailureSelfTest.class);
 
         suite.addTestSuite(TxRecoveryStoreEnabledTest.class);
+        suite.addTestSuite(IgniteTxRecoveryAfterStoreCommitSelfTest.class);
 
         return suite;
     }

From dbc340a37664f8e5313ab085af34f59ce963032b Mon Sep 17 00:00:00 2001
From: Konstantin Boudnik <konstantin_boudnik@epam.com>
Date: Mon, 5 Jun 2017 19:47:02 -0700
Subject: [PATCH 393/446] IGNITE-5413. Ignite shouldn't expose nor send
 (clear-text) env variables to a 3rd endpoint

---
 .../ignite/internal/processors/cluster/ClusterProcessor.java | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
index 6500cf3e8faee..1d02749d009cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
@@ -58,7 +58,7 @@ public class ClusterProcessor extends GridProcessorAdapter {
     private IgniteClusterImpl cluster;
 
     /** */
-    private final AtomicBoolean notifyEnabled = new AtomicBoolean();
+    private final AtomicBoolean notifyEnabled = new AtomicBoolean(false);
 
     /** */
     @GridToStringExclude
@@ -74,9 +74,6 @@ public class ClusterProcessor extends GridProcessorAdapter {
     public ClusterProcessor(GridKernalContext ctx) {
         super(ctx);
 
-        notifyEnabled.set(IgniteSystemProperties.getBoolean(IGNITE_UPDATE_NOTIFIER,
-            Boolean.parseBoolean(IgniteProperties.get("ignite.update.notifier.enabled.by.default"))));
-
         cluster = new IgniteClusterImpl(ctx);
     }
 

From 825409fb17e414cdb376a96074b62262eead859a Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Thu, 13 Jul 2017 18:34:01 +0300
Subject: [PATCH 394/446] Functionality of GridVersionSelfTest is debated now

---
 .../java/org/apache/ignite/internal/GridVersionSelfTest.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridVersionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridVersionSelfTest.java
index 4751a0c4fb49b..13af907489f4f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridVersionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridVersionSelfTest.java
@@ -32,6 +32,8 @@ public class GridVersionSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testVersions() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-5413");
+
         String propVal = System.getProperty(IGNITE_UPDATE_NOTIFIER);
 
         System.setProperty(IGNITE_UPDATE_NOTIFIER, "true");

From 41d92442c16cce673523295dfe0a7ffac686003a Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Thu, 13 Jul 2017 17:32:06 +0300
Subject: [PATCH 395/446] Functionality of muted test is debated now

---
 .../internal/IgniteUpdateNotifierPerClusterSettingSelfTest.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteUpdateNotifierPerClusterSettingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteUpdateNotifierPerClusterSettingSelfTest.java
index a255f15345782..9fca528a691d5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteUpdateNotifierPerClusterSettingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteUpdateNotifierPerClusterSettingSelfTest.java
@@ -66,6 +66,8 @@ public class IgniteUpdateNotifierPerClusterSettingSelfTest extends GridCommonAbs
      * @throws Exception If failed.
      */
     public void testNotifierEnabledForCluster() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-5413");
+
         checkNotifierStatusForCluster(true);
     }
 

From d8fe81f827b4db1800276e5d76ead86e1648c224 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Wed, 16 Aug 2017 00:24:07 +0300
Subject: [PATCH 396/446] IgniteCacheNearRestartRollbackSelfTest#testRestarts
 is muted.

---
 .../distributed/IgniteCacheNearRestartRollbackSelfTest.java     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index 3f242b5726867..aea4d7782d858 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -132,6 +132,8 @@ protected CacheConfiguration<Object, Object> cacheConfiguration(String gridName)
      */
     @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
     public void testRestarts() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-12398");
+
         startGrids(4);
 
         Ignite tester = ignite(3);

From 882f4b40883b11a2cda5a86b1fa2f2af9f103d32 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Thu, 31 Aug 2017 16:50:33 +0300
Subject: [PATCH 397/446] Fixed flaky test IgniteCacheEntryListener*

---
 .../query/continuous/CacheContinuousQueryEntry.java      | 9 +--------
 .../query/continuous/CacheContinuousQueryHandler.java    | 2 --
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index 366a1e05fa46c..ffbbc2c1e05ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -199,13 +199,6 @@ void markFiltered() {
         depInfo = null;
     }
 
-    /**
-     * @param topVer Topology version.
-     */
-    void topologyVersion(AffinityTopologyVersion topVer) {
-        this.topVer = topVer;
-    }
-
     /**
      * @return Size include this event and filtered.
      */
@@ -222,7 +215,7 @@ CacheContinuousQueryEntry forBackupQueue() {
             return this;
 
         CacheContinuousQueryEntry e =
-            new CacheContinuousQueryEntry(cacheId, null, null, null, null, keepBinary, part, updateCntr, null);
+            new CacheContinuousQueryEntry(cacheId, null, null, null, null, keepBinary, part, updateCntr, topVer);
 
         e.flags = flags;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 926c7ce58b293..0cdbec5013b15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -465,8 +465,6 @@ public void keepBinary(boolean keepBinary) {
                         for (CacheContinuousQueryEntry e : backupQueue0) {
                             if (!e.isFiltered())
                                 prepareEntry(cctx, nodeId, e);
-
-                            e.topologyVersion(topVer);
                         }
 
                         ctx.continuous().addBackupNotification(nodeId, routineId, backupQueue0, topic);

From fcdf7a874147c1c9e4241a6663e320c82588d4bd Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 1 Sep 2017 14:52:32 +0300
Subject: [PATCH 398/446] ignite-6053: fixed clear() on local cache

---
 .../processors/cache/GridCacheAdapter.java    | 42 +++++++--
 .../local/GridCacheLocalFullApiSelfTest.java  | 92 ++++++++++++++++++-
 2 files changed, 127 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 9b1e0cc7721c1..2d2b4f2ce12b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1196,8 +1196,17 @@ public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, bool
      * @throws IgniteCheckedException In case of error.
      */
     private void clear(@Nullable Set<? extends K> keys) throws IgniteCheckedException {
-        executeClearTask(keys, false).get();
-        executeClearTask(keys, true).get();
+        if (isLocal()) {
+            if (keys == null)
+                clearLocally(true, false, false);
+            else
+                clearLocallyAll(keys, true, false, false);
+        }
+        else {
+            executeClearTask(keys, false).get();
+
+            executeClearTask(keys, true).get();
+        }
     }
 
     /**
@@ -1205,13 +1214,34 @@ private void clear(@Nullable Set<? extends K> keys) throws IgniteCheckedExceptio
      * @return Future.
      */
     private IgniteInternalFuture<?> clearAsync(@Nullable final Set<? extends K> keys) {
-        return executeClearTask(keys, false).chain(new CX1<IgniteInternalFuture<?>, Object>() {
-            @Override public Object applyx(IgniteInternalFuture<?> fut) throws IgniteCheckedException {
-                executeClearTask(keys, true).get();
+        if (isLocal())
+            return clearLocallyAsync(keys);
+        else {
+            return executeClearTask(keys, false).chain(new CX1<IgniteInternalFuture<?>, Object>() {
+                @Override public Object applyx(IgniteInternalFuture<?> fut) throws IgniteCheckedException {
+                    executeClearTask(keys, true).get();
+
+                    return null;
+                }
+            });
+        }
+    }
+
+    /**
+     * @param keys Keys to clear.
+     * @return Clear future.
+     */
+    private IgniteInternalFuture<?> clearLocallyAsync(@Nullable Set<? extends K> keys) {
+        return ctx.closures().callLocalSafe(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                if (keys == null)
+                    clearLocally(true, false, false);
+                else
+                    clearLocallyAll(keys, true, false, false);
 
                 return null;
             }
-        });
+        }, false);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalFullApiSelfTest.java
index f499c26d547c4..8a844bcaa6daa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalFullApiSelfTest.java
@@ -17,7 +17,9 @@
 
 package org.apache.ignite.internal.processors.cache.local;
 
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Map;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
@@ -89,4 +91,92 @@ public void testMapKeysToNodes() throws Exception {
         for (String key : keys)
             assert "key1".equals(key) || "key2".equals(key);
     }
-}
\ No newline at end of file
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLocalClearAsync() throws Exception {
+        localCacheClear(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLocalClear() throws Exception {
+        localCacheClear(false);
+    }
+
+    /**
+     * @param async If {@code true} uses async method.
+     * @throws Exception If failed.
+     */
+    private void localCacheClear(boolean async) throws Exception {
+        // In addition to the existing tests, it confirms the data is cleared only on one node,
+        // not on all nodes that have local caches with same names.
+        try {
+            startGrid(1);
+
+            IgniteCache<String, Integer> cache = jcache();
+
+            IgniteCache<String, Integer> asyncCache = cache.withAsync();
+
+            for (int i = 0; i < 5; i++) {
+                cache.put(String.valueOf(i), i);
+                jcache(1).put(String.valueOf(i), i);
+            }
+
+            if (async) {
+                asyncCache.clear("4");
+
+                asyncCache.future().get();
+            }
+            else
+                cache.clear("4");
+
+            assertNull(peek(cache, "4"));
+            assertNotNull(peek(jcache(1), "4"));
+
+            if (async) {
+                asyncCache.clearAll(new HashSet<>(Arrays.asList("2", "3")));
+
+                asyncCache.future().get();
+            }
+            else
+                cache.clearAll(new HashSet<>(Arrays.asList("2", "3")));
+
+            for (int i = 2; i < 4; i++) {
+                assertNull(peek(cache, String.valueOf(i)));
+                assertNotNull(peek(jcache(1), String.valueOf(i)));
+            }
+
+            if (async) {
+                asyncCache.clear();
+
+                asyncCache.future().get();
+            }
+            else
+                cache.clear();
+
+            for (int i = 0; i < 2; i++) {
+                assertNull(peek(cache, String.valueOf(i)));
+                assertNotNull(peek(jcache(1), String.valueOf(i)));
+            }
+
+            if (async) {
+                IgniteCache<String, Integer> asyncCache1 = jcache(1).withAsync();
+
+                asyncCache1.clear();
+
+                asyncCache1.future().get();
+            }
+            else
+                jcache(1).clear();
+
+            for (int i = 0; i < 2; i++)
+                assert jcache(i).localSize() == 0;
+        }
+        finally {
+            stopGrid(1);
+        }
+    }
+}

From eb6db758af64610745957ad5b0d165302d7972d3 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 1 Sep 2017 18:14:05 +0300
Subject: [PATCH 399/446] ignite-6053: fixed clear() on local cache ('keys'
 parameter must be final)

---
 .../ignite/internal/processors/cache/GridCacheAdapter.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 2d2b4f2ce12b4..d947aa9cb13b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1231,7 +1231,7 @@ private IgniteInternalFuture<?> clearAsync(@Nullable final Set<? extends K> keys
      * @param keys Keys to clear.
      * @return Clear future.
      */
-    private IgniteInternalFuture<?> clearLocallyAsync(@Nullable Set<? extends K> keys) {
+    private IgniteInternalFuture<?> clearLocallyAsync(@Nullable final Set<? extends K> keys) {
         return ctx.closures().callLocalSafe(new Callable<Object>() {
             @Override public Object call() throws Exception {
                 if (keys == null)

From 04143cfacf887b8bd1fd10f807b97aaa2af61ead Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 5 Sep 2017 19:11:12 +0300
Subject: [PATCH 400/446] GG-12699: Fix
 GridCacheAbstractFullApiSelfTest.testTransformResourceInjection.

Squashed commit of the following:

commit 3f20fe8dcc796406f7a7791e3ae9ddb5c26183ca
Author: Nikolay Izhikov <nizhikov.dev@gmail.com>
Date:   Wed Aug 9 13:37:11 2017 +0300

    IGNITE-5897 Fix session init/end logic. This fixes tests.

    Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>

    (cherry picked from commit 5a559df)

commit 52e89d387874a0653c58a608cc000950a76fb6b0
Author: dpavlov <dpavlov@gridgain.com>
Date:   Wed Jul 26 17:23:05 2017 +0300

    IGNITE-5806 - Fixed assertion with a side-effect - Fixes #2335.

    Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

    (cherry picked from commit 9e79c4b)

commit d3c40e418dce6ab640fe06e8c18ada4b93f1edf5
Author: Andrey V. Mashenkov <andrey.mashenkov@gmail.com>
Date:   Mon Sep 4 15:57:16 2017 +0300

    Fix javadoc.

(cherry picked from commit d4f2885)
---
 .../cache/store/GridCacheStoreManagerAdapter.java           | 3 ++-
 .../internal/processors/resource/GridResourceProcessor.java | 6 +++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 14ec92237c3d4..142e5ae28f67f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -879,7 +879,8 @@ private void sessionEnd0(@Nullable IgniteInternalTx tx, boolean threwEx) throws
                         lsnr.onSessionEnd(locSes, !threwEx);
                 }
 
-                store.sessionEnd(!threwEx);
+                if (!sesHolder.get().ended(store))
+                    store.sessionEnd(!threwEx);
             }
         }
         catch (Exception e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
index 84d07b64e50e5..02616b5e983a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
@@ -20,7 +20,6 @@
 import java.lang.annotation.Annotation;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.util.Collection;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.store.CacheStoreSession;
 import org.apache.ignite.compute.ComputeJob;
@@ -318,6 +317,11 @@ private GridResourceInjector injectorByAnnotation(GridResourceIoc.ResourceAnnota
 
     /**
      * @param obj Object to inject.
+     * @param ann Annotation enum.
+     * @param dep Grid deployment object.
+     * @param depCls Grid deployment class.
+     * @param param Resource to inject.
+     * @return {@code True} if resource was injected.
      * @throws IgniteCheckedException If failed to inject.
      */
     private boolean inject(Object obj, GridResourceIoc.ResourceAnnotation ann, @Nullable GridDeployment dep,

From 733ca74fcc094c83a3c4769b291c87d56e17c4a9 Mon Sep 17 00:00:00 2001
From: Vyacheslav Daradur <daradurvs@gmail.com>
Date: Fri, 10 Feb 2017 16:51:37 +0300
Subject: [PATCH 401/446] GG-12647: Backport IGNITE-3196 Add support for
 BigDecimals with negative scale in BinaryMarshaller (cherry picked from
 commit 5efefcb)

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../ignite/internal/binary/BinaryUtils.java   | 12 ++---
 .../internal/binary/BinaryWriterExImpl.java   | 14 +++---
 .../binary/BinaryMarshallerSelfTest.java      | 30 +++++++++++++
 modules/platforms/cpp/odbc/src/utility.cpp    | 17 ++++---
 .../Compute/ComputeApiTest.cs                 | 18 ++++----
 .../Impl/Binary/BinaryUtils.cs                | 44 +++++++++++--------
 6 files changed, 92 insertions(+), 43 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 1153d155c0400..6831ef9bb0d66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -65,8 +65,8 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 
 /**
  * Binary utils.
@@ -1193,13 +1193,15 @@ public static BigDecimal doReadDecimal(BinaryInputStream in) {
         int scale = in.readInt();
         byte[] mag = doReadByteArray(in);
 
-        BigInteger intVal = new BigInteger(mag);
+        boolean negative = mag[0] < 0;
 
-        if (scale < 0) {
-            scale &= 0x7FFFFFFF;
+        if (negative)
+            mag[0] &= 0x7F;
 
+        BigInteger intVal = new BigInteger(mag);
+
+        if (negative)
             intVal = intVal.negate();
-        }
 
         return new BigDecimal(intVal, scale);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index adaacdda40676..3289780e3fd0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -403,18 +403,20 @@ public void doWriteDecimal(@Nullable BigDecimal val) {
 
             out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL);
 
+            out.unsafeWriteInt(val.scale());
+
             BigInteger intVal = val.unscaledValue();
 
-            if (intVal.signum() == -1) {
-                intVal = intVal.negate();
+            boolean negative = intVal.signum() == -1;
 
-                out.unsafeWriteInt(val.scale() | 0x80000000);
-            }
-            else
-                out.unsafeWriteInt(val.scale());
+            if (negative)
+                intVal = intVal.negate();
 
             byte[] vals = intVal.toByteArray();
 
+            if (negative)
+                vals[0] |= -0x80;
+
             out.unsafeWriteInt(vals.length);
             out.writeByteArray(vals);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index cd8a487d6d876..31d6f31eef961 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -29,6 +29,7 @@
 import java.lang.reflect.Proxy;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.math.RoundingMode;
 import java.net.InetSocketAddress;
 import java.sql.Timestamp;
 import java.util.AbstractQueue;
@@ -178,6 +179,35 @@ public void testDecimal() throws Exception {
         assertEquals((val = new BigDecimal(new BigInteger("-79228162514264337593543950336"))), marshalUnmarshal(val));
     }
 
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -1)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -2)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -3)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -4)), marshalUnmarshal(val));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleRoundingModeDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.ZERO.setScale(-1, RoundingMode.HALF_UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE).setScale(-3, RoundingMode.HALF_DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE).setScale(-5, RoundingMode.HALF_EVEN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MAX_VALUE).setScale(-8, RoundingMode.UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MIN_VALUE).setScale(-10, RoundingMode.DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MAX_VALUE).setScale(-12, RoundingMode.CEILING)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MIN_VALUE).setScale(-15, RoundingMode.FLOOR)), marshalUnmarshal(val));
+    }
+
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/platforms/cpp/odbc/src/utility.cpp b/modules/platforms/cpp/odbc/src/utility.cpp
index 22191eb8b045e..2c8f9f3a3cdab 100644
--- a/modules/platforms/cpp/odbc/src/utility.cpp
+++ b/modules/platforms/cpp/odbc/src/utility.cpp
@@ -87,8 +87,14 @@ namespace ignite
 
             impl::binary::BinaryUtils::ReadInt8Array(reader.GetStream(), mag.data(), static_cast<int32_t>(mag.size()));
 
-            int32_t sign = (scale & 0x80000000) ? -1 : 1;
-            scale = scale & 0x7FFFFFFF;
+            int32_t sign = 1;
+            
+            if (mag[0] < 0)
+            {
+                mag[0] &= 0x7F;
+
+                sign = -1;
+            }
 
             common::Decimal res(mag.data(), static_cast<int32_t>(mag.size()), scale, sign);
 
@@ -101,14 +107,15 @@ namespace ignite
 
             const common::BigInteger &unscaled = decimal.GetUnscaledValue();
 
-            int32_t signFlag = unscaled.GetSign() == -1 ? 0x80000000 : 0;
-
-            writer.WriteInt32(decimal.GetScale() | signFlag);
+            writer.WriteInt32(decimal.GetScale());
 
             common::FixedSizeArray<int8_t> magnitude;
 
             unscaled.MagnitudeToBytes(magnitude);
 
+            if (unscaled.GetSign() == -1)
+                magnitude[0] |= -0x80;
+
             writer.WriteInt32(magnitude.GetSize());
 
             impl::binary::BinaryUtils::WriteInt8Array(writer.GetStream(), magnitude.GetData(), magnitude.GetSize());
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
index 71a4718130475..3a3100a6b3131 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
@@ -817,16 +817,16 @@ public void TestEchoDecimal()
             Assert.AreEqual(val = decimal.Parse("-11,12"), _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { val, val.ToString() }));
 
             // Test echo with overflow.
-            try
-            {
-                _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { null, decimal.MaxValue.ToString() + 1 });
+            var ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, decimal.MaxValue.ToString() + 1}));
 
-                Assert.Fail();
-            }
-            catch (IgniteException)
-            {
-                // No-op.
-            }
+            Assert.AreEqual("Decimal magnitude overflow (must be less than 96 bits): 104", ex.Message);
+
+            // Negative scale. 1E+1 parses to "1 scale -1" on Java side.
+            ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, "1E+1"}));
+
+            Assert.AreEqual("Decimal value scale overflow (must be between 0 and 28): -1", ex.Message);
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
index cc5d8a109ce86..5ce77d0ee4347 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
@@ -893,7 +893,9 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
             // Write scale and negative flag.
             int scale = (vals[3] & 0x00FF0000) >> 16; 
 
-            stream.WriteInt(((vals[3] & 0x80000000) == 0x80000000) ? (int)((uint)scale | 0x80000000) : scale);
+            stream.WriteInt(scale);
+
+            Boolean neg = vals[3] < 0;
 
             if (idx == -1)
             {
@@ -923,13 +925,15 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             if ((part24 & 0x80) == 0x80)
                             {
                                 stream.WriteInt(len + 1);
+                                
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
 
-                                stream.WriteByte(ByteZero);
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len);
 
-                            stream.WriteByte((byte)part24);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part24 | -0x80) : part24));
                             stream.WriteByte((byte)part16);
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
@@ -940,12 +944,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 1);
-
-                            stream.WriteByte((byte)part16);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part16 | -0x80) : part16));
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
                         }
@@ -955,12 +961,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 1);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 2);
-
-                            stream.WriteByte((byte)part8);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part8 | -0x80) : part8));
                             stream.WriteByte((byte)part0);
                         }
                         else
@@ -969,12 +977,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 2);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 3);
 
-                            stream.WriteByte((byte)part0);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part0 | -0x80) : part0));
                         }
                     }
                     else
@@ -997,18 +1007,16 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
         {
             int scale = stream.ReadInt();
 
-            bool neg;
+            bool neg = false;
+
+            byte[] mag = ReadByteArray(stream);
 
-            if (scale < 0)
+            if ((sbyte)mag[0] < 0)
             {
-                scale = scale & 0x7FFFFFFF;
+                mag[0] &= 0x7F;
 
                 neg = true;
             }
-            else
-                neg = false;
-
-            byte[] mag = ReadByteArray(stream);
 
             if (scale < 0 || scale > 28)
                 throw new BinaryObjectException("Decimal value scale overflow (must be between 0 and 28): " + scale);

From 9e3a0ad5d6144525ef5b058bcf3e6429611bec1b Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Tue, 5 Sep 2017 19:36:59 +0300
Subject: [PATCH 402/446] IGNITE-4557 "Fixed wrong affinity manager call"

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../processors/cache/GridCacheAdapter.java    | 10 ++--
 .../cache/GridCacheAffinityManager.java       | 60 +++++++------------
 .../processors/cache/GridCacheContext.java    | 17 ------
 .../cache/GridCacheEvictionManager.java       |  6 +-
 .../processors/cache/GridCacheUtils.java      | 20 -------
 .../cache/affinity/GridCacheAffinityImpl.java | 16 ++---
 .../CacheDataStructuresManager.java           |  2 +-
 .../distributed/dht/GridDhtCacheAdapter.java  |  2 +-
 .../distributed/dht/GridDhtCacheEntry.java    |  2 +-
 .../dht/GridDhtLocalPartition.java            |  4 +-
 .../dht/GridDhtPartitionTopologyImpl.java     |  4 +-
 .../distributed/dht/GridDhtTxRemote.java      |  2 +-
 .../dht/GridPartitionedGetFuture.java         |  2 +-
 .../dht/GridPartitionedSingleGetFuture.java   |  2 +-
 .../dht/atomic/GridDhtAtomicCache.java        | 13 ++--
 .../GridNearAtomicSingleUpdateFuture.java     |  2 +-
 .../atomic/GridNearAtomicUpdateFuture.java    |  4 +-
 .../dht/colocated/GridDhtColocatedCache.java  |  6 +-
 .../colocated/GridDhtColocatedLockFuture.java |  4 +-
 .../preloader/GridDhtPartitionDemander.java   |  8 +--
 .../preloader/GridDhtPartitionSupplier.java   | 12 ++--
 .../dht/preloader/GridDhtPreloader.java       |  4 +-
 .../distributed/near/GridNearAtomicCache.java |  2 +-
 .../distributed/near/GridNearCacheEntry.java  |  6 +-
 .../distributed/near/GridNearGetFuture.java   |  4 +-
 .../distributed/near/GridNearLockFuture.java  |  2 +-
 ...OptimisticSerializableTxPrepareFuture.java |  2 +-
 .../GridNearOptimisticTxPrepareFuture.java    |  2 +-
 .../GridNearPessimisticTxPrepareFuture.java   |  2 +-
 .../near/GridNearTransactionalCache.java      |  6 +-
 .../near/GridNearTxFinishFuture.java          |  2 +-
 .../cache/query/GridCacheQueryManager.java    |  9 +--
 .../CacheContinuousQueryHandler.java          |  1 -
 .../CacheContinuousQueryManager.java          |  2 +-
 .../cache/transactions/IgniteTxAdapter.java   |  4 +-
 .../transactions/IgniteTxLocalAdapter.java    |  4 +-
 .../transactions/TxDeadlockDetection.java     |  2 +-
 .../datastreamer/DataStreamerImpl.java        |  2 +-
 .../datastructures/GridCacheSetImpl.java      |  2 +-
 .../datastructures/GridSetQueryPredicate.java |  2 +-
 .../processors/job/GridJobProcessor.java      |  2 +-
 .../cache/CacheAffinityCallSelfTest.java      |  4 +-
 .../GridCacheAbstractFullApiSelfTest.java     |  6 +-
 ...gniteCacheConfigVariationsFullApiTest.java |  6 +-
 .../IgniteCachePeekModesAbstractTest.java     |  8 +--
 ...tQueueFailoverDataConsistencySelfTest.java |  2 +-
 ...teCacheClientNodeChangingTopologyTest.java |  8 +--
 .../TxOptimisticDeadlockDetectionTest.java    |  2 +-
 .../TxPessimisticDeadlockDetectionTest.java   |  2 +-
 .../processors/query/h2/IgniteH2Indexing.java |  2 +-
 .../query/h2/opt/GridH2IndexBase.java         |  2 +-
 51 files changed, 126 insertions(+), 176 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index f0a263e74540c..aab0ed46fba15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -778,7 +778,7 @@ public String toString() {
                 boolean nearKey;
 
                 if (!(modes.near && modes.primary && modes.backup)) {
-                    boolean keyPrimary = ctx.affinity().primary(ctx.localNode(), part, topVer);
+                    boolean keyPrimary = ctx.affinity().primaryByPartition(ctx.localNode(), part, topVer);
 
                     if (keyPrimary) {
                         if (!modes.primary)
@@ -787,7 +787,7 @@ public String toString() {
                         nearKey = false;
                     }
                     else {
-                        boolean keyBackup = ctx.affinity().belongs(ctx.localNode(), part, topVer);
+                        boolean keyBackup = ctx.affinity().partitionBelongs(ctx.localNode(), part, topVer);
 
                         if (keyBackup) {
                             if (!modes.backup)
@@ -808,7 +808,7 @@ public String toString() {
                     }
                 }
                 else {
-                    nearKey = !ctx.affinity().belongs(ctx.localNode(), part, topVer);
+                    nearKey = !ctx.affinity().partitionBelongs(ctx.localNode(), part, topVer);
 
                     if (nearKey) {
                         // Swap and offheap are disabled for near cache.
@@ -3813,8 +3813,8 @@ IgniteInternalFuture<?> globalLoadCacheAsync(@Nullable IgniteBiPredicate<K, V> p
             /** {@inheritDoc} */
             @Override public boolean apply(ClusterNode clusterNode) {
                 return clusterNode.version().compareTo(PartitionSizeLongTask.SINCE_VER) >= 0 &&
-                    ((modes.primary && aff.primary(clusterNode, part, topVer)) ||
-                        (modes.backup && aff.backup(clusterNode, part, topVer)));
+                    ((modes.primary && aff.primaryByPartition(clusterNode, part, topVer)) ||
+                        (modes.backup && aff.backupByPartition(clusterNode, part, topVer)));
             }
         }).nodes();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
index d9ff61684e7d0..8d335fe46b397 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
@@ -238,8 +238,8 @@ public Object affinityKey(Object key) {
      * @param topVer Topology version.
      * @return Affinity nodes.
      */
-    public List<ClusterNode> nodes(Object key, AffinityTopologyVersion topVer) {
-        return nodes(partition(key), topVer);
+    public List<ClusterNode> nodesByKey(Object key, AffinityTopologyVersion topVer) {
+        return nodesByPartition(partition(key), topVer);
     }
 
     /**
@@ -247,7 +247,7 @@ public List<ClusterNode> nodes(Object key, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Affinity nodes.
      */
-    public List<ClusterNode> nodes(int part, AffinityTopologyVersion topVer) {
+    public List<ClusterNode> nodesByPartition(int part, AffinityTopologyVersion topVer) {
         if (cctx.isLocal())
             topVer = LOC_CACHE_TOP_VER;
 
@@ -282,8 +282,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Primary node for given key.
      */
-    @Nullable public ClusterNode primary(Object key, AffinityTopologyVersion topVer) {
-        return primary(partition(key), topVer);
+    @Nullable public ClusterNode primaryByKey(Object key, AffinityTopologyVersion topVer) {
+        return primaryByPartition(partition(key), topVer);
     }
 
     /**
@@ -291,8 +291,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Primary node for given key.
      */
-    @Nullable public ClusterNode primary(int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    @Nullable public ClusterNode primaryByPartition(int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         if (nodes.isEmpty())
             return null;
@@ -306,8 +306,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code True} if checked node is primary for given key.
      */
-    public boolean primary(ClusterNode n, Object key, AffinityTopologyVersion topVer) {
-        return F.eq(primary(key, topVer), n);
+    public boolean primaryByKey(ClusterNode n, Object key, AffinityTopologyVersion topVer) {
+        return F.eq(primaryByKey(key, topVer), n);
     }
 
     /**
@@ -316,8 +316,8 @@ public boolean primary(ClusterNode n, Object key, AffinityTopologyVersion topVer
      * @param topVer Topology version.
      * @return {@code True} if checked node is primary for given partition.
      */
-    public boolean primary(ClusterNode n, int part, AffinityTopologyVersion topVer) {
-        return F.eq(primary(part, topVer), n);
+    public boolean primaryByPartition(ClusterNode n, int part, AffinityTopologyVersion topVer) {
+        return F.eq(primaryByPartition(part, topVer), n);
     }
 
     /**
@@ -325,8 +325,8 @@ public boolean primary(ClusterNode n, int part, AffinityTopologyVersion topVer)
      * @param topVer Topology version.
      * @return Backup nodes.
      */
-    public Collection<ClusterNode> backups(Object key, AffinityTopologyVersion topVer) {
-        return backups(partition(key), topVer);
+    public Collection<ClusterNode> backupsByKey(Object key, AffinityTopologyVersion topVer) {
+        return backupsByPartition(partition(key), topVer);
     }
 
     /**
@@ -334,8 +334,8 @@ public Collection<ClusterNode> backups(Object key, AffinityTopologyVersion topVe
      * @param topVer Topology version.
      * @return Backup nodes.
      */
-    public Collection<ClusterNode> backups(int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    private Collection<ClusterNode> backupsByPartition(int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         assert !F.isEmpty(nodes);
 
@@ -351,35 +351,21 @@ public Collection<ClusterNode> backups(int part, AffinityTopologyVersion topVer)
      * @param topVer Topology version.
      * @return {@code True} if checked node is a backup node for given partition.
      */
-    public boolean backup(ClusterNode n, int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    public boolean backupByPartition(ClusterNode n, int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         assert !F.isEmpty(nodes);
 
         return nodes.indexOf(n) > 0;
     }
 
-    /**
-     * @param keys keys.
-     * @param topVer Topology version.
-     * @return Nodes for the keys.
-     */
-    public Collection<ClusterNode> remoteNodes(Iterable keys, AffinityTopologyVersion topVer) {
-        Collection<Collection<ClusterNode>> colcol = new GridLeanSet<>();
-
-        for (Object key : keys)
-            colcol.add(nodes(key, topVer));
-
-        return F.view(F.flatCollections(colcol), F.remoteNodes(cctx.localNodeId()));
-    }
-
     /**
      * @param key Key to check.
      * @param topVer Topology version.
      * @return {@code true} if given key belongs to local node.
      */
-    public boolean localNode(Object key, AffinityTopologyVersion topVer) {
-        return localNode(partition(key), topVer);
+    public boolean keyLocalNode(Object key, AffinityTopologyVersion topVer) {
+        return partitionLocalNode(partition(key), topVer);
     }
 
     /**
@@ -387,10 +373,10 @@ public boolean localNode(Object key, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code true} if given partition belongs to local node.
      */
-    public boolean localNode(int part, AffinityTopologyVersion topVer) {
+    public boolean partitionLocalNode(int part, AffinityTopologyVersion topVer) {
         assert part >= 0 : "Invalid partition: " + part;
 
-        return nodes(part, topVer).contains(cctx.localNode());
+        return nodesByPartition(part, topVer).contains(cctx.localNode());
     }
 
     /**
@@ -399,11 +385,11 @@ public boolean localNode(int part, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code true} if given partition belongs to specified node.
      */
-    public boolean belongs(ClusterNode node, int part, AffinityTopologyVersion topVer) {
+    public boolean partitionBelongs(ClusterNode node, int part, AffinityTopologyVersion topVer) {
         assert node != null;
         assert part >= 0 : "Invalid partition: " + part;
 
-        return nodes(part, topVer).contains(node);
+        return nodesByPartition(part, topVer).contains(node);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 6322f9f03550a..3b44b5096f0b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -1589,23 +1589,6 @@ private void map(GridDhtCacheEntry entry, Iterable<ClusterNode> nodes,
         }
     }
 
-    /**
-     * Checks if at least one of the given keys belongs to one of the given partitions.
-     *
-     * @param keys Collection of keys to check.
-     * @param movingParts Collection of partitions to check against.
-     * @return {@code True} if there exist a key in collection {@code keys} that belongs
-     *      to one of partitions in {@code movingParts}
-     */
-    public boolean hasKey(Iterable<? extends K> keys, Collection<Integer> movingParts) {
-        for (K key : keys) {
-            if (movingParts.contains(affinity().partition(key)))
-                return true;
-        }
-
-        return false;
-    }
-
     /**
      * Check whether conflict resolution is required.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
index 134e743d4fd6e..f8722d6a4f338 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
@@ -808,7 +808,7 @@ public void touch(GridCacheEntryEx e, AffinityTopologyVersion topVer) {
             return;
 
         // Don't track non-primary entries if evicts are synchronized.
-        if (!cctx.isNear() && evictSync && !cctx.affinity().primary(cctx.localNode(), e.partition(), topVer))
+        if (!cctx.isNear() && evictSync && !cctx.affinity().primaryByPartition(cctx.localNode(), e.partition(), topVer))
             return;
 
         if (!busyLock.enterBusy())
@@ -910,7 +910,7 @@ public boolean evict(@Nullable GridCacheEntryEx entry, @Nullable GridCacheVersio
         if (evictSyncAgr) {
             assert !cctx.isNear(); // Make sure cache is not NEAR.
 
-            if (cctx.affinity().backups(
+            if (cctx.affinity().backupsByKey(
                     entry.key(),
                     cctx.topology().topologyVersion()).contains(cctx.localNode()) &&
                 evictSync)
@@ -1498,7 +1498,7 @@ void addEvent(DiscoveryEvent evt) {
                         if (!evts.isEmpty())
                             break;
 
-                        if (!cctx.affinity().primary(loc, it.next(), topVer))
+                        if (!cctx.affinity().primaryByPartition(loc, it.next(), topVer))
                             it.remove();
                     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 33d0085f3ff7c..af8f016a146f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -955,26 +955,6 @@ public static <K, V> void unwindEvicts(GridCacheSharedContext<K, V> ctx) {
             unwindEvicts(cacheCtx);
     }
 
-    /**
-     * Gets primary node on which given key is cached.
-     *
-     * @param ctx Cache.
-     * @param key Key to find primary node for.
-     * @return Primary node for the key.
-     */
-    @SuppressWarnings( {"unchecked"})
-    @Nullable public static ClusterNode primaryNode(GridCacheContext ctx, Object key) {
-        assert ctx != null;
-        assert key != null;
-
-        CacheConfiguration cfg = ctx.cache().configuration();
-
-        if (cfg.getCacheMode() != PARTITIONED)
-            return ctx.localNode();
-
-        return ctx.affinity().primary(key, ctx.affinity().affinityTopologyVersion());
-    }
-
     /**
      * @param asc {@code True} for ascending.
      * @return Descending order comparator.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
index 9e85bad610432..11361a27d384a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
@@ -82,21 +82,21 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public boolean isPrimary(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().primary(n, key, topologyVersion());
+        return cctx.affinity().primaryByKey(n, key, topologyVersion());
     }
 
     /** {@inheritDoc} */
     @Override public boolean isBackup(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().backups(key, topologyVersion()).contains(n);
+        return cctx.affinity().backupsByKey(key, topologyVersion()).contains(n);
     }
 
     /** {@inheritDoc} */
     @Override public boolean isPrimaryOrBackup(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().belongs(n, cctx.affinity().partition(key), topologyVersion());
+        return cctx.affinity().partitionBelongs(n, cctx.affinity().partition(key), topologyVersion());
     }
 
     /** {@inheritDoc} */
@@ -126,7 +126,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
         AffinityTopologyVersion topVer = topologyVersion();
 
         for (int partsCnt = partitions(), part = 0; part < partsCnt; part++) {
-            for (ClusterNode affNode : cctx.affinity().nodes(part, topVer)) {
+            for (ClusterNode affNode : cctx.affinity().nodesByPartition(part, topVer)) {
                 if (n.id().equals(affNode.id())) {
                     parts.add(part);
 
@@ -142,7 +142,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public ClusterNode mapPartitionToNode(int part) {
         A.ensure(part >= 0 && part < partitions(), "part >= 0 && part < total partitions");
 
-        return F.first(cctx.affinity().nodes(part, topologyVersion()));
+        return F.first(cctx.affinity().nodesByPartition(part, topologyVersion()));
     }
 
     /** {@inheritDoc} */
@@ -204,7 +204,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
         Map<ClusterNode, Collection<K>> res = new HashMap<>(nodesCnt, 1.0f);
 
         for (K key : keys) {
-            ClusterNode primary = cctx.affinity().primary(key, topVer);
+            ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
             if (primary == null)
                 throw new IgniteException("Failed to get primary node [topVer=" + topVer + ", key=" + key + ']');
@@ -227,14 +227,14 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public Collection<ClusterNode> mapKeyToPrimaryAndBackups(K key) {
         A.notNull(key, "key");
 
-        return cctx.affinity().nodes(partition(key), topologyVersion());
+        return cctx.affinity().nodesByPartition(partition(key), topologyVersion());
     }
 
     /** {@inheritDoc} */
     @Override public Collection<ClusterNode> mapPartitionToPrimaryAndBackups(int part) {
         A.ensure(part >= 0 && part < partitions(), "part >= 0 && part < total partitions");
 
-        return cctx.affinity().nodes(part, topologyVersion());
+        return cctx.affinity().nodesByPartition(part, topologyVersion());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index 366a4a920b9ee..2b3080981ec36 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -455,7 +455,7 @@ private void removeSetData(IgniteUuid setId, AffinityTopologyVersion topVer) thr
         Collection<SetItemKey> keys = new ArrayList<>(BATCH_SIZE);
 
         for (SetItemKey key : set) {
-            if (!loc && !aff.primary(cctx.localNode(), key, topVer))
+            if (!loc && !aff.primaryByKey(cctx.localNode(), key, topVer))
                 continue;
 
             keys.add(key);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 6f95e0d6a1e7e..417eb3536536f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -934,7 +934,7 @@ public void sendTtlUpdateRequest(@Nullable final IgniteCacheExpiryPolicy expiryP
                     AffinityTopologyVersion topVer = ctx.shared().exchange().readyAffinityVersion();
 
                     for (Map.Entry<KeyCacheObject, GridCacheVersion> e : entries.entrySet()) {
-                        List<ClusterNode> nodes = ctx.affinity().nodes(e.getKey(), topVer);
+                        List<ClusterNode> nodes = ctx.affinity().nodesByKey(e.getKey(), topVer);
 
                         for (int i = 0; i < nodes.size(); i++) {
                             ClusterNode node = nodes.get(i);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
index cf4085ba02335..39571ff4720e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
@@ -402,7 +402,7 @@ public Collection<UUID> readers() throws GridCacheEntryRemovedException {
         }
 
         // If remote node is (primary?) or back up, don't add it as a reader.
-        if (cctx.affinity().belongs(node, partition(), topVer)) {
+        if (cctx.affinity().partitionBelongs(node, partition(), topVer)) {
             if (log.isDebugEnabled())
                 log.debug("Ignoring near reader because remote node is affinity node [locNodeId=" + cctx.localNodeId()
                     + ", rmtNodeId=" + nodeId + ", key=" + key + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index a0ccc28c33b0c..44d3f92160770 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -616,7 +616,7 @@ void onUnlock() {
      * @return {@code True} if local node is primary for this partition.
      */
     public boolean primary(AffinityTopologyVersion topVer) {
-        return cctx.affinity().primary(cctx.localNode(), id, topVer);
+        return cctx.affinity().primaryByPartition(cctx.localNode(), id, topVer);
     }
 
     /**
@@ -624,7 +624,7 @@ public boolean primary(AffinityTopologyVersion topVer) {
      * @return {@code True} if local node is backup for this partition.
      */
     public boolean backup(AffinityTopologyVersion topVer) {
-        return cctx.affinity().backup(cctx.localNode(), id, topVer);
+        return cctx.affinity().backupByPartition(cctx.localNode(), id, topVer);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 628db37ec3ef9..0cea80de02aac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -576,7 +576,7 @@ else if (!node2part.nodeId().equals(loc.id())) {
             for (int p = 0; p < num; p++) {
                 GridDhtLocalPartition locPart = localPartition(p, topVer, false, false);
 
-                if (cctx.affinity().localNode(p, topVer)) {
+                if (cctx.affinity().partitionLocalNode(p, topVer)) {
                     // This partition will be created during next topology event,
                     // which obviously has not happened at this point.
                     if (locPart == null) {
@@ -699,7 +699,7 @@ private GridDhtLocalPartition localPartition(int p,
         try {
             loc = locParts.get(p);
 
-            boolean belongs = cctx.affinity().localNode(p, topVer);
+            boolean belongs = cctx.affinity().partitionLocalNode(p, topVer);
 
             if (loc != null && loc.state() == EVICTED) {
                 locParts.set(p, loc = null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
index 8942ef9d9178c..399736e8f4278 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
@@ -286,7 +286,7 @@ IgniteUuid remoteFutureId() {
             return true;
 
         // Check if we are on the backup node.
-        return !cacheCtx.affinity().backups(key, topVer).contains(cctx.localNode());
+        return !cacheCtx.affinity().backupsByKey(key, topVer).contains(cctx.localNode());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index c41711c245028..519239aecb532 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -380,7 +380,7 @@ private boolean map(
     ) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index 2b5624b489b6f..a3f6b72376e8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -325,7 +325,7 @@ private void map(AffinityTopologyVersion topVer) {
     @Nullable private ClusterNode mapKeyToNode(AffinityTopologyVersion topVer) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 8a8221b428f13..ffe68ed2cfee7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -2490,7 +2490,7 @@ private UpdateSingleResult updateSingle(
 
                 assert !(newConflictVer instanceof GridCacheVersionEx) : newConflictVer;
 
-                boolean primary = !req.fastMap() || ctx.affinity().primary(ctx.localNode(), entry.partition(),
+                boolean primary = !req.fastMap() || ctx.affinity().primaryByPartition(ctx.localNode(), entry.partition(),
                     req.topologyVersion());
 
                 Object writeVal = op == TRANSFORM ? req.entryProcessor(i) : req.writeValue(i);
@@ -2580,7 +2580,7 @@ else if (conflictCtx.isMerge())
 
                 if (hasNear) {
                     if (primary && updRes.sendToDht()) {
-                        if (!ctx.affinity().belongs(node, entry.partition(), topVer)) {
+                        if (!ctx.affinity().partitionBelongs(node, entry.partition(), topVer)) {
                             // If put the same value as in request then do not need to send it back.
                             if (op == TRANSFORM || writeVal != updRes.newValue()) {
                                 res.addNearValue(i,
@@ -2711,7 +2711,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
                 Map<KeyCacheObject, CacheObject> storeMap = req.fastMap() ?
                     F.view(putMap, new P1<CacheObject>() {
                         @Override public boolean apply(CacheObject key) {
-                            return ctx.affinity().primary(ctx.localNode(), key, req.topologyVersion());
+                            return ctx.affinity().primaryByKey(ctx.localNode(), key, req.topologyVersion());
                         }
                     }) :
                     putMap;
@@ -2734,7 +2734,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
                 Collection<KeyCacheObject> storeKeys = req.fastMap() ?
                     F.view(rmvKeys, new P1<Object>() {
                         @Override public boolean apply(Object key) {
-                            return ctx.affinity().primary(ctx.localNode(), key, req.topologyVersion());
+                            return ctx.affinity().primaryByKey(ctx.localNode(), key, req.topologyVersion());
                         }
                     }) :
                     rmvKeys;
@@ -2773,7 +2773,8 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
 
                     assert writeVal != null || op == DELETE : "null write value found.";
 
-                    boolean primary = !req.fastMap() || ctx.affinity().primary(ctx.localNode(), entry.key(),
+                    boolean primary = !req.fastMap() || ctx.affinity().primaryByPartition(ctx.localNode(),
+                        entry.partition(),
                         req.topologyVersion());
 
                     Collection<UUID> readers = null;
@@ -2869,7 +2870,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
 
                     if (hasNear) {
                         if (primary) {
-                            if (!ctx.affinity().belongs(node, entry.partition(), topVer)) {
+                            if (!ctx.affinity().partitionBelongs(node, entry.partition(), topVer)) {
                                 int idx = firstEntryIdx + i;
 
                                 if (req.operation() == TRANSFORM) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
index 7376affca4455..891a20c2884a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
@@ -543,7 +543,7 @@ private GridNearAtomicAbstractUpdateRequest mapSingleUpdate(AffinityTopologyVers
         else
             val = EntryProcessorResourceInjectorProxy.wrap(cctx.kernalContext(), (EntryProcessor)val);
 
-        ClusterNode primary = cctx.affinity().primary(cacheKey, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(cacheKey, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to map keys for cache (all partition nodes " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index a252d9ae30c2b..9bdd1becb7600 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -937,7 +937,7 @@ else if (conflictRmvVals != null) {
         else
             val = EntryProcessorResourceInjectorProxy.wrap(cctx.kernalContext(), (EntryProcessor)val);
 
-        ClusterNode primary = cctx.affinity().primary(cacheKey.partition(), topVer);
+        ClusterNode primary = cctx.affinity().primaryByPartition(cacheKey.partition(), topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to map keys for cache (all partition nodes " +
@@ -988,7 +988,7 @@ private List<ClusterNode> mapKey(KeyCacheObject key, AffinityTopologyVersion top
 
         // If we can send updates in parallel - do it.
         return fastMap ? cctx.topology().nodes(affMgr.partition(key), topVer) :
-            Collections.singletonList(affMgr.primary(key, topVer));
+            Collections.singletonList(affMgr.primaryByKey(key, topVer));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index c9fc983d4850c..e1e0ec2bf087e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -174,7 +174,7 @@ public GridDistributedCacheEntry entryExx(
         AffinityTopologyVersion topVer,
         boolean allowDetached
     ) {
-        return allowDetached && !ctx.affinity().primary(ctx.localNode(), key, topVer) ?
+        return allowDetached && !ctx.affinity().primaryByKey(ctx.localNode(), key, topVer) ?
             createEntry(key) : entryExx(key, topVer);
     }
 
@@ -670,7 +670,7 @@ else if (!skipVals && ctx.config().isStatisticsEnabled())
                     assert topVer.compareTo(AffinityTopologyVersion.ZERO) > 0;
 
                     // Send request to remove from remote nodes.
-                    ClusterNode primary = ctx.affinity().primary(key, topVer);
+                    ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                     if (primary == null) {
                         if (log.isDebugEnabled())
@@ -790,7 +790,7 @@ public void removeLocks(long threadId, GridCacheVersion ver, Collection<KeyCache
                         map = U.newHashMap(affNodes.size());
                     }
 
-                    ClusterNode primary = ctx.affinity().primary(key, topVer);
+                    ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                     if (primary == null) {
                         if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index 69b66f9bfdcd3..79ca1087a3b88 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -1203,7 +1203,7 @@ private boolean mapAsPrimary(Collection<KeyCacheObject> keys, AffinityTopologyVe
         boolean explicit = false;
 
         for (KeyCacheObject key : keys) {
-            if (!cctx.affinity().primary(cctx.localNode(), key, topVer)) {
+            if (!cctx.affinity().primaryByKey(cctx.localNode(), key, topVer)) {
                 // Remove explicit locks added so far.
                 for (KeyCacheObject k : keys)
                     cctx.mvcc().removeExplicitLock(threadId, cctx.txKey(k), lockVer);
@@ -1287,7 +1287,7 @@ private GridNearLockMapping map(
     ) throws IgniteCheckedException {
         assert mapping == null || mapping.node() != null;
 
-        ClusterNode primary = cctx.affinity().primary(key, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to lock keys " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 260533d948e0e..c2b0c1390def7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -625,7 +625,7 @@ public void handleSupplyMessage(
             for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
                 int p = e.getKey();
 
-                if (cctx.affinity().localNode(p, topVer)) {
+                if (cctx.affinity().partitionLocalNode(p, topVer)) {
                     GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                     assert part != null;
@@ -695,7 +695,7 @@ public void handleSupplyMessage(
 
             // Only request partitions based on latest topology version.
             for (Integer miss : supply.missed()) {
-                if (cctx.affinity().localNode(miss, topVer))
+                if (cctx.affinity().partitionLocalNode(miss, topVer))
                     fut.partitionMissed(id, miss);
             }
 
@@ -1386,7 +1386,7 @@ private void demandFromNode(
                         for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
                             int p = e.getKey();
 
-                            if (cctx.affinity().localNode(p, topVer)) {
+                            if (cctx.affinity().partitionLocalNode(p, topVer)) {
                                 GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                                 assert part != null;
@@ -1463,7 +1463,7 @@ private void demandFromNode(
 
                         // Only request partitions based on latest topology version.
                         for (Integer miss : s.supply().missed()) {
-                            if (cctx.affinity().localNode(miss, topVer))
+                            if (cctx.affinity().partitionLocalNode(miss, topVer))
                                 fut.partitionMissed(node.id(), miss);
                         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index b082c4736cb9c..994242354c05d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -302,7 +302,7 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
                             (Iterator<GridCacheMapEntry>)sctx.entryIt : loc.allEntries().iterator();
 
                         while (entIt.hasNext()) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition, so we send '-1' partition and move on.
                                 s.missed(part);
 
@@ -387,7 +387,7 @@ else if (log.isDebugEnabled())
                             boolean prepared = false;
 
                             while (iter.hasNext()) {
-                                if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                                if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                     // Demander no longer needs this partition,
                                     // so we send '-1' partition and move on.
                                     s.missed(part);
@@ -510,7 +510,7 @@ else if (log.isDebugEnabled())
                             (Iterator<GridCacheEntryInfo>)sctx.entryIt : entries.iterator();
 
                         while (lsnrIt.hasNext()) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition,
                                 // so we send '-1' partition and move on.
                                 s.missed(part);
@@ -808,7 +808,7 @@ private void processOldDemandMessage(GridDhtPartitionDemandMessage d, UUID id) {
                     boolean partMissing = false;
 
                     for (GridCacheEntryEx e : loc.allEntries()) {
-                        if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                        if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                             // Demander no longer needs this partition, so we send '-1' partition and move on.
                             s.missed(part);
 
@@ -859,7 +859,7 @@ else if (log.isDebugEnabled())
                                 boolean prepared = false;
 
                                 for (Map.Entry<byte[], GridCacheSwapEntry> e : iter) {
-                                    if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                                    if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                         // Demander no longer needs this partition,
                                         // so we send '-1' partition and move on.
                                         s.missed(part);
@@ -947,7 +947,7 @@ else if (log.isDebugEnabled())
                         swapLsnr = null;
 
                         for (GridCacheEntryInfo info : entries) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition,
                                 // so we send '-1' partition and move on.
                                 s.missed(part);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 2efaed859b714..2d2e7a8154a69 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -289,7 +289,7 @@ private IgniteCheckedException stopError() {
             }
 
             // If partition belongs to local node.
-            if (cctx.affinity().localNode(p, topVer)) {
+            if (cctx.affinity().partitionLocalNode(p, topVer)) {
                 GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                 assert part != null;
@@ -349,7 +349,7 @@ private IgniteCheckedException stopError() {
      * @return Picked owners.
      */
     private Collection<ClusterNode> pickedOwners(int p, AffinityTopologyVersion topVer) {
-        Collection<ClusterNode> affNodes = cctx.affinity().nodes(p, topVer);
+        Collection<ClusterNode> affNodes = cctx.affinity().nodesByPartition(p, topVer);
 
         int affCnt = affNodes.size();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index b843e4e6d3db1..41632ef338b58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -161,7 +161,7 @@ public void processNearAtomicUpdateResponse(
             if (F.contains(failed, key))
                 continue;
 
-            if (ctx.affinity().belongs(ctx.localNode(), ctx.affinity().partition(key), req.topologyVersion())) { // Reader became backup.
+            if (ctx.affinity().partitionBelongs(ctx.localNode(), ctx.affinity().partition(key), req.topologyVersion())) { // Reader became backup.
                 GridCacheEntryEx entry = peekEx(key);
 
                 if (entry != null && entry.markObsolete(ver))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index 30fc213fc63e2..d022805a76bf2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -112,7 +112,7 @@ public GridNearCacheEntry(
                 return false;
             }
 
-            if (cctx.affinity().backup(cctx.localNode(), part, topVer)) {
+            if (cctx.affinity().backupByPartition(cctx.localNode(), part, topVer)) {
                 this.topVer = AffinityTopologyVersion.NONE;
 
                 return false;
@@ -162,7 +162,7 @@ public void initializeFromDht(AffinityTopologyVersion topVer) throws GridCacheEn
                                 }
                             }
 
-                            ClusterNode primaryNode = cctx.affinity().primary(key, topVer);
+                            ClusterNode primaryNode = cctx.affinity().primaryByKey(key, topVer);
 
                             if (primaryNode == null)
                                 this.topVer = AffinityTopologyVersion.NONE;
@@ -686,7 +686,7 @@ private void primaryNode(UUID nodeId, AffinityTopologyVersion topVer) {
         ClusterNode primary = null;
 
         try {
-            primary = cctx.affinity().primary(part, topVer);
+            primary = cctx.affinity().primaryByPartition(part, topVer);
         }
         catch (IllegalStateException ignore) {
             // Do not have affinity history.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index cb47498be0167..fb2843c02eec9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -413,7 +413,7 @@ private Map<KeyCacheObject, GridNearCacheEntry> map(
     ) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
@@ -726,7 +726,7 @@ private Map<K, V> loadEntries(
                     info.unmarshalValue(cctx, cctx.deploy().globalLoader());
 
                     // Entries available locally in DHT should not be loaded into near cache for reading.
-                    if (!cctx.affinity().localNode(info.key(), cctx.affinity().affinityTopologyVersion())) {
+                    if (!cctx.affinity().keyLocalNode(info.key(), cctx.affinity().affinityTopologyVersion())) {
                         GridNearCacheEntry entry = savedEntries.get(info.key());
 
                         if (entry == null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index d7a0fb5f8213d..d3e3a15831bb2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -1373,7 +1373,7 @@ private GridNearLockMapping map(
     ) throws IgniteCheckedException {
         assert mapping == null || mapping.node() != null;
 
-        ClusterNode primary = cctx.affinity().primary(key, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to lock keys " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index c464b36217fd8..a8448dcfd9680 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -529,7 +529,7 @@ private void map(
         GridCacheContext cacheCtx = entry.context();
 
         List<ClusterNode> nodes = cacheCtx.isLocal() ?
-            cacheCtx.affinity().nodes(entry.key(), topVer) :
+            cacheCtx.affinity().nodesByKey(entry.key(), topVer) :
             cacheCtx.topology().nodes(cacheCtx.affinity().partition(entry.key()), topVer);
 
         txMapping.addMapping(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index b314b81bf7b50..606d70ffb17fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -605,7 +605,7 @@ private GridDistributedTxMapping map(
             nodes = cacheCtx.topology().nodes(cached0.partition(), topVer);
         else
             nodes = cacheCtx.isLocal() ?
-                cacheCtx.affinity().nodes(entry.key(), topVer) :
+                cacheCtx.affinity().nodesByKey(entry.key(), topVer) :
                 cacheCtx.topology().nodes(cacheCtx.affinity().partition(entry.key()), topVer);
 
         txMapping.addMapping(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index f9a2f90787b3c..a4132f22b2b6b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -196,7 +196,7 @@ private void preparePessimistic() {
             GridCacheContext cacheCtx = txEntry.context();
 
             List<ClusterNode> nodes = cacheCtx.isLocal() ?
-                cacheCtx.affinity().nodes(txEntry.key(), topVer) :
+                cacheCtx.affinity().nodesByKey(txEntry.key(), topVer) :
                 cacheCtx.topology().nodes(cacheCtx.affinity().partition(txEntry.key()), topVer);
 
             ClusterNode primary = F.first(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
index b3eb7551f05a0..940dd809779a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
@@ -476,7 +476,7 @@ private void processLockResponse(UUID nodeId, GridNearLockResponse res) {
      * @return {@code True} if entry is locally mapped as a primary or back up node.
      */
     protected boolean isNearLocallyMapped(GridCacheEntryEx e, AffinityTopologyVersion topVer) {
-        return ctx.affinity().belongs(ctx.localNode(), e.partition(), topVer);
+        return ctx.affinity().partitionBelongs(ctx.localNode(), e.partition(), topVer);
     }
 
     /**
@@ -548,7 +548,7 @@ protected boolean evictNearEntry(GridCacheEntryEx e, GridCacheVersion obsoleteVe
                             topVer = cand.topologyVersion();
 
                             // Send request to remove from remote nodes.
-                            ClusterNode primary = ctx.affinity().primary(key, topVer);
+                            ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                             if (primary == null) {
                                 if (log.isDebugEnabled())
@@ -668,7 +668,7 @@ public void removeLocks(GridCacheVersion ver, Collection<KeyCacheObject> keys) {
                                     map = U.newHashMap(affNodes.size());
                                 }
 
-                                ClusterNode primary = ctx.affinity().primary(key, cand.topologyVersion());
+                                ClusterNode primary = ctx.affinity().primaryByKey(key, cand.topologyVersion());
 
                                 if (primary == null) {
                                     if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 9acab56ae2d8e..aed1ab0ac18de 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -350,7 +350,7 @@ else if (err != null)
                             GridCacheContext cacheCtx = e.context();
 
                             try {
-                                if (e.op() != NOOP && !cacheCtx.affinity().localNode(e.key(), topVer)) {
+                                if (e.op() != NOOP && !cacheCtx.affinity().keyLocalNode(e.key(), topVer)) {
                                     GridCacheEntryEx entry = cacheCtx.cache().peekEx(e.key());
 
                                     if (entry != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 7efb746d4428e..47f1bed98680c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -1580,11 +1580,11 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                     // Other types are filtered in indexing manager.
                     if (!cctx.isReplicated() && qry.type() == SCAN && qry.partition() == null &&
                         cctx.config().getCacheMode() != LOCAL && !incBackups &&
-                        !cctx.affinity().primary(cctx.localNode(), key, topVer)) {
+                        !cctx.affinity().primaryByKey(cctx.localNode(), key, topVer)) {
                         if (log.isDebugEnabled())
                             log.debug("Ignoring backup element [row=" + row +
                                 ", cacheMode=" + cctx.config().getCacheMode() + ", incBackups=" + incBackups +
-                                ", primary=" + cctx.affinity().primary(cctx.localNode(), key, topVer) + ']');
+                                ", primary=" + cctx.affinity().primaryByKey(cctx.localNode(), key, topVer) + ']');
 
                         continue;
                     }
@@ -1592,7 +1592,8 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                     V val = row.getValue();
 
                     if (log.isDebugEnabled()) {
-                        ClusterNode primaryNode = CU.primaryNode(cctx, key);
+                        ClusterNode primaryNode = cctx.affinity().primaryByKey(key,
+                            cctx.affinity().affinityTopologyVersion());
 
                         log.debug(S.toString("Record",
                             "key", key, true,
@@ -2355,7 +2356,7 @@ public Collection<GridCacheSqlMetadata> sqlMetadata() throws IgniteCheckedExcept
 
                 return new IgniteBiPredicate<K, V>() {
                     @Override public boolean apply(K k, V v) {
-                        return cache.context().affinity().primary(ctx.discovery().localNode(), k, NONE);
+                        return cache.context().affinity().primaryByKey(ctx.discovery().localNode(), k, NONE);
                     }
                 };
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 26c361498fbab..ceebbe96dbd56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -56,7 +56,6 @@
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheDeploymentManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 9c300e70adab6..acd5bf1a454a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -372,7 +372,7 @@ public void onEntryExpired(GridCacheEntryEx e, KeyCacheObject key, CacheObject o
         if (F.isEmpty(lsnrCol))
             return;
 
-        boolean primary = cctx.affinity().primary(cctx.localNode(), e.partition(), AffinityTopologyVersion.NONE);
+        boolean primary = cctx.affinity().primaryByPartition(cctx.localNode(), e.partition(), AffinityTopologyVersion.NONE);
 
         if (cctx.isReplicated() || primary) {
             boolean recordIgniteEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index cd5babe7118b0..266c5a83cd8c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -1293,7 +1293,7 @@ protected void batchStoreCommit(Iterable<IgniteTxEntry> writeEntries) throws Ign
                         if (!skip && skipNonPrimary) {
                             skip = e.cached().isNear() ||
                                 e.cached().detached() ||
-                                !e.context().affinity().primary(e.cached().partition(), topologyVersion()).isLocal();
+                                !e.context().affinity().primaryByPartition(e.cached().partition(), topologyVersion()).isLocal();
                         }
 
                         if (!skip && !local() && // Update local store at backups only if needed.
@@ -1712,7 +1712,7 @@ protected boolean isNearLocallyMapped(IgniteTxEntry e, boolean primaryOnly) {
 
         int part = cached != null ? cached.partition() : cacheCtx.affinity().partition(e.key());
 
-        List<ClusterNode> affNodes = cacheCtx.affinity().nodes(part, topologyVersion());
+        List<ClusterNode> affNodes = cacheCtx.affinity().nodesByPartition(part, topologyVersion());
 
         e.locallyMapped(F.contains(affNodes, cctx.localNode()));
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 7f161859da101..0d992286114b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1106,7 +1106,7 @@ assert isWriteToStoreFromDhtValid(stores) :
      * @return {@code True} if local node is current primary for given entry.
      */
     private boolean primaryLocal(GridCacheEntryEx entry) {
-        return entry.context().affinity().primary(cctx.localNode(), entry.partition(), AffinityTopologyVersion.NONE);
+        return entry.context().affinity().primaryByPartition(cctx.localNode(), entry.partition(), AffinityTopologyVersion.NONE);
     }
 
     /**
@@ -1407,7 +1407,7 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                     finally {
                         if (entry != null && readCommitted()) {
                             if (cacheCtx.isNear()) {
-                                if (cacheCtx.affinity().belongs(cacheCtx.localNode(), entry.partition(), topVer)) {
+                                if (cacheCtx.affinity().partitionBelongs(cacheCtx.localNode(), entry.partition(), topVer)) {
                                     if (entry.markObsolete(xidVer))
                                         cacheCtx.cache().removeEntry(entry);
                                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
index 70d938e144ead..67d00ea782923 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
@@ -401,7 +401,7 @@ private void mapTxKeys(@Nullable Set<IgniteTxKey> txKeys, Map<IgniteTxKey, TxLoc
         private UUID primary(IgniteTxKey txKey) {
             GridCacheContext ctx = cctx.cacheContext(txKey.cacheId());
 
-            ClusterNode node = ctx.affinity().primary(txKey.key(), topVer);
+            ClusterNode node = ctx.affinity().primaryByKey(txKey.key(), topVer);
 
             assert node != null : topVer;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index 8978fbf34dffe..604b35d85a91e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -1951,7 +1951,7 @@ else if (ttl == CU.TTL_NOT_CHANGED)
                         expiryTime = CU.toExpireTime(ttl);
                     }
 
-                    boolean primary = cctx.affinity().primary(cctx.localNode(), entry.key(), topVer);
+                    boolean primary = cctx.affinity().primaryByKey(cctx.localNode(), entry.key(), topVer);
 
                     entry.initialValue(e.getValue(),
                         ver,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
index 70232afe717fa..d5f277ebf3f16 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
@@ -496,7 +496,7 @@ private Collection<ClusterNode> dataNodes(AffinityTopologyVersion topVer) throws
         Collection<ClusterNode> nodes;
 
         if (collocated) {
-            List<ClusterNode> nodes0 = ctx.affinity().nodes(hdrPart, topVer);
+            List<ClusterNode> nodes0 = ctx.affinity().nodesByPartition(hdrPart, topVer);
 
             nodes = !nodes0.isEmpty() ?
                 Collections.singleton(nodes0.contains(ctx.localNode()) ? ctx.localNode() : F.first(nodes0)) : nodes0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
index e8b2cc7f0f4b8..bc6c1827a0774 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
@@ -91,7 +91,7 @@ public IgniteUuid setId() {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public boolean apply(K k, V v) {
-        return !filter || ctx.affinity().primary(ctx.localNode(), k, ctx.affinity().affinityTopologyVersion());
+        return !filter || ctx.affinity().primaryByKey(ctx.localNode(), k, ctx.affinity().affinityTopologyVersion());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index 7889a5579c7bb..992e02a6fc570 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -1568,7 +1568,7 @@ public PartitionsReservation(int[] cacheIds, int partId,
                         }
                     }
                     finally {
-                        if (checkPartMapping && !cctx.affinity().primary(partId, topVer).id().equals(ctx.localNodeId()))
+                        if (checkPartMapping && !cctx.affinity().primaryByPartition(partId, topVer).id().equals(ctx.localNodeId()))
                             throw new IgniteException("Failed partition reservation. " +
                                 "Partition is not primary on the node. [partition=" + partId + ", cacheName=" + cctx.name() +
                                 ", nodeId=" + ctx.localNodeId() + ", topology=" + topVer + ']');
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
index 92e2b9b2ba21d..b0337d62967e0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
@@ -214,12 +214,12 @@ public CheckCallable(Object key, AffinityTopologyVersion topVer) {
 
                 ClusterNode loc = ignite.cluster().localNode();
 
-                if (loc.equals(aff.primary(key, topVer)))
+                if (loc.equals(aff.primaryByKey(key, topVer)))
                     return true;
 
                 AffinityTopologyVersion topVer0 = new AffinityTopologyVersion(topVer.topologyVersion() + 1, 0);
 
-                assertEquals(loc, aff.primary(key, topVer0));
+                assertEquals(loc, aff.primaryByKey(key, topVer0));
             }
 
             return null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
index 1cfb330da8cd6..53b4900410033 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
@@ -5837,7 +5837,7 @@ public CheckEntriesTask(Collection<String> keys) {
             int size = 0;
 
             for (String key : keys) {
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx())) {
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx())) {
                     GridCacheEntryEx e =
                         ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key);
 
@@ -5873,7 +5873,7 @@ private static class CheckCacheSizeTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : map.keySet())
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(ctx.name()).localSize(ALL));
@@ -6116,7 +6116,7 @@ public CheckKeySizeTask(Collection<String> keys) {
             int size = 0;
 
             for (String key : keys)
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(null).localSize(ALL));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
index 6b0e1932e1f2c..d4449f9ee9228 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
@@ -5548,7 +5548,7 @@ private static class CheckEntriesTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : keys) {
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx())) {
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx())) {
                     GridCacheEntryEx e =
                         ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key);
 
@@ -5589,7 +5589,7 @@ private static class CheckCacheSizeTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : map.keySet())
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(ctx.name()).localSize(ALL));
@@ -5850,7 +5850,7 @@ public CheckKeySizeTask(Collection<String> keys, String s) {
             int size = 0;
 
             for (String key : keys)
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(cacheName).localSize(ALL));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
index 6c577c63ab693..fac24ccd00045 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
@@ -1009,9 +1009,9 @@ private T2<Integer, Integer> swapKeysCount(int nodeIdx, int part) throws IgniteC
         //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primary(ctx.localNode(), part, topVer))
+        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             primaryCnt = cnt;
-        else if (affinity.backup(ctx.localNode(), part, topVer))
+        else if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             backupCnt = cnt;
         return new T2<>(primaryCnt, backupCnt);
     }
@@ -1081,9 +1081,9 @@ private T2<Integer, Integer> offheapKeysCount(int nodeIdx, int part) throws Igni
         //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primary(ctx.localNode(), part, topVer))
+        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             primaryCnt = cnt;
-        else if (affinity.backup(ctx.localNode(), part, topVer))
+        else if (affinity.backupByPartition(ctx.localNode(), part, topVer))
             backupCnt = cnt;
         return new T2<>(primaryCnt, backupCnt);
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
index 45b4b9f299aa3..aeca2fb2279ff 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
@@ -365,7 +365,7 @@ private int primaryQueueNode(IgniteQueue queue) {
 
         for (int i = 0; i < gridCount(); i++) {
             for (GridCacheEntryEx e : ((IgniteKernal)grid(i)).context().cache().internalCache(cctx.name()).allEntries()) {
-                if (aff.primary(grid(i).localNode(), e.key(), AffinityTopologyVersion.NONE)
+                if (aff.primaryByKey(grid(i).localNode(), e.key(), AffinityTopologyVersion.NONE)
                     && e.key().value(cctx.cacheObjectContext(), false) instanceof GridCacheQueueHeaderKey)
                     return i;
             }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
index b4ef11a82c0c1..8709d05664d94 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
@@ -853,13 +853,13 @@ public void testPessimisticTx2() throws Exception {
 
         GridCacheAffinityManager aff = ignite0.context().cache().internalCache(null).context().affinity();
 
-        List<ClusterNode> nodes1 = aff.nodes(key1, topVer1);
-        List<ClusterNode> nodes2 = aff.nodes(key1, topVer2);
+        List<ClusterNode> nodes1 = aff.nodesByKey(key1, topVer1);
+        List<ClusterNode> nodes2 = aff.nodesByKey(key1, topVer2);
 
         assertEquals(nodes1, nodes2);
 
-        nodes1 = aff.nodes(key2, topVer1);
-        nodes2 = aff.nodes(key2, topVer2);
+        nodes1 = aff.nodesByKey(key2, topVer1);
+        nodes2 = aff.nodesByKey(key2, topVer2);
 
         assertFalse(nodes1.get(0).equals(nodes2.get(0)));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
index f6a06c29a9fbe..66cfc4ee7338c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
@@ -289,7 +289,7 @@ private void doTestDeadlock(
                     key = keys.get(1);
 
                     ClusterNode primaryNode =
-                        ((IgniteCacheProxy)cache).context().affinity().primary(key, NONE);
+                        ((IgniteCacheProxy)cache).context().affinity().primaryByKey(key, NONE);
 
                     List<Integer> primaryKeys =
                         primaryKeys(grid(primaryNode).cache(CACHE_NAME), 5, key + (100 * threadNum));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
index 83eb908d21978..ced8b61166876 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
@@ -293,7 +293,7 @@ private void doTestDeadlock(
                     key = keys.get(1);
 
                     ClusterNode primaryNode =
-                        ((IgniteCacheProxy)cache).context().affinity().primary(key, NONE);
+                        ((IgniteCacheProxy)cache).context().affinity().primaryByKey(key, NONE);
 
                     List<Integer> primaryKeys =
                         primaryKeys(grid(primaryNode).cache(CACHE_NAME), 5, key + (100 * threadNum));
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 66fb7ae94f5be..e4b0c1feb7a21 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -2106,7 +2106,7 @@ private void createSqlFunctions(String schema, Class<?>[] clss) throws IgniteChe
 
                 return new IgniteBiPredicate<K, V>() {
                     @Override public boolean apply(K k, V v) {
-                        return aff.primary(locNode, k, topVer0);
+                        return aff.primaryByKey(locNode, k, topVer0);
                     }
                 };
             }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 22b94c7ac796b..3700774260ded 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -612,7 +612,7 @@ private ClusterNode rangeNode(GridCacheContext<?,?> cctx, GridH2QueryContext qct
             node = cctx.discovery().node(nodeId);
         }
         else // Get primary node for current topology version.
-            node = cctx.affinity().primary(affKeyObj, qctx.topologyVersion());
+            node = cctx.affinity().primaryByKey(affKeyObj, qctx.topologyVersion());
 
         if (node == null) // Node was not found, probably topology changed and we need to retry the whole query.
             throw new GridH2RetryException("Failed to find node.");

From 61e2809fa3a65b96d283a44de4c9e42c4dae1a0d Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 6 Sep 2017 11:54:19 +0300
Subject: [PATCH 403/446] IGNITE-6219 - IgniteCache#loadCache executes local
 load in caller thread

(cherry picked from commit 0e63f59)
---
 .../processors/task/GridTaskWorker.java       |  22 +++-
 .../cache/store/GridStoreLoadCacheTest.java   | 120 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite.java      |   5 +-
 3 files changed, 145 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/GridStoreLoadCacheTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
index c426008f5fb11..8787f4b92b9bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
@@ -579,7 +579,7 @@ private void processMappedJobs(Map<? extends ComputeJob, ClusterNode> jobs) thro
         if (F.isEmpty(jobs))
             return;
 
-        Collection<GridJobResultImpl> jobResList = new ArrayList<>(jobs.size());
+        List<GridJobResultImpl> jobResList = new ArrayList<>(jobs.size());
 
         Collection<ComputeJobSibling> sibs = new ArrayList<>(jobs.size());
 
@@ -637,6 +637,26 @@ private void processMappedJobs(Map<? extends ComputeJob, ClusterNode> jobs) thro
         // Set mapped flag.
         ses.onMapped();
 
+        // Move local jobs to the end of the list, because
+        // they will be invoked in current thread that will hold other
+        // jobs.
+        int jobResSize = jobResList.size();
+
+        if (jobResSize > 1) {
+            UUID locId = ctx.discovery().localNode().id();
+
+            for (int i = 0; i < jobResSize; i++) {
+                UUID jobNodeId = jobResList.get(i).getNode().id();
+
+                if (jobNodeId.equals(locId) && i < jobResSize - 1) {
+                    Collections.swap(jobResList, i, jobResSize - 1);
+
+                    jobResSize--;
+                    i--;
+                }
+            }
+        }
+
         // Send out all remote mappedJobs.
         for (GridJobResultImpl res : jobResList) {
             evtLsnr.onJobSend(this, res.getSibling());
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/GridStoreLoadCacheTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/GridStoreLoadCacheTest.java
new file mode 100644
index 0000000000000..d88c4318ee27e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/GridStoreLoadCacheTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store;
+
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.lang.IgniteBiInClosure;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test checks that local cacheLoad task never blocks remote
+ * cacheLoad.
+ */
+public class GridStoreLoadCacheTest extends GridCommonAbstractTest {
+    /** Barrier. */
+    private static final CyclicBarrier BARRIER = new CyclicBarrier(3);
+
+    /** Cache name. */
+    public static final String CACHE_NAME = "test";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        //noinspection unchecked
+        cfg.setCacheConfiguration(new CacheConfiguration(CACHE_NAME).setCacheStoreFactory(new TestFactory()));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void test() throws Exception {
+        for (int i = 0; i < 3; i++) {
+            IgniteEx srv1 = startGrid(0);
+            startGrid(1);
+            startGrid(2);
+
+            awaitPartitionMapExchange();
+
+            srv1.cache(CACHE_NAME).loadCache(null);
+
+            stopAllGrids();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestFactory implements Factory<CacheStore> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public CacheStore create() {
+            return new TestStore();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestStore extends CacheStoreAdapter<Object, Object> {
+        /** {@inheritDoc} */
+        @Override public void loadCache(IgniteBiInClosure<Object, Object> clo, Object... args) {
+            try {
+                BARRIER.await(3, TimeUnit.SECONDS);
+            }
+            catch (InterruptedException | BrokenBarrierException | TimeoutException e) {
+                throw new IgniteException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object load(Object key) throws CacheLoaderException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<?, ?> entry) throws CacheWriterException {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            // No-op
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index a24f020103e54..995a2023b8de9 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.cache.affinity.local.LocalAffinityFunctionTest;
 import org.apache.ignite.cache.store.GridCacheBalancingStoreSelfTest;
 import org.apache.ignite.cache.store.GridCacheLoadOnlyStoreAdapterSelfTest;
+import org.apache.ignite.cache.store.GridStoreLoadCacheTest;
 import org.apache.ignite.cache.store.StoreResourceInjectionSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest;
@@ -40,12 +41,12 @@
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreTest;
 import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreMultithreadedSelfTest;
 import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreSelfTest;
+import org.apache.ignite.cache.store.jdbc.JdbcTypesDefaultTransformerTest;
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceMultipleConnectionsTest;
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalancePairedConnectionsTest;
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceTest;
 import org.apache.ignite.internal.managers.communication.IgniteIoTestMessagesTest;
 import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest;
-import org.apache.ignite.cache.store.jdbc.JdbcTypesDefaultTransformerTest;
 import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest;
 import org.apache.ignite.internal.processors.cache.CacheEntryProcessorCopySelfTest;
@@ -346,6 +347,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(IgniteCommunicationBalanceMultipleConnectionsTest.class);
         suite.addTestSuite(IgniteIoTestMessagesTest.class);
 
+        suite.addTestSuite(GridStoreLoadCacheTest.class);
+
         return suite;
     }
 }

From 5b1fdfaee7422c02177513f6f8c1365a262fd49b Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 6 Sep 2017 18:33:31 +0300
Subject: [PATCH 404/446] IGNITE-6256: DiscoCache should always contains local
 node.

---
 .../managers/discovery/GridDiscoveryManager.java     | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 40dea984ee56d..8a625fc728da3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -613,7 +613,7 @@ else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
                     topHist.clear();
 
                     topSnap.set(new Snapshot(AffinityTopologyVersion.ZERO,
-                        createDiscoCache(locNode, Collections.<ClusterNode>emptySet())));
+                        createDiscoCache(locNode, Collections.<ClusterNode>singleton(locNode))));
                 }
                 else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                     assert locNode.isClient() : locNode;
@@ -1969,6 +1969,8 @@ public void reconnect() {
      * @return Newly created discovery cache.
      */
     @NotNull private DiscoCache createDiscoCache(ClusterNode loc, Collection<ClusterNode> topSnapshot) {
+        assert topSnapshot.contains(loc);
+
         HashSet<UUID> alives = U.newHashSet(topSnapshot.size());
         HashMap<UUID, ClusterNode> nodeMap = U.newHashMap(topSnapshot.size());
 
@@ -2172,15 +2174,15 @@ public void scheduleSegmentCheck() {
                     lastChk = now;
 
                     if (!segValid) {
-                        List<ClusterNode> empty = Collections.emptyList();
-
                         ClusterNode node = getSpi().getLocalNode();
 
+                        Collection<ClusterNode> locNodeOnlyTopology = Collections.singleton(node);
+
                         discoWrk.addEvent(EVT_NODE_SEGMENTED,
                             AffinityTopologyVersion.NONE,
                             node,
-                            createDiscoCache(node, empty),
-                            empty,
+                            createDiscoCache(node, locNodeOnlyTopology),
+                            locNodeOnlyTopology,
                             null);
 
                         lastSegChkRes.set(false);

From e1f1a51670bff70c4cd906505e0df6fa85b7a38c Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 5 Sep 2017 19:11:12 +0300
Subject: [PATCH 405/446] GG-12699: Fix
 GridCacheAbstractFullApiSelfTest.testTransformResourceInjection.

Squashed commit of the following:

commit 3f20fe8dcc796406f7a7791e3ae9ddb5c26183ca
Author: Nikolay Izhikov <nizhikov.dev@gmail.com>
Date:   Wed Aug 9 13:37:11 2017 +0300

    IGNITE-5897 Fix session init/end logic. This fixes tests.

    Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>

    (cherry picked from commit 5a559df)

commit 52e89d387874a0653c58a608cc000950a76fb6b0
Author: dpavlov <dpavlov@gridgain.com>
Date:   Wed Jul 26 17:23:05 2017 +0300

    IGNITE-5806 - Fixed assertion with a side-effect - Fixes #2335.

    Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

    (cherry picked from commit 9e79c4b)

commit d3c40e418dce6ab640fe06e8c18ada4b93f1edf5
Author: Andrey V. Mashenkov <andrey.mashenkov@gmail.com>
Date:   Mon Sep 4 15:57:16 2017 +0300

    Fix javadoc.
---
 .../cache/store/CacheStoreManager.java        |  4 +++-
 .../store/GridCacheStoreManagerAdapter.java   | 23 +++++++++----------
 .../cache/transactions/IgniteTxAdapter.java   |  9 ++++++--
 .../resource/GridResourceProcessor.java       |  6 ++++-
 4 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
index 67c9334fa6ed2..459c7029fbde8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
@@ -165,9 +165,11 @@ public boolean removeAll(@Nullable IgniteInternalTx tx, Collection<Object> keys)
     /**
      * @param tx Transaction.
      * @param commit Commit.
+     * @param last {@code True} if this is last store in transaction.
+     * @param storeSessionEnded {@code True} if session for underlying store already ended.
      * @throws IgniteCheckedException If failed.
      */
-    public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException;
+    public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last, boolean storeSessionEnded) throws IgniteCheckedException;
 
     /**
      * End session initiated by write-behind store.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index c35b8fb9d2c01..6ab8c56578593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -777,7 +777,8 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException {
+    @Override public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last,
+        boolean storeSessionEnded) throws IgniteCheckedException {
         assert store != null;
 
         sessionInit0(tx);
@@ -788,7 +789,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
                     lsnr.onSessionEnd(locSes, commit);
             }
 
-            if (!sesHolder.get().ended(store))
+            if (!sesHolder.get().ended(store) && !storeSessionEnded)
                 store.sessionEnd(commit);
         }
         catch (Throwable e) {
@@ -855,7 +856,7 @@ private void sessionInit0(@Nullable IgniteInternalTx tx) {
 
         sesHolder.set(ses);
 
-        if (sesLsnrs != null && !ses.started(this)) {
+        if (!ses.started(store) && sesLsnrs != null) {
             for (CacheStoreSessionListener lsnr : sesLsnrs)
                 lsnr.onSessionStart(locSes);
         }
@@ -872,7 +873,8 @@ private void sessionEnd0(@Nullable IgniteInternalTx tx, boolean threwEx) throws
                         lsnr.onSessionEnd(locSes, !threwEx);
                 }
 
-                store.sessionEnd(!threwEx);
+                if (!sesHolder.get().ended(store))
+                    store.sessionEnd(!threwEx);
             }
         }
         catch (Exception e) {
@@ -914,11 +916,8 @@ private static class SessionData {
         private Object attachment;
 
         /** */
-        private final Set<CacheStoreManager> started =
-            new GridSetWrapper<>(new IdentityHashMap<CacheStoreManager, Object>());
-
-        /** */
-        private final Set<CacheStore> ended = new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
+        private final Set<CacheStore> started =
+            new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
 
         /**
          * @param tx Current transaction.
@@ -981,8 +980,8 @@ private void cacheName(String cacheName) {
         /**
          * @return If session is started.
          */
-        private boolean started(CacheStoreManager mgr) {
-            return !started.add(mgr);
+        private boolean started(CacheStore store) {
+            return !started.add(store);
         }
 
         /**
@@ -990,7 +989,7 @@ private boolean started(CacheStoreManager mgr) {
          * @return Whether session already ended on this store instance.
          */
         private boolean ended(CacheStore store) {
-            return !ended.add(store);
+            return !started.remove(store);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index b07a1175c436d..266c5a83cd8c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -40,6 +41,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -61,6 +63,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
+import org.apache.ignite.internal.util.GridSetWrapper;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
 import org.apache.ignite.internal.util.lang.GridTuple;
@@ -1238,13 +1241,15 @@ protected boolean isWriteToStoreFromDhtValid(Collection<CacheStoreManager> store
      * @param commit Commit flag.
      * @throws IgniteCheckedException In case of error.
      */
-    protected void sessionEnd(Collection<CacheStoreManager> stores, boolean commit) throws IgniteCheckedException {
+    protected void sessionEnd(final Collection<CacheStoreManager> stores, boolean commit) throws IgniteCheckedException {
         Iterator<CacheStoreManager> it = stores.iterator();
 
+        Set<CacheStore> visited = new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
+
         while (it.hasNext()) {
             CacheStoreManager store = it.next();
 
-            store.sessionEnd(this, commit, !it.hasNext());
+            store.sessionEnd(this, commit, !it.hasNext(), !visited.add(store.store()));
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
index 84d07b64e50e5..02616b5e983a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
@@ -20,7 +20,6 @@
 import java.lang.annotation.Annotation;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.util.Collection;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.store.CacheStoreSession;
 import org.apache.ignite.compute.ComputeJob;
@@ -318,6 +317,11 @@ private GridResourceInjector injectorByAnnotation(GridResourceIoc.ResourceAnnota
 
     /**
      * @param obj Object to inject.
+     * @param ann Annotation enum.
+     * @param dep Grid deployment object.
+     * @param depCls Grid deployment class.
+     * @param param Resource to inject.
+     * @return {@code True} if resource was injected.
      * @throws IgniteCheckedException If failed to inject.
      */
     private boolean inject(Object obj, GridResourceIoc.ResourceAnnotation ann, @Nullable GridDeployment dep,

From c746277c2c6ce07871ba29fa3cd1bab7a78f8662 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Tue, 5 Sep 2017 18:04:54 +0300
Subject: [PATCH 406/446] ignite-5145 Support multiple service deployment in
 API

(cherry picked from commit 0b6da97)
---
 .../org/apache/ignite/IgniteServices.java     |  57 +-
 .../ignite/internal/IgniteServicesImpl.java   |  25 +-
 .../GridServiceDeploymentCompoundFuture.java  | 196 +++++
 .../service/GridServiceProcessor.java         | 481 ++++++++---
 .../service/PreparedConfigurations.java       |  53 ++
 .../service/ServiceDeploymentException.java   |  78 ++
 .../util/future/GridCompoundFuture.java       |  15 +-
 .../util/future/GridFutureAdapter.java        |   2 +-
 ...rviceDeploymentCompoundFutureSelfTest.java | 241 ++++++
 ...idServiceProcessorBatchDeploySelfTest.java | 765 ++++++++++++++++++
 .../testsuites/IgniteKernalSelfTestSuite.java |   4 +
 11 files changed, 1786 insertions(+), 131 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
index 8365ec7e9523b..ae3958fc269cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
@@ -20,6 +20,7 @@
 import java.util.Collection;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.service.ServiceDeploymentException;
 import org.apache.ignite.lang.IgniteAsyncSupport;
 import org.apache.ignite.lang.IgniteAsyncSupported;
 import org.apache.ignite.resources.IgniteInstanceResource;
@@ -154,10 +155,10 @@ public interface IgniteServices extends IgniteAsyncSupport {
      *
      * @param name Service name.
      * @param svc Service instance.
-     * @throws IgniteException If failed to deploy service.
+     * @throws ServiceDeploymentException If failed to deploy service.
      */
     @IgniteAsyncSupported
-    public void deployClusterSingleton(String name, Service svc) throws IgniteException;
+    public void deployClusterSingleton(String name, Service svc) throws ServiceDeploymentException;
 
     /**
      * Deploys a per-node singleton service. Ignite will guarantee that there is always
@@ -170,10 +171,10 @@ public interface IgniteServices extends IgniteAsyncSupport {
      *
      * @param name Service name.
      * @param svc Service instance.
-     * @throws IgniteException If failed to deploy service.
+     * @throws ServiceDeploymentException If failed to deploy service.
      */
     @IgniteAsyncSupported
-    public void deployNodeSingleton(String name, Service svc) throws IgniteException;
+    public void deployNodeSingleton(String name, Service svc) throws ServiceDeploymentException;
 
     /**
      * Deploys one instance of this service on the primary node for a given affinity key.
@@ -204,11 +205,11 @@ public interface IgniteServices extends IgniteAsyncSupport {
      * @param cacheName Name of the cache on which affinity for key should be calculated, {@code null} for
      *      default cache.
      * @param affKey Affinity cache key.
-     * @throws IgniteException If failed to deploy service.
+     * @throws ServiceDeploymentException If failed to deploy service.
      */
     @IgniteAsyncSupported
     public void deployKeyAffinitySingleton(String name, Service svc, @Nullable String cacheName, Object affKey)
-        throws IgniteException;
+        throws ServiceDeploymentException;
 
     /**
      * Deploys multiple instances of the service on the grid. Ignite will deploy a
@@ -238,10 +239,11 @@ public void deployKeyAffinitySingleton(String name, Service svc, @Nullable Strin
      * @param svc Service instance.
      * @param totalCnt Maximum number of deployed services in the grid, {@code 0} for unlimited.
      * @param maxPerNodeCnt Maximum number of deployed services on each node, {@code 0} for unlimited.
-     * @throws IgniteException If failed to deploy service.
+     * @throws ServiceDeploymentException If failed to deploy service.
      */
     @IgniteAsyncSupported
-    public void deployMultiple(String name, Service svc, int totalCnt, int maxPerNodeCnt) throws IgniteException;
+    public void deployMultiple(String name, Service svc, int totalCnt, int maxPerNodeCnt)
+        throws ServiceDeploymentException;
 
     /**
      * Deploys multiple instances of the service on the grid according to provided
@@ -277,10 +279,31 @@ public void deployKeyAffinitySingleton(String name, Service svc, @Nullable Strin
      * </pre>
      *
      * @param cfg Service configuration.
-     * @throws IgniteException If failed to deploy service.
+     * @throws ServiceDeploymentException If failed to deploy service.
      */
     @IgniteAsyncSupported
-    public void deploy(ServiceConfiguration cfg) throws IgniteException;
+    public void deploy(ServiceConfiguration cfg) throws ServiceDeploymentException;
+
+    /**
+     * Deploys multiple services described by provided configurations. Depending on specified parameters, multiple
+     * instances of the same service may be deployed (see {@link ServiceConfiguration}).
+     * Whenever topology changes, Ignite will automatically rebalance
+     * the deployed services within cluster to make sure that each node will end up with
+     * about equal number of deployed instances whenever possible.
+     *
+     * If deployment fails, then {@link ServiceDeploymentException} containing a list of failed services will be
+     * thrown. It is guaranteed that all services that were provided to this method and are not present in the list of
+     * failed services are successfully deployed by the moment of the exception being thrown.
+     *
+     * @param cfgs {@link Collection} of service configurations to be deployed.
+     * @param allOrNone Specifies behavior in case when errors during deployment occur. If {@code true}, then two
+     * outcomes are possible: either all services will be deployed, or none of them. If {@code false}, then partial
+     * deployments are permitted.
+     * @throws ServiceDeploymentException If failed to deploy services.
+     * @see IgniteServices#deploy(ServiceConfiguration)
+     */
+    @IgniteAsyncSupported
+    public void deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) throws ServiceDeploymentException;
 
     /**
      * Cancels service deployment. If a service with specified name was deployed on the grid,
@@ -298,6 +321,20 @@ public void deployKeyAffinitySingleton(String name, Service svc, @Nullable Strin
     @IgniteAsyncSupported
     public void cancel(String name) throws IgniteException;
 
+    /**
+     * Cancels services with specified names.
+     * <p>
+     * Note that depending on user logic, it may still take extra time for a service to
+     * finish execution, even after it was cancelled.
+     * <p>
+     * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+     *
+     * @param names Names of services to cancel.
+     * @throws IgniteException If failed to cancel services.
+     */
+    @IgniteAsyncSupported
+    public void cancelAll(Collection<String> names) throws IgniteException;
+
     /**
      * Cancels all deployed services.
      * <p>
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
index df6e5df4fc411..a51f3f1ba95fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
@@ -23,6 +23,7 @@
 import java.io.ObjectOutput;
 import java.io.ObjectStreamException;
 import java.util.Collection;
+import java.util.Collections;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteServices;
@@ -150,10 +151,17 @@ public IgniteServicesImpl(GridKernalContext ctx, ClusterGroupAdapter prj, boolea
     @Override public void deploy(ServiceConfiguration cfg) {
         A.notNull(cfg, "cfg");
 
+        deployAll(Collections.singleton(cfg), false);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+        A.notNull(cfgs, "cfgs");
+
         guard();
 
         try {
-            saveOrGet(ctx.service().deploy(cfg));
+            saveOrGet(ctx.service().deployAll(cfgs, allOrNone));
         }
         catch (IgniteCheckedException e) {
             throw U.convertException(e);
@@ -180,6 +188,21 @@ public IgniteServicesImpl(GridKernalContext ctx, ClusterGroupAdapter prj, boolea
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void cancelAll(Collection<String> names) {
+        guard();
+
+        try {
+            saveOrGet(ctx.service().cancelAll(names));
+        }
+        catch (IgniteCheckedException e) {
+            throw U.convertException(e);
+        }
+        finally {
+            unguard();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void cancelAll() {
         guard();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
new file mode 100644
index 0000000000000..12b88e5f119f5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Service deployment compound future, {@code allOrNone} parameter specifies failing policy.
+ * <p>
+ * If {@code allOrNone} parameter is set to {@code false}, then this future waits for completion of all child futures.
+ * If any exceptions are thrown during deployment, then {@link IgniteCheckedException} with {@link
+ * ServiceDeploymentException} as a cause will be thrown from {@link IgniteInternalFuture#get get()} method after all
+ * futures complete or fail. Inner exception will contain configurations of failed services.
+ */
+public class GridServiceDeploymentCompoundFuture extends GridCompoundFuture<Object, Object> {
+    /** */
+    private final boolean allOrNone;
+
+    /** Kernal context. */
+    private final GridKernalContext ctx;
+
+    /** Logger. */
+    private final IgniteLogger log;
+
+    /** Names of services written to cache during current deployment. */
+    private Collection<String> svcsToRollback;
+
+    /** */
+    private volatile ServiceDeploymentException err;
+
+    /**
+     * @param allOrNone Failing policy.
+     * @param ctx Kernal context.
+     */
+    GridServiceDeploymentCompoundFuture(boolean allOrNone, GridKernalContext ctx) {
+        this.allOrNone = allOrNone;
+        this.ctx = ctx;
+        this.log = ctx.log(getClass());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean processFailure(Throwable err, IgniteInternalFuture<Object> fut) {
+        assert fut instanceof GridServiceDeploymentFuture : fut;
+
+        GridServiceDeploymentFuture depFut = (GridServiceDeploymentFuture)fut;
+
+        if (allOrNone) {
+            if (initialized()) {
+                onDone(new IgniteCheckedException(
+                    new ServiceDeploymentException("Failed to deploy provided services.", err, getConfigurations())));
+            }
+            else {
+                synchronized (this) {
+                    if (this.err == null) {
+                        this.err = new ServiceDeploymentException("Failed to deploy provided services.", err,
+                            new ArrayList<ServiceConfiguration>());
+                    }
+                    else
+                        this.err.addSuppressed(err);
+                }
+            }
+        }
+        else {
+            synchronized (this) {
+                if (this.err == null)
+                    this.err = new ServiceDeploymentException("Failed to deploy some services.",
+                        new ArrayList<ServiceConfiguration>());
+
+                this.err.getFailedConfigurations().add(depFut.configuration());
+                this.err.addSuppressed(err);
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Marks this future as initialized. Will complete with error if failures before initialization occurred and
+     * all-or-none policy is followed.
+     */
+    public void serviceDeploymentMarkInitialized() {
+        if (allOrNone && this.err != null) {
+            this.err.getFailedConfigurations().addAll(getConfigurations());
+
+            onDone(new IgniteCheckedException(this.err));
+        }
+        else
+            super.markInitialized();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean onDone(@Nullable final Object res, @Nullable Throwable err, final boolean cancel) {
+        final Throwable resErr;
+
+        if (err == null && this.err != null)
+            resErr = new IgniteCheckedException(this.err);
+        else
+            resErr = err;
+
+        if (allOrNone && this.err != null && svcsToRollback != null) {
+            U.warn(log, "Failed to deploy provided services. The following services will be cancelled:" + svcsToRollback);
+
+            IgniteInternalFuture<?> fut = ctx.service().cancelAll(svcsToRollback);
+
+            /*
+            Can not call fut.get() since it is possible we are in system pool now and
+            fut also should be completed from system pool.
+             */
+            fut.listen(new IgniteInClosure<IgniteInternalFuture>() {
+                @Override public void apply(IgniteInternalFuture fut) {
+                    try {
+                        fut.get();
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to cancel deployed services.", e);
+                    }
+                    finally {
+                        svcsToRollback = null;
+                    }
+
+                    GridServiceDeploymentCompoundFuture.super.onDone(res, resErr, cancel);
+                }
+            });
+
+            return false;
+        }
+
+        return super.onDone(res, resErr, cancel);
+    }
+
+    /**
+     * @param fut Child future.
+     * @param own If {@code true}, then corresponding service will be cancelled on failure.
+     */
+    public void add(GridServiceDeploymentFuture fut, boolean own) {
+        super.add(fut);
+
+        if (own) {
+            if (svcsToRollback == null)
+                svcsToRollback = new ArrayList<>();
+
+            svcsToRollback.add(fut.configuration().getName());
+        }
+    }
+
+    /**
+     * @return Collection of names of services that were written to cache during current deployment.
+     */
+    public Collection<String> servicesToRollback() {
+        if (svcsToRollback != null)
+            return svcsToRollback;
+        else
+            return Collections.emptyList();
+    }
+
+    /**
+     * @return Collection of configurations, stored in child futures.
+     */
+    private Collection<ServiceConfiguration> getConfigurations() {
+        Collection<IgniteInternalFuture<Object>> futs = futures();
+
+        List<ServiceConfiguration> cfgs = new ArrayList<>(futs.size());
+
+        for (IgniteInternalFuture<Object> fut : futs)
+            cfgs.add(((GridServiceDeploymentFuture)fut).configuration());
+
+        return cfgs;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index c961eb0443cdc..f457d8091a4ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.service;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -92,6 +93,7 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.plugin.security.SecurityException;
 import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
@@ -101,6 +103,7 @@
 import org.apache.ignite.services.ServiceDescriptor;
 import org.apache.ignite.spi.IgniteNodeValidationResult;
 import org.apache.ignite.thread.IgniteThreadFactory;
+import org.apache.ignite.transactions.Transaction;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
@@ -111,6 +114,7 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.UTILITY_CACHE_NAME;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
 import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
 /**
@@ -294,19 +298,13 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         ServiceConfiguration[] cfgs = ctx.config().getServiceConfiguration();
 
         if (cfgs != null) {
-            Collection<IgniteInternalFuture<?>> futs = new ArrayList<>();
-
             for (ServiceConfiguration c : cfgs) {
                 // Deploy only on server nodes by default.
                 if (c.getNodeFilter() == null)
                     c.setNodeFilter(ctx.cluster().get().forServers().predicate());
-
-                futs.add(deploy(c));
             }
 
-            // Await for services to deploy.
-            for (IgniteInternalFuture<?> f : futs)
-                f.get();
+            deployAll(Arrays.asList(cfgs), true).get();
         }
 
         if (log.isDebugEnabled())
@@ -487,130 +485,276 @@ public IgniteInternalFuture<?> deployKeyAffinitySingleton(String name, Service s
     }
 
     /**
-     * @param cfg Service configuration.
-     * @return Future for deployment.
+     * @param cfgs Service configurations.
+     * @param allOrNone Failure processing policy.
+     * @return Configurations to deploy.
      */
-    public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
-        A.notNull(cfg, "cfg");
+    private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+        List<ServiceConfiguration> cfgsCp = new ArrayList<>(cfgs.size());
 
         ServicesCompatibilityState state = markCompatibilityStateAsUsed();
 
-        validate(cfg);
-
-        ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
+        Marshaller marsh = ctx.config().getMarshaller();
 
-        if (!state.srvcCompatibility) {
-            Marshaller marsh = ctx.config().getMarshaller();
+        List<GridServiceDeploymentFuture> failedFuts = null;
 
-            LazyServiceConfiguration cfg0;
+        for (ServiceConfiguration cfg : cfgs) {
+            Exception err = null;
 
             try {
-                byte[] srvcBytes = U.marshal(marsh, cfg.getService());
-
-                cfg0 = new LazyServiceConfiguration(cfg, srvcBytes);
+                validate(cfg);
             }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to marshal service with configured marshaller [srvc=" + cfg.getService()
-                    + ", marsh=" + marsh + "]", e);
+            catch (Exception e) {
+                U.error(log, "Failed to validate service configuration [name=" + cfg.getName() +
+                    ", srvc=" + cfg.getService() + ']', e);
 
-                return new GridFinishedFuture<>(e);
+                err = e;
             }
 
-            cfg = cfg0;
-        }
+            if (err == null) {
+                try {
+                    ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
+                }
+                catch (Exception e) {
+                    U.error(log, "Failed to authorize service creation [name=" + cfg.getName() +
+                        ", srvc=" + cfg.getService() + ']', e);
 
-        GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+                    err = e;
+                }
+            }
 
-        GridServiceDeploymentFuture old = depFuts.putIfAbsent(cfg.getName(), fut);
+            if (err == null) {
+                if (!state.srvcCompatibility) {
+                    try {
+                        byte[] srvcBytes = U.marshal(marsh, cfg.getService());
 
-        if (old != null) {
-            if (!old.configuration().equalsIgnoreNodeFilter(cfg)) {
-                fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
-                    "different configuration) [deployed=" + old.configuration() + ", new=" + cfg + ']'));
+                        cfgsCp.add(new LazyServiceConfiguration(cfg, srvcBytes));
+                    }
+                    catch (Exception e) {
+                        U.error(log, "Failed to marshal service with configured marshaller [name=" + cfg.getName() +
+                            ", srvc=" + cfg.getService() + ", marsh=" + marsh + "]", e);
 
-                return fut;
+                        err = e;
+                    }
+                }
+                else
+                    cfgsCp.add(cfg);
             }
 
-            return old;
-        }
+            if (err != null) {
+                if (allOrNone) {
+                    return new PreparedConfigurations(null,
+                        null,
+                        new IgniteCheckedException(
+                            new ServiceDeploymentException("None of the provided services were deplyed.", err, cfgs)));
+                }
+                else {
+                    if (failedFuts == null)
+                        failedFuts = new ArrayList<>();
 
-        if (ctx.clientDisconnected()) {
-            fut.onDone(new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
-                "Failed to deploy service, client node disconnected."));
+                    GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+
+                    fut.onDone(err);
 
-            depFuts.remove(cfg.getName(), fut);
+                    failedFuts.add(fut);
+                }
+            }
         }
 
-        while (true) {
-            try {
-                GridServiceDeploymentKey key = new GridServiceDeploymentKey(cfg.getName());
+        return new PreparedConfigurations(cfgsCp, failedFuts, null);
+    }
 
-                if (ctx.deploy().enabled())
-                    ctx.cache().context().deploy().ignoreOwnership(true);
+    /**
+     * @param cfgs Service configurations.
+     * @param allOrNone Failure processing policy.
+     * @return Future for deployment.
+     */
+    public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+        assert cfgs != null;
 
-                try {
-                    GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
-                        new GridServiceDeployment(ctx.localNodeId(), cfg));
+        PreparedConfigurations srvCfg = prepareServiceConfigurations(cfgs, allOrNone);
 
-                    if (dep != null) {
-                        if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
-                            // Remove future from local map.
-                            depFuts.remove(cfg.getName(), fut);
+        if (srvCfg.err != null)
+            return new GridFinishedFuture<>(srvCfg.err);
 
-                            fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
-                                "different configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']'));
-                        }
-                        else {
-                            Iterator<Cache.Entry<Object, Object>> it = serviceEntries(
-                                ServiceAssignmentsPredicate.INSTANCE);
+        List<ServiceConfiguration> cfgsCp = srvCfg.cfgs;
+
+        List<GridServiceDeploymentFuture> failedFuts = srvCfg.failedFuts;
+
+        Collections.sort(cfgsCp, new Comparator<ServiceConfiguration>() {
+            @Override public int compare(ServiceConfiguration cfg1, ServiceConfiguration cfg2) {
+                return cfg1.getName().compareTo(cfg2.getName());
+            }
+        });
 
-                            while (it.hasNext()) {
-                                Cache.Entry<Object, Object> e = it.next();
+        GridServiceDeploymentCompoundFuture res;
 
-                                if (e.getKey() instanceof GridServiceAssignmentsKey) {
-                                    GridServiceAssignments assigns = (GridServiceAssignments)e.getValue();
+        while (true) {
+            res = new GridServiceDeploymentCompoundFuture(allOrNone, ctx);
 
-                                    if (assigns.name().equals(cfg.getName())) {
-                                        // Remove future from local map.
-                                        depFuts.remove(cfg.getName(), fut);
+            if (ctx.deploy().enabled())
+                ctx.cache().context().deploy().ignoreOwnership(true);
 
-                                        fut.onDone();
+            try {
+                if (cfgsCp.size() == 1)
+                    writeServiceToCache(res, cfgsCp.get(0));
+                else if (cfgsCp.size() > 1) {
+                    try (Transaction tx = cache.txStart(PESSIMISTIC, READ_COMMITTED)) {
+                        for (ServiceConfiguration cfg : cfgsCp) {
+                            try {
+                                writeServiceToCache(res, cfg);
+                            }
+                            catch (IgniteCheckedException e) {
+                                if (X.hasCause(e, ClusterTopologyCheckedException.class))
+                                    throw e; // Retry.
 
-                                        break;
-                                    }
+                                if (allOrNone) {
+                                    for (String name : res.servicesToRollback())
+                                        depFuts.remove(name).onDone(e);
+
+                                    res.onDone(new IgniteCheckedException(new ServiceDeploymentException(
+                                        "Failed to deploy provided services.", e, cfgs)));
+
+                                    return res;
                                 }
                             }
-
-                            if (!dep.configuration().equalsIgnoreNodeFilter(cfg))
-                                U.warn(log, "Service already deployed with different configuration (will ignore) " +
-                                    "[deployed=" + dep.configuration() + ", new=" + cfg + ']');
                         }
+
+                        tx.commit();
                     }
                 }
-                finally {
-                    if (ctx.deploy().enabled())
-                        ctx.cache().context().deploy().ignoreOwnership(false);
+
+                break;
+            }
+            catch (IgniteException | IgniteCheckedException e) {
+                for (String name : res.servicesToRollback())
+                    depFuts.remove(name).onDone(e);
+
+                if (X.hasCause(e, ClusterTopologyCheckedException.class)) {
+                    if (log.isDebugEnabled())
+                        log.debug("Topology changed while deploying services (will retry): " + e.getMessage());
                 }
+                else {
+                    res.onDone(new IgniteCheckedException(
+                        new ServiceDeploymentException("Failed to deploy provided services.", e, cfgs)));
 
-                return fut;
+                    return res;
+                }
             }
-            catch (ClusterTopologyCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
+            finally {
+                if (ctx.deploy().enabled())
+                    ctx.cache().context().deploy().ignoreOwnership(false);
             }
-            catch (IgniteCheckedException e) {
-                if (e.hasCause(ClusterTopologyCheckedException.class)) {
-                    if (log.isDebugEnabled())
-                        log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
+        }
 
-                    continue;
+        if (ctx.clientDisconnected()) {
+            IgniteClientDisconnectedCheckedException err =
+                new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
+                "Failed to deploy services, client node disconnected: " + cfgs);
+
+            for (String name : res.servicesToRollback()) {
+                GridServiceDeploymentFuture fut = depFuts.remove(name);
+
+                if (fut != null)
+                    fut.onDone(err);
+            }
+
+            return new GridFinishedFuture<>(err);
+        }
+
+        if (failedFuts != null) {
+            for (GridServiceDeploymentFuture fut : failedFuts)
+                res.add(fut, false);
+        }
+
+        res.serviceDeploymentMarkInitialized();
+
+        return res;
+    }
+
+    /**
+     * @param res Resulting compound future.
+     * @param cfg Service configuration.
+     * @throws IgniteCheckedException If operation failed.
+     */
+    private void writeServiceToCache(GridServiceDeploymentCompoundFuture res, ServiceConfiguration cfg)
+        throws IgniteCheckedException {
+        String name = cfg.getName();
+
+        GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+
+        GridServiceDeploymentFuture old = depFuts.putIfAbsent(name, fut);
+
+        try {
+            if (old != null) {
+                if (!old.configuration().equalsIgnoreNodeFilter(cfg))
+                    throw new IgniteCheckedException("Failed to deploy service (service already exists with different " +
+                        "configuration) [deployed=" + old.configuration() + ", new=" + cfg + ']');
+                else {
+                    res.add(old, false);
+
+                    return;
                 }
+            }
+
+            GridServiceDeploymentKey key = new GridServiceDeploymentKey(name);
+
+            GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
+                new GridServiceDeployment(ctx.localNodeId(), cfg));
+
+            if (dep != null) {
+                if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
+                    String err = "Failed to deploy service (service already exists with different " +
+                        "configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']';
+
+                    U.error(log, err);
+
+                    throw new IgniteCheckedException(err);
+                }
+                else {
+                    res.add(fut, false);
+
+                    Iterator<Cache.Entry<Object, Object>> it = serviceEntries(ServiceAssignmentsPredicate.INSTANCE);
+
+                    while (it.hasNext()) {
+                        Cache.Entry<Object, Object> e = it.next();
+
+                        if (e.getKey() instanceof GridServiceAssignmentsKey) {
+                            GridServiceAssignments assigns = (GridServiceAssignments)e.getValue();
 
-                U.error(log, "Failed to deploy service: " + cfg.getName(), e);
+                            if (assigns.name().equals(name)) {
+                                fut.onDone();
 
-                return new GridFinishedFuture<>(e);
+                                depFuts.remove(name, fut);
+
+                                break;
+                            }
+                        }
+                    }
+                }
             }
+            else
+                res.add(fut, true);
         }
+        catch (IgniteCheckedException e) {
+            fut.onDone(e);
+
+            res.add(fut, false);
+
+            depFuts.remove(name, fut);
+
+            throw e;
+        }
+    }
+
+    /**
+     * @param cfg Service configuration.
+     * @return Future for deployment.
+     */
+    public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
+        A.notNull(cfg, "cfg");
+
+        return deployAll(Collections.singleton(cfg), false);
     }
 
     /**
@@ -635,37 +779,19 @@ private ServicesCompatibilityState markCompatibilityStateAsUsed() {
      * @return Future.
      */
     public IgniteInternalFuture<?> cancel(String name) {
-        ctx.security().authorize(name, SecurityPermission.SERVICE_CANCEL, null);
-
         while (true) {
             try {
-                GridFutureAdapter<?> fut = new GridFutureAdapter<>();
-
-                GridFutureAdapter<?> old;
-
-                if ((old = undepFuts.putIfAbsent(name, fut)) != null)
-                    fut = old;
-                else {
-                    GridServiceDeploymentKey key = new GridServiceDeploymentKey(name);
-
-                    if (cache.getAndRemove(key) == null) {
-                        // Remove future from local map if service was not deployed.
-                        undepFuts.remove(name);
-
-                        fut.onDone();
-                    }
-                }
-
-                return fut;
+                return removeServiceFromCache(name).fut;
             }
-            catch (ClusterTopologyCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
-            }
-            catch (IgniteCheckedException e) {
-                log.error("Failed to undeploy service: " + name, e);
+            catch (IgniteException | IgniteCheckedException e) {
+                if (X.hasCause(e, ClusterTopologyCheckedException.class)) {
+                    if (log.isDebugEnabled())
+                        log.debug("Topology changed while cancelling service (will retry): " + e.getMessage());
+                } else {
+                    U.error(log, "Failed to undeploy service: " + name, e);
 
-                return new GridFinishedFuture<>(e);
+                    return new GridFinishedFuture<>(e);
+                }
             }
         }
     }
@@ -677,7 +803,7 @@ public IgniteInternalFuture<?> cancel(String name) {
     public IgniteInternalFuture<?> cancelAll() {
         Iterator<Cache.Entry<Object, Object>> it = serviceEntries(ServiceDeploymentPredicate.INSTANCE);
 
-        GridCompoundFuture res = null;
+        List<String> svcNames = new ArrayList<>();
 
         while (it.hasNext()) {
             Cache.Entry<Object, Object> e = it.next();
@@ -687,11 +813,68 @@ public IgniteInternalFuture<?> cancelAll() {
 
             GridServiceDeployment dep = (GridServiceDeployment)e.getValue();
 
-            if (res == null)
-                res = new GridCompoundFuture<>();
+            svcNames.add(dep.configuration().getName());
+        }
+
+        return cancelAll(svcNames);
+    }
+
+    /**
+     * @param svcNames Name of service to deploy.
+     * @return Future.
+     */
+    @SuppressWarnings("unchecked")
+    public IgniteInternalFuture<?> cancelAll(Collection<String> svcNames) {
+        List<String> svcNamesCp = new ArrayList<>(svcNames);
+
+        Collections.sort(svcNamesCp);
+
+        GridCompoundFuture res;
+
+        while (true) {
+            res = null;
+
+            List<String> toRollback = new ArrayList<>();
+
+            try (Transaction tx = cache.txStart(PESSIMISTIC, READ_COMMITTED)) {
+                for (String name : svcNames) {
+                    if (res == null)
+                        res = new GridCompoundFuture<>();
+
+                    try {
+                        CancelResult cr = removeServiceFromCache(name);
+
+                        if (cr.rollback)
+                            toRollback.add(name);
 
-            // Cancel each service separately.
-            res.add(cancel(dep.configuration().getName()));
+                        res.add(cr.fut);
+                    }
+                    catch (IgniteException | IgniteCheckedException e) {
+                        if (X.hasCause(e, ClusterTopologyCheckedException.class))
+                            throw e; // Retry.
+                        else {
+                            U.error(log, "Failed to undeploy service: " + name, e);
+
+                            res.add(new GridFinishedFuture<>(e));
+                        }
+                    }
+                }
+
+                tx.commit();
+
+                break;
+            }
+            catch (IgniteException | IgniteCheckedException e) {
+                for (String name : toRollback)
+                    undepFuts.remove(name).onDone(e);
+
+                if (X.hasCause(e, ClusterTopologyCheckedException.class)) {
+                    if (log.isDebugEnabled())
+                        log.debug("Topology changed while cancelling service (will retry): " + e.getMessage());
+                }
+                else
+                    return new GridFinishedFuture<>(e);
+            }
         }
 
         if (res != null) {
@@ -703,6 +886,50 @@ public IgniteInternalFuture<?> cancelAll() {
             return new GridFinishedFuture<>();
     }
 
+    /**
+     * @param name Name of service to remove from internal cache.
+     * @return Cancellation future and a flag whether it should be completed and removed on error.
+     * @throws IgniteCheckedException If operation failed.
+     */
+    private CancelResult removeServiceFromCache(String name) throws IgniteCheckedException {
+        try {
+            ctx.security().authorize(name, SecurityPermission.SERVICE_CANCEL, null);
+        }
+        catch (SecurityException e) {
+            return new CancelResult(new GridFinishedFuture<>(e), false);
+        }
+
+        GridFutureAdapter<?> fut = new GridFutureAdapter<>();
+
+        GridFutureAdapter<?> old = undepFuts.putIfAbsent(name, fut);
+
+        if (old != null)
+            return new CancelResult(old, false);
+        else {
+            GridServiceDeploymentKey key = new GridServiceDeploymentKey(name);
+
+            try {
+                if (cache.getAndRemove(key) == null) {
+                    // Remove future from local map if service was not deployed.
+                    undepFuts.remove(name, fut);
+
+                    fut.onDone();
+
+                    return new CancelResult(fut, false);
+                }
+                else
+                    return new CancelResult(fut, true);
+            }
+            catch (IgniteCheckedException e) {
+                undepFuts.remove(name, fut);
+
+                fut.onDone(e);
+
+                throw e;
+            }
+        }
+    }
+
     /**
      * @param name Service name.
      * @param timeout If greater than 0 limits task execution time. Cannot be negative.
@@ -1854,6 +2081,26 @@ private void undeploy(String name) {
         }
     }
 
+    /**
+     *
+     */
+    private static class CancelResult {
+        /** */
+        IgniteInternalFuture<?> fut;
+
+        /** */
+        boolean rollback;
+
+        /**
+         * @param fut Future.
+         * @param rollback {@code True} if service was cancelled during current call.
+         */
+        CancelResult(IgniteInternalFuture<?> fut, boolean rollback) {
+            this.fut = fut;
+            this.rollback = rollback;
+        }
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
new file mode 100644
index 0000000000000..a581e159348a8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.services.ServiceConfiguration;
+
+/**
+ * Result of services validation before deployment.
+ */
+class PreparedConfigurations {
+    /** */
+    final List<ServiceConfiguration> cfgs;
+
+    /** */
+    final List<GridServiceDeploymentFuture> failedFuts;
+
+    /** */
+    final Exception err;
+
+    /**
+     * @param cfgs Configurations to deploy.
+     * @param failedFuts Finished futures for failed configurations.
+     * @param err Error if need to stop deploy.
+     */
+    PreparedConfigurations(List<ServiceConfiguration> cfgs, List<GridServiceDeploymentFuture> failedFuts,
+        Exception err) {
+        this.cfgs = cfgs;
+        this.failedFuts = failedFuts;
+        this.err = err;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(PreparedConfigurations.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java
new file mode 100644
index 0000000000000..32fbf6f254250
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.Collection;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Exception indicating service deployment failure.
+ */
+public class ServiceDeploymentException extends IgniteException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final Collection<ServiceConfiguration> cfgs;
+
+    /**
+     * Creates service deployment exception with error message.
+     *
+     * @param msg Error message.
+     * @param cfgs Configurations of services that failed to deploy.
+     */
+    public ServiceDeploymentException(String msg, Collection<ServiceConfiguration> cfgs) {
+        super(msg);
+
+        this.cfgs = cfgs;
+    }
+
+    /**
+     * Creates service deployment exception with {@link Throwable} as a cause.
+     *
+     * @param cause Cause.
+     * @param cfgs Configurations of services that failed to deploy.
+     */
+    public ServiceDeploymentException(Throwable cause, Collection<ServiceConfiguration> cfgs) {
+        super(cause);
+
+        this.cfgs = cfgs;
+    }
+
+    /**
+     * Creates service deployment exception with error message and {@link Throwable} as a cause.
+     *
+     * @param msg Error message.
+     * @param cause Cause.
+     * @param cfgs Configurations of services that failed to deploy.
+     */
+    public ServiceDeploymentException(String msg, @Nullable Throwable cause, Collection<ServiceConfiguration> cfgs) {
+        super(msg, cause);
+
+        this.cfgs = cfgs;
+    }
+
+    /**
+     * @return Configurations of services that failed to deploy.
+     */
+    public Collection<ServiceConfiguration> getFailedConfigurations() {
+        return cfgs;
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
index 7abd3673df9c8..0e9575fe0d70d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
@@ -113,11 +113,11 @@ public GridCompoundFuture(@Nullable IgniteReducer<T, R> rdc) {
         }
         catch (IgniteTxOptimisticCheckedException | IgniteFutureCancelledCheckedException |
             ClusterTopologyCheckedException e) {
-            if (!ignoreFailure(e))
+            if (!processFailure(e, fut))
                 onDone(e);
         }
         catch (IgniteCheckedException e) {
-            if (!ignoreFailure(e)) {
+            if (!processFailure(e, fut)) {
                 U.error(null, "Failed to execute compound future reducer: " + this, e);
 
                 onDone(e);
@@ -182,6 +182,17 @@ protected boolean ignoreFailure(Throwable err) {
         return false;
     }
 
+    /**
+     * Processes error thrown by some of the inner futures.
+     *
+     * @param err Thrown exception.
+     * @param fut Failed future.
+     * @return {@code True} if this error should be ignored.
+     */
+    protected boolean processFailure(Throwable err, IgniteInternalFuture<T> fut) {
+        return ignoreFailure(err);
+    }
+
     /**
      * Checks if there are pending futures. This is not the same as
      * {@link #isDone()} because child classes may override {@link #onDone(Object, Throwable)}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
index 723dff7a3e38f..257f1996ddd08 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
@@ -361,7 +361,7 @@ public boolean onDone(@Nullable R res, @Nullable Throwable err) {
      * @param cancel {@code True} if future is being cancelled.
      * @return {@code True} if result was set by this call.
      */
-    private boolean onDone(@Nullable R res, @Nullable Throwable err, boolean cancel) {
+    protected boolean onDone(@Nullable R res, @Nullable Throwable err, boolean cancel) {
         boolean notify = false;
 
         try {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
new file mode 100644
index 0000000000000..51c3407e65da1
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/** */
+public class GridServiceDeploymentCompoundFutureSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static GridKernalContext ctx;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        IgniteKernal kernal = (IgniteKernal)startGrid(0);
+
+        ctx = kernal.context();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWaitForCompletionOnFailingFuturePartial() throws Exception {
+        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(false, ctx);
+
+        int failingFutsNum = 2;
+
+        int completingFutsNum = 5;
+
+        Collection<GridServiceDeploymentFuture> failingFuts = new ArrayList<>(completingFutsNum);
+
+        for (int i = 0; i < failingFutsNum; i++) {
+            ServiceConfiguration failingCfg = config("Failed-" + i);
+
+            GridServiceDeploymentFuture failingFut = new GridServiceDeploymentFuture(failingCfg);
+
+            failingFuts.add(failingFut);
+
+            compFut.add(failingFut);
+        }
+
+        List<GridFutureAdapter<Object>> futs = new ArrayList<>(completingFutsNum);
+
+        for (int i = 0; i < completingFutsNum; i++) {
+            GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(config(String.valueOf(i)));
+
+            futs.add(fut);
+
+            compFut.add(fut);
+        }
+
+        compFut.serviceDeploymentMarkInitialized();
+
+        List<Exception> causes = new ArrayList<>();
+
+        for (GridServiceDeploymentFuture fut : failingFuts) {
+            Exception cause = new Exception("Test error");
+
+            causes.add(cause);
+
+            fut.onDone(cause);
+        }
+
+        try {
+            compFut.get(100);
+
+            fail("Should never reach here.");
+        }
+        catch (IgniteFutureTimeoutCheckedException e) {
+            log.info("Expected exception: " + e.getMessage());
+        }
+
+        for (GridFutureAdapter<Object> fut : futs)
+            fut.onDone();
+
+        try {
+            compFut.get();
+
+            fail("Should never reach here.");
+        }
+        catch (IgniteCheckedException ce) {
+            log.info("Expected exception: " + ce.getMessage());
+
+            IgniteException e = U.convertException(ce);
+
+            assertTrue(e instanceof ServiceDeploymentException);
+
+            Throwable[] supErrs = e.getSuppressed();
+
+            assertEquals(failingFutsNum, supErrs.length);
+
+            for (int i = 0; i < failingFutsNum; i++)
+                assertEquals(causes.get(i), supErrs[i].getCause());
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testFailAllAfterInitialized() throws Exception {
+        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(true, ctx);
+
+        ServiceConfiguration failingCfg = config("Failed");
+
+        GridServiceDeploymentFuture failingFut = new GridServiceDeploymentFuture(failingCfg);
+
+        compFut.add(failingFut);
+
+        int futsNum = 5;
+
+        List<ServiceConfiguration> cfgs = new ArrayList<>(futsNum + 1);
+
+        cfgs.add(failingCfg);
+
+        for (int i = 0; i < futsNum; i++) {
+            ServiceConfiguration cfg = config(String.valueOf(i));
+
+            cfgs.add(cfg);
+
+            compFut.add(new GridServiceDeploymentFuture(cfg));
+        }
+
+        compFut.serviceDeploymentMarkInitialized();
+
+        Exception expCause = new Exception("Test error");
+
+        failingFut.onDone(expCause);
+
+        assertFailAll(compFut, cfgs, expCause);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testFailAllBeforeInitialized() throws Exception {
+        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(true, ctx);
+
+        ServiceConfiguration failingCfg = config("Failed");
+
+        GridServiceDeploymentFuture failingFut = new GridServiceDeploymentFuture(failingCfg);
+
+        Exception expCause = new Exception("Test error");
+
+        failingFut.onDone(expCause);
+
+        compFut.add(failingFut);
+
+        assertFalse(compFut.isDone());
+
+        int futsNum = 5;
+
+        List<ServiceConfiguration> cfgs = new ArrayList<>(futsNum + 1);
+
+        cfgs.add(failingCfg);
+
+        for (int i = 0; i < futsNum; i++) {
+            ServiceConfiguration cfg = config(String.valueOf(i));
+
+            cfgs.add(cfg);
+
+            compFut.add(new GridServiceDeploymentFuture(cfg));
+        }
+
+        compFut.serviceDeploymentMarkInitialized();
+
+        assertFailAll(compFut, cfgs, expCause);
+    }
+
+    /**
+     * Try waiting for the future completion and check that a proper exception is thrown.
+     *
+     * @param fut Future.
+     * @param expCfgs Expected cfgs.
+     * @param expCause Expected cause.
+     */
+    private void assertFailAll(GridServiceDeploymentCompoundFuture fut, Collection<ServiceConfiguration> expCfgs,
+        Exception expCause) {
+        try {
+            fut.get();
+
+            fail("Should never reach here.");
+        }
+        catch (IgniteCheckedException ce) {
+            log.info("Expected exception: " + ce.getMessage());
+
+            IgniteException e = U.convertException(ce);
+
+            assertTrue(e instanceof ServiceDeploymentException);
+
+            assertEqualsCollections(expCfgs, ((ServiceDeploymentException)e).getFailedConfigurations());
+
+            Throwable actCause = e.getCause();
+
+            assertTrue(actCause instanceof IgniteCheckedException);
+
+            assertEquals(expCause, actCause.getCause());
+        }
+    }
+
+    /**
+     * @param name Name.
+     * @return Dummy configuration with a specified name.
+     */
+    private ServiceConfiguration config(String name) {
+        ServiceConfiguration cfg = new ServiceConfiguration();
+
+        cfg.setName(name);
+
+        return cfg;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
new file mode 100644
index 0000000000000..c3232f23e9c2f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
@@ -0,0 +1,765 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteServices;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceDescriptor;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+
+/**
+ * Test for deployment of multiple configurations at a time.
+ */
+public class GridServiceProcessorBatchDeploySelfTest extends GridCommonAbstractTest {
+    /** Number of services to be deployed. */
+    private static final int NUM_SERVICES = 100;
+
+    /** Number of nodes in the test cluster. */
+    private static final int NUM_NODES = 4;
+
+    /** Client node name. */
+    private static final String CLIENT_NODE_NAME = "client";
+
+    /** IP finder. */
+    private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+        discoSpi.setIpFinder(ipFinder);
+
+        c.setDiscoverySpi(discoSpi);
+
+        return c;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        for (int i = 0; i < NUM_NODES; i++)
+            startGrid(i);
+
+        startGrid(CLIENT_NODE_NAME, getConfiguration(CLIENT_NODE_NAME).setClientMode(true));
+
+        DummyService.reset();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAll() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(cfgs, false);
+
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+        assertDeployedServices(client, cfgs);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAllAsync() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        IgniteServices services =  client.services().withAsync();
+
+        services.deployAll(cfgs, false);
+
+        services.future().get();
+
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+        assertDeployedServices(client, cfgs);
+    }
+
+    /**
+     * TODO: enable when IGNITE-6259 is fixed
+     *
+     * @throws Exception If failed.
+     */
+    public void _testDeployAllTopologyChange() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        final AtomicBoolean finished = new AtomicBoolean();
+
+        IgniteInternalFuture<Object> topChangeFut = runTopChanger(finished);
+
+        try {
+            int numServices = 500;
+            int batchSize = 5;
+
+            CountDownLatch latch = new CountDownLatch(numServices);
+
+            IgnitePredicate<ClusterNode> depPred = client.cluster().forServers()
+                .forPredicate(new IgnitePredicate<ClusterNode>() {
+                    @Override public boolean apply(ClusterNode node) {
+                        String gridName = node.attribute(IgniteNodeAttributes.ATTR_GRID_NAME);
+
+                        assert gridName != null;
+
+                        return gridName.startsWith(getTestGridName());
+                    }
+                }).predicate();
+
+            List<ServiceConfiguration> cfgs = getConfigs(depPred, numServices);
+
+            subscribeExeLatch(cfgs, latch);
+
+            IgniteServices services = client.services().withAsync();
+
+            int from = 0;
+
+            while (from < numServices) {
+                int to = Math.min(numServices, from + batchSize);
+
+                services.deployAll(cfgs.subList(from, to), false);
+
+                services.future().get(5000);
+
+                from = to;
+            }
+
+            assertTrue(latch.await(30, TimeUnit.SECONDS));
+
+            assertDeployedServices(client, cfgs);
+        }
+        finally {
+            finished.set(true);
+        }
+
+        topChangeFut.get();
+    }
+
+    /**
+     * TODO: enable when IGNITE-6259 is fixed
+     *
+     * @throws Exception If failed.
+     */
+    public void _testDeployAllTopologyChangeFail() throws Exception {
+        final Ignite client = grid(CLIENT_NODE_NAME);
+
+        final AtomicBoolean finished = new AtomicBoolean();
+
+        IgniteInternalFuture<Object> topChangeFut = runTopChanger(finished);
+
+        try {
+            int numServices = 500;
+            int batchSize = 5;
+
+            CountDownLatch latch = new CountDownLatch(numServices);
+
+            IgnitePredicate<ClusterNode> depPred = client.cluster().forServers()
+                .forPredicate(new IgnitePredicate<ClusterNode>() {
+                    @Override public boolean apply(ClusterNode node) {
+                        String gridName = node.attribute(IgniteNodeAttributes.ATTR_GRID_NAME);
+
+                        assert gridName != null;
+
+                        return gridName.startsWith(getTestGridName());
+                    }
+                }).predicate();
+
+            List<ServiceConfiguration> cfgs = getConfigs(depPred, numServices);
+
+            List<ServiceConfiguration> failingCfgs = new ArrayList<>();
+
+            subscribeExeLatch(cfgs, latch);
+
+            IgniteServices services = client.services().withAsync();
+
+            int from = 0;
+
+            while (from < numServices) {
+                int to = Math.min(numServices, from + batchSize);
+
+                List<ServiceConfiguration> cfgsBatch = cfgs.subList(from, to);
+
+                ServiceConfiguration failingCfg = cfgsBatch.get(0);
+
+                failingCfg.setName(null);
+
+                failingCfgs.add(failingCfg);
+
+                try {
+                    services.deployAll(cfgsBatch, false);
+
+                    services.future().get(5000);
+
+                    fail("Should never reach here.");
+                }
+                catch (ServiceDeploymentException e) {
+                    assertEquals(1, e.getFailedConfigurations().size());
+
+                    ServiceConfiguration actFailedCfg = copyService(e.getFailedConfigurations().iterator().next());
+
+                    assertEquals(failingCfg, actFailedCfg);
+
+                    latch.countDown();
+                }
+
+                from = to;
+            }
+
+            assertTrue(latch.await(30, TimeUnit.SECONDS));
+
+            cfgs.removeAll(failingCfgs);
+
+            assertDeployedServices(client, cfgs);
+        }
+        finally {
+            finished.set(true);
+        }
+
+        topChangeFut.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAllFailAll() throws Exception {
+        deployAllFail(false, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAllPartial() throws Exception {
+        deployAllFail(false, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAllAsyncFailAll() throws Exception {
+        deployAllFail(true, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAllAsyncFailPartial() throws Exception {
+        deployAllFail(true, false);
+    }
+
+    /**
+     * @param async If {@code true}, then asynchronous method of deployment will be performed.
+     * @param allOrNone Failing strategy.
+     * @throws Exception If failed.
+     */
+    private void deployAllFail(boolean async, boolean allOrNone) throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES - 1);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        ServiceConfiguration failingCfg = cfgs.get(cfgs.size() - 1);
+
+        failingCfg.setName(null);
+
+        assertFailingDeploy(client, async, allOrNone, cfgs, failingCfg);
+
+        if (allOrNone) {
+            assertFalse("Some of the services were deployed.", latch.await(2, TimeUnit.SECONDS));
+
+            assertEquals(NUM_SERVICES - 1, latch.getCount());
+
+            assertTrue(client.services().serviceDescriptors().isEmpty());
+        }
+        else {
+            assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+            assertDeployedServices(client, cfgs.subList(0, cfgs.size() - 1));
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClashingNames() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        List<ServiceConfiguration> fstBatch = cfgs.subList(0, NUM_SERVICES / 2);
+        List<ServiceConfiguration> sndBatch = cfgs.subList(NUM_SERVICES / 4, NUM_SERVICES);
+
+        IgniteServices svcs1 = client.services().withAsync();
+        IgniteServices svcs2 = client.services().withAsync();
+
+        svcs1.deployAll(fstBatch, false);
+        svcs2.deployAll(sndBatch, false);
+
+        svcs1.future().get();
+        svcs2.future().get();
+
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+        assertDeployedServices(client, cfgs);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClashingNamesFailAll() throws Exception {
+        clashingNamesFail(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClashingNamesPartial() throws Exception {
+        clashingNamesFail(false);
+    }
+
+    /**
+     * @param allOrNone Failing strategy.
+     * @throws Exception If failed.
+     */
+    private void clashingNamesFail(boolean allOrNone) throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        int numDepSvcs;
+
+        if (allOrNone)
+            numDepSvcs = NUM_SERVICES / 2;
+        else
+            numDepSvcs = NUM_SERVICES - 1;
+
+        CountDownLatch latch = new CountDownLatch(numDepSvcs);
+
+        List<ServiceConfiguration> fstBatch = cfgs.subList(0, NUM_SERVICES / 2);
+        List<ServiceConfiguration> sndBatch = cfgs.subList(NUM_SERVICES / 4, NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        IgniteServices services = client.services().withAsync();
+
+        services.deployAll(fstBatch, false);
+
+        ServiceConfiguration failingCfg = cfgs.get(NUM_SERVICES - 1);
+
+        failingCfg.setName(null);
+
+        assertFailingDeploy(client, false, allOrNone, sndBatch, failingCfg);
+
+        services.future().get();
+
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+        assertDeployedServices(client, cfgs.subList(0, numDepSvcs));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClashingNameDifferentConfigFailAll() throws Exception {
+        testClashingNameDifferentConfig(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClashingNameDifferentConfigPartial() throws Exception {
+        testClashingNameDifferentConfig(false);
+    }
+
+    /**
+     * @param allOrNone Failing strategy.
+     * @throws Exception If failed.
+     */
+    private void testClashingNameDifferentConfig(boolean allOrNone) throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        int numDepSvcs;
+
+        if (allOrNone)
+            numDepSvcs = NUM_SERVICES / 2;
+        else
+            numDepSvcs = NUM_SERVICES - 1;
+
+
+        CountDownLatch latch = new CountDownLatch(numDepSvcs);
+
+        List<ServiceConfiguration> fstBatch = cfgs.subList(0, NUM_SERVICES / 2);
+        List<ServiceConfiguration> sndBatch = cfgs.subList(NUM_SERVICES / 4, NUM_SERVICES - 1);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(fstBatch, false);
+
+        ServiceConfiguration failingCfg = copyService(cfgs.get(NUM_SERVICES - 1));
+
+        // Same name, different config.
+        failingCfg.setName(fstBatch.get(0).getName());
+        failingCfg.setTotalCount(fstBatch.get(0).getTotalCount() + 1);
+
+        sndBatch.add(failingCfg);
+
+        assertFailingDeploy(client, false, allOrNone, sndBatch, failingCfg);
+
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+        assertDeployedServices(client, cfgs.subList(0, numDepSvcs));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCancelAll() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(cfgs, true);
+
+        latch.await(30, TimeUnit.SECONDS);
+
+        client.services().cancelAll();
+
+        assertDeployedServices(client, Collections.<ServiceConfiguration>emptyList());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCancelAllAsync() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(cfgs, true);
+
+        latch.await(30, TimeUnit.SECONDS);
+
+        IgniteServices services = client.services().withAsync();
+
+        services.cancelAll();
+
+        services.future().get();
+
+        assertDeployedServices(client, Collections.<ServiceConfiguration>emptyList());
+    }
+
+    /**
+     * TODO: enable when IGNITE-6259 is fixed
+     *
+     * @throws Exception If failed.
+     */
+    public void _testCancelAllTopologyChange() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        int numServices = 500;
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), numServices);
+
+        CountDownLatch latch = new CountDownLatch(numServices);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(cfgs, true);
+
+        latch.await(30, TimeUnit.SECONDS);
+
+        final AtomicBoolean finished = new AtomicBoolean();
+
+        IgniteInternalFuture<Object> topChangeFut = runTopChanger(finished);
+
+        List<String> names = new ArrayList<>();
+
+        for (ServiceConfiguration cfg : cfgs)
+            names.add(cfg.getName());
+
+        try {
+            IgniteServices services = client.services().withAsync();
+
+            int batchSize = 5;
+            int from = 0;
+
+            while (from < numServices) {
+                int to = Math.min(numServices, from + batchSize);
+
+                log.info("Trying to cancel services [" + from + ".." + to + ")");
+
+                services.cancelAll(names.subList(from, to));
+
+                services.future().get(5000);
+
+                from = to;
+            }
+
+            assertDeployedServices(client, Collections.<ServiceConfiguration>emptyList());
+        }
+        finally {
+            finished.set(true);
+        }
+
+        topChangeFut.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCancelAllClashingNames() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(cfgs, true);
+
+        latch.await(30, TimeUnit.SECONDS);
+
+        List<String> names = new ArrayList<>();
+
+        for (ServiceConfiguration cfg : cfgs)
+            names.add(cfg.getName());
+
+        int batchSize = 5;
+        int from = 0;
+
+        while (from < NUM_SERVICES) {
+            int to = Math.min(NUM_SERVICES, from + batchSize);
+
+            List<String> toCancel = new ArrayList<>(names.subList(from, to));
+
+            toCancel.add(toCancel.get(0));
+
+            client.services().cancelAll(toCancel);
+
+            from = to;
+        }
+
+        assertDeployedServices(client, Collections.<ServiceConfiguration>emptyList());
+    }
+
+    /**
+     * @param client Client.
+     * @param async If {@code true}, then async version of deploy method will be used.
+     * @param cfgs Service configurations.
+     * @param allOrNone Failing policy.
+     * @param failingCfg Configuration of the failing service.
+     * @throws Exception If failed.
+     */
+    private void assertFailingDeploy(Ignite client, boolean async, boolean allOrNone, List<ServiceConfiguration> cfgs,
+        ServiceConfiguration failingCfg) throws Exception {
+
+        IgniteServices services = client.services();
+
+        if (async) {
+            services = services.withAsync();
+
+            services.deployAll(cfgs, allOrNone);
+        }
+
+        try {
+            if (async)
+                services.future().get();
+            else
+                services.deployAll(cfgs, allOrNone);
+
+            fail("Should never reach here.");
+        }
+        catch (ServiceDeploymentException e) {
+            info("Expected exception: " + e.getMessage());
+
+            Collection<ServiceConfiguration> expFails;
+
+            if (allOrNone)
+                expFails = cfgs;
+            else
+                expFails = Collections.singleton(failingCfg);
+
+            Collection<ServiceConfiguration> actFails = e.getFailedConfigurations();
+
+            // Some cfgs may be lazy. Construct ServiceConfiguration from them for comparison.
+            Collection<ServiceConfiguration> actFailsCp = new ArrayList<>(actFails.size());
+
+            for (ServiceConfiguration cfg : actFails)
+                actFailsCp.add(copyService(cfg));
+
+            assertEqualsCollections(expFails, actFailsCp);
+        }
+    }
+
+    /**
+     * @param cfg Config.
+     * @return Copy of provided configuration.
+     */
+    private ServiceConfiguration copyService(ServiceConfiguration cfg) {
+        ServiceConfiguration cfgCp = new ServiceConfiguration();
+
+        cfgCp.setName(cfg.getName());
+
+        cfgCp.setMaxPerNodeCount(cfg.getMaxPerNodeCount());
+
+        cfgCp.setTotalCount(cfg.getTotalCount());
+
+        cfgCp.setAffinityKey(cfg.getAffinityKey());
+
+        cfgCp.setCacheName(cfg.getCacheName());
+
+        cfgCp.setName(cfg.getName());
+
+        cfgCp.setService(cfg.getService());
+
+        cfgCp.setNodeFilter(cfg.getNodeFilter());
+
+        return cfgCp;
+    }
+
+    /**
+     * @param client Client Ignite instance.
+     * @param expCfgs Configurations of services that are expected to be deployed.
+     */
+    private void assertDeployedServices(Ignite client, Collection<ServiceConfiguration> expCfgs) {
+        Set<String> expNames = new HashSet<>();
+        Set<String> actNames = new HashSet<>();
+
+        for (ServiceConfiguration cfg : expCfgs)
+            expNames.add(cfg.getName());
+
+        for (ServiceDescriptor desc : client.services().serviceDescriptors())
+            actNames.add(desc.name());
+
+        assertEquals(expNames, actNames);
+    }
+
+    /**
+     * @param nodePred Node predicate.
+     * @param numServices Number of configurations to generate.
+     * @return Generated services configurations.
+     */
+    private List<ServiceConfiguration> getConfigs(IgnitePredicate<ClusterNode> nodePred, int numServices) {
+        List<ServiceConfiguration> cfgs = new ArrayList<>(numServices);
+
+        for (int i = 0; i < numServices; i++) {
+            String name = "testService-" + i;
+
+            ServiceConfiguration cfg = new ServiceConfiguration();
+
+            cfg.setName(name);
+            cfg.setTotalCount(1);
+            cfg.setMaxPerNodeCount(1);
+            cfg.setService(new DummyService());
+            cfg.setNodeFilter(nodePred);
+
+            cfgs.add(cfg);
+        }
+        return cfgs;
+    }
+
+    /**
+     * @param cfgs Configurations.
+     * @param latch Latch.
+     */
+    private void subscribeExeLatch(List<ServiceConfiguration> cfgs, CountDownLatch latch) {
+        for (ServiceConfiguration cfg : cfgs)
+            DummyService.exeLatch(cfg.getName(), latch);
+    }
+
+    /**
+     * @param finished Finished flag.
+     * @throws Exception If failed.
+     * @return Future.
+     */
+    private IgniteInternalFuture<Object> runTopChanger(final AtomicBoolean finished) throws Exception {
+        return runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                String namePrefix = "extra-node-";
+
+                int extraNodesNum = 3;
+
+                while (!finished.get()) {
+                    for (int i = 0; i < extraNodesNum; i++)
+                        startGrid(namePrefix + i);
+
+                    for (int i = 0; i < extraNodesNum; i++)
+                        stopGrid(namePrefix + i);
+
+                    awaitPartitionMapExchange();
+                }
+
+                return null;
+            }
+        });
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 43bd3a44c6388..8ff393c9eea32 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -50,7 +50,9 @@
 import org.apache.ignite.internal.processors.port.GridPortProcessorSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceClientNodeTest;
 import org.apache.ignite.internal.processors.service.GridServiceContinuousQueryRedeploy;
+import org.apache.ignite.internal.processors.service.GridServiceDeploymentCompoundFutureSelfTest;
 import org.apache.ignite.internal.processors.service.GridServicePackagePrivateSelfTest;
+import org.apache.ignite.internal.processors.service.GridServiceProcessorBatchDeploySelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceProcessorMultiNodeConfigSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceProcessorMultiNodeSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceProcessorProxySelfTest;
@@ -146,6 +148,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(IgniteServiceDynamicCachesSelfTest.class);
         suite.addTestSuite(GridServiceContinuousQueryRedeploy.class);
         suite.addTestSuite(ServiceThreadPoolSelfTest.class);
+        suite.addTestSuite(GridServiceProcessorBatchDeploySelfTest.class);
+        suite.addTestSuite(GridServiceDeploymentCompoundFutureSelfTest.class);
 
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class);

From ec4cd64a894f85f20d4ff8d24b143a0da61d1fc5 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 5 Sep 2017 17:03:57 +0300
Subject: [PATCH 407/446] ignite-6273 Moved ServiceDeploymentException to
 org.apache.ignite.services

(cherry picked from commit d7e215d)
---
 .../core/src/main/java/org/apache/ignite/IgniteServices.java   | 2 +-
 .../service/GridServiceDeploymentCompoundFuture.java           | 1 +
 .../internal/processors/service/GridServiceProcessor.java      | 1 +
 .../service => services}/ServiceDeploymentException.java       | 3 +--
 .../service/GridServiceDeploymentCompoundFutureSelfTest.java   | 1 +
 .../service/GridServiceProcessorBatchDeploySelfTest.java       | 1 +
 6 files changed, 6 insertions(+), 3 deletions(-)
 rename modules/core/src/main/java/org/apache/ignite/{internal/processors/service => services}/ServiceDeploymentException.java (95%)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
index ae3958fc269cf..369baebfb6237 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
@@ -20,7 +20,7 @@
 import java.util.Collection;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.processors.service.ServiceDeploymentException;
+import org.apache.ignite.services.ServiceDeploymentException;
 import org.apache.ignite.lang.IgniteAsyncSupport;
 import org.apache.ignite.lang.IgniteAsyncSupported;
 import org.apache.ignite.resources.IgniteInstanceResource;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
index 12b88e5f119f5..bbf03702a226f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceDeploymentException;
 import org.jetbrains.annotations.Nullable;
 
 /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index f457d8091a4ac..2a30bad8d4950 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -100,6 +100,7 @@
 import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceDeploymentException;
 import org.apache.ignite.services.ServiceDescriptor;
 import org.apache.ignite.spi.IgniteNodeValidationResult;
 import org.apache.ignite.thread.IgniteThreadFactory;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java b/modules/core/src/main/java/org/apache/ignite/services/ServiceDeploymentException.java
similarity index 95%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java
rename to modules/core/src/main/java/org/apache/ignite/services/ServiceDeploymentException.java
index 32fbf6f254250..843a8fe37c59a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java
+++ b/modules/core/src/main/java/org/apache/ignite/services/ServiceDeploymentException.java
@@ -15,11 +15,10 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.service;
+package org.apache.ignite.services;
 
 import java.util.Collection;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.services.ServiceConfiguration;
 import org.jetbrains.annotations.Nullable;
 
 /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
index 51c3407e65da1..ca95198cf2fc9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceDeploymentException;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /** */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
index c3232f23e9c2f..337e69565968b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceDeploymentException;
 import org.apache.ignite.services.ServiceDescriptor;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;

From 1831f41d80497012b9bfff38d62099734a3af245 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Wed, 6 Sep 2017 13:27:25 +0300
Subject: [PATCH 408/446] IGNITE-6271 .NET: Propagate
 ServiceDeploymentException

(cherry picked from commit 56860d2)
---
 .../services/PlatformAbstractService.java     |   8 ++
 .../platform/services/PlatformServices.java   |  63 ++++++++--
 .../platform/utils/PlatformUtils.java         |   2 +-
 .../Services/ServicesAsyncWrapper.cs          |   9 +-
 .../Services/ServicesTest.cs                  |  80 +++++++++++--
 .../Apache.Ignite.Core.csproj                 |   1 +
 .../Apache.Ignite.Core/Impl/ExceptionUtils.cs |   6 +-
 .../Impl/Services/ServiceProxySerializer.cs   |  36 ++++++
 .../Impl/Services/Services.cs                 |  29 ++++-
 .../Impl/Unmanaged/UnmanagedCallbacks.cs      |  37 +++++-
 .../Services/ServiceDeploymentException.cs    | 111 ++++++++++++++++++
 .../dotnet/Apache.Ignite.sln.DotSettings      |   1 +
 12 files changed, 350 insertions(+), 33 deletions(-)
 create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Services/ServiceDeploymentException.cs

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformAbstractService.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformAbstractService.java
index d6a6e16f0f2c0..fc0fa14ebc5dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformAbstractService.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformAbstractService.java
@@ -94,6 +94,14 @@ public PlatformAbstractService(Object svc, PlatformContext ctx, boolean srvKeepB
             out.synchronize();
 
             ptr = platformCtx.gateway().serviceInit(mem.pointer());
+
+            PlatformInputStream in = mem.input();
+
+            in.synchronize();
+
+            BinaryRawReaderEx reader = platformCtx.reader(in);
+
+            PlatformUtils.readInvocationResult(platformCtx, reader);
         }
         catch (IgniteCheckedException e) {
             throw U.convertException(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
index c266986eb6e76..4259220752647 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.processors.platform.PlatformContext;
 import org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetService;
 import org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetServiceImpl;
+import org.apache.ignite.internal.processors.platform.utils.PlatformFutureUtils;
 import org.apache.ignite.internal.processors.platform.utils.PlatformUtils;
 import org.apache.ignite.internal.processors.platform.utils.PlatformWriterBiClosure;
 import org.apache.ignite.internal.processors.platform.utils.PlatformWriterClosure;
@@ -37,6 +38,7 @@
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.services.ServiceDescriptor;
+import org.jetbrains.annotations.NotNull;
 
 import java.lang.reflect.Method;
 import java.util.ArrayList;
@@ -103,6 +105,9 @@ public class PlatformServices extends PlatformAbstractTarget {
     private static final CopyOnWriteConcurrentMap<T3<Class, String, Integer>, Method> SVC_METHODS
         = new CopyOnWriteConcurrentMap<>();
 
+    /** Future result writer. */
+    private static final PlatformFutureUtils.Writer RESULT_WRITER = new ServiceDeploymentResultWriter();
+
     /** */
     private final IgniteServices services;
 
@@ -147,20 +152,10 @@ private ServiceDescriptor findDescriptor(String name) {
     @Override protected long processInStreamOutLong(int type, BinaryRawReaderEx reader)
         throws IgniteCheckedException {
         switch (type) {
-            case OP_DOTNET_DEPLOY: {
-                dotnetDeploy(reader, services);
-
-                return TRUE;
-            }
-
             case OP_DOTNET_DEPLOY_ASYNC: {
                 dotnetDeploy(reader, servicesAsync);
 
-                return readAndListenFuture(reader);
-            }
-
-            case OP_DOTNET_DEPLOY_MULTIPLE: {
-                dotnetDeployMultiple(reader, services);
+                readAndListenFuture(reader, currentFuture(), RESULT_WRITER);
 
                 return TRUE;
             }
@@ -168,7 +163,9 @@ private ServiceDescriptor findDescriptor(String name) {
             case OP_DOTNET_DEPLOY_MULTIPLE_ASYNC: {
                 dotnetDeployMultiple(reader, servicesAsync);
 
-                return readAndListenFuture(reader);
+                readAndListenFuture(reader, currentFuture(), RESULT_WRITER);
+
+                return TRUE;
             }
 
             case OP_CANCEL: {
@@ -217,6 +214,32 @@ private ServiceDescriptor findDescriptor(String name) {
                 return;
             }
 
+            case OP_DOTNET_DEPLOY: {
+                try {
+                    dotnetDeploy(reader, services);
+
+                    PlatformUtils.writeInvocationResult(writer, null, null);
+                }
+                catch (Exception e) {
+                    PlatformUtils.writeInvocationResult(writer, null, e);
+                }
+
+                return;
+            }
+
+            case OP_DOTNET_DEPLOY_MULTIPLE: {
+                try {
+                    dotnetDeployMultiple(reader, services);
+
+                    PlatformUtils.writeInvocationResult(writer, null, null);
+                }
+                catch (Exception e) {
+                    PlatformUtils.writeInvocationResult(writer, null, e);
+                }
+
+                return;
+            }
+
             default:
                 super.processInStreamOutStream(type, reader, writer);
         }
@@ -579,4 +602,20 @@ public void put(K key, V val) {
             }
         }
     }
+
+    /**
+     * Writes an EventBase.
+     */
+    private static class ServiceDeploymentResultWriter implements PlatformFutureUtils.Writer {
+        /** <inheritDoc /> */
+        @Override public void write(BinaryRawWriterEx writer, Object obj, Throwable err) {
+            PlatformUtils.writeInvocationResult(writer, obj, err);
+        }
+
+        /** <inheritDoc /> */
+        @Override public boolean canWrite(Object obj, Throwable err) {
+            return true;
+        }
+    }
+
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
index 959ff68a4e19f..b1f3cb0ef8d0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
@@ -765,7 +765,7 @@ public static byte[] errorData(Throwable err) {
      * @param resObj Result.
      * @param err Error.
      */
-    public static void writeInvocationResult(BinaryRawWriterEx writer, Object resObj, Exception err)
+    public static void writeInvocationResult(BinaryRawWriterEx writer, Object resObj, Throwable err)
     {
         if (err == null) {
             writer.writeBoolean(true);
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesAsyncWrapper.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesAsyncWrapper.cs
index f0740e04761cf..470804c018292 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesAsyncWrapper.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesAsyncWrapper.cs
@@ -104,7 +104,14 @@ public Task DeployMultipleAsync(string name, IService service, int totalCount, i
         /** <inheritDoc /> */
         public void Deploy(ServiceConfiguration configuration)
         {
-            _services.DeployAsync(configuration).Wait();
+            try
+            {
+                _services.DeployAsync(configuration).Wait();
+            }
+            catch (AggregateException ex)
+            {
+                throw ex.InnerException;
+            }
         }
 
         /** <inheritDoc /> */
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesTest.cs
index 0558d1120496e..26650863c0cd8 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesTest.cs
@@ -431,17 +431,71 @@ public void TestWithKeepBinaryBoth()
         /// Tests exception in Initialize.
         /// </summary>
         [Test]
-        public void TestInitException()
+        public void TestDeployMultipleException([Values(true, false)] bool keepBinary)
+        {
+            VerifyDeploymentException((services, svc) =>
+                services.DeployMultiple(SvcName, svc, Grids.Length, 1), keepBinary);
+        }
+
+        /// <summary>
+        /// Tests exception in Initialize.
+        /// </summary>
+        [Test]
+        public void TestDeployException([Values(true, false)] bool keepBinary)
+        {
+            VerifyDeploymentException((services, svc) =>
+                services.Deploy(new ServiceConfiguration
+                {
+                    Name = SvcName,
+                    Service = svc,
+                    TotalCount = Grids.Length,
+                    MaxPerNodeCount = 1
+                }), keepBinary);
+        }
+
+        /// <summary>
+        /// Verifies the deployment exception.
+        /// </summary>
+        private void VerifyDeploymentException(Action<IServices, IService> deploy, bool keepBinary)
         {
             var svc = new TestIgniteServiceSerializable { ThrowInit = true };
 
-            var ex = Assert.Throws<IgniteException>(() => Services.DeployMultiple(SvcName, svc, Grids.Length, 1));
+            var services = Services;
+
+            if (keepBinary)
+            {
+                services = services.WithKeepBinary();
+            }
+
+            var deploymentException = Assert.Throws<ServiceDeploymentException>(() => deploy(services, svc));
+
+            var text = keepBinary 
+                ? "Service deployment failed with a binary error. Examine BinaryCause for details."
+                : "Service deployment failed with an exception. Examine InnerException for details.";
+
+            Assert.AreEqual(text, deploymentException.Message);
+
+            Exception ex;
+
+            if (keepBinary)
+            {
+                Assert.IsNull(deploymentException.InnerException);
+
+                ex = deploymentException.BinaryCause.Deserialize<Exception>();
+            }
+            else
+            {
+                Assert.IsNull(deploymentException.BinaryCause);
+
+                ex = deploymentException.InnerException;
+            }
+
+            Assert.IsNotNull(ex);
             Assert.AreEqual("Expected exception", ex.Message);
-            Assert.IsNotNull(ex.InnerException);
-            Assert.IsTrue(ex.InnerException.Message.Contains("PlatformCallbackUtils.serviceInit(Native Method)"));
+            Assert.IsTrue(ex.StackTrace.Trim().StartsWith(
+                "at Apache.Ignite.Core.Tests.Services.ServicesTest.TestIgniteServiceSerializable.Init"));
 
             var svc0 = Services.GetService<TestIgniteServiceSerializable>(SvcName);
-
             Assert.IsNull(svc0);
         }
 
@@ -480,19 +534,27 @@ public void TestCancelException()
             AssertNoService();
         }
 
+        /// <summary>
+        /// Tests exception in binarizable implementation.
+        /// </summary>
         [Test]
         public void TestMarshalExceptionOnRead()
         {
             var svc = new TestIgniteServiceBinarizableErr();
 
-            var ex = Assert.Throws<IgniteException>(() => Services.DeployMultiple(SvcName, svc, Grids.Length, 1));
-            Assert.AreEqual("Expected exception", ex.Message);
+            var ex = Assert.Throws<ServiceDeploymentException>(() =>
+                Services.DeployMultiple(SvcName, svc, Grids.Length, 1));
+            
+            Assert.AreEqual("Expected exception", ex.InnerException.Message);
 
             var svc0 = Services.GetService<TestIgniteServiceSerializable>(SvcName);
 
             Assert.IsNull(svc0);
         }
 
+        /// <summary>
+        /// Tests exception in binarizable implementation.
+        /// </summary>
         [Test]
         public void TestMarshalExceptionOnWrite()
         {
@@ -506,6 +568,9 @@ public void TestMarshalExceptionOnWrite()
             Assert.IsNull(svc0);
         }
 
+        /// <summary>
+        /// Tests Java service invocation.
+        /// </summary>
         [Test]
         public void TestCallJavaService()
         {
@@ -777,6 +842,7 @@ private class TestIgniteServiceSerializable : IService, ITestIgniteService
             /** <inheritdoc /> */
             public Guid NodeId
             {
+                // ReSharper disable once InconsistentlySynchronizedField
                 get { return _grid.GetCluster().GetLocalNode().Id; }
             }
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
index f945efea27d3c..4f9d41e675b08 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
@@ -473,6 +473,7 @@
     <Compile Include="Services\IServices.cs" />
     <Compile Include="Services\Package-Info.cs" />
     <Compile Include="Services\ServiceConfiguration.cs" />
+    <Compile Include="Services\ServiceDeploymentException.cs" />
     <Compile Include="Services\ServiceInvocationException.cs" />
     <Compile Include="SwapSpace\File\FileSwapSpaceSpi.cs" />
     <Compile Include="SwapSpace\ISwapSpaceSpi.cs" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs
index ddbdd86fc559b..e4b2b062e7883 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs
@@ -30,6 +30,7 @@ namespace Apache.Ignite.Core.Impl
     using Apache.Ignite.Core.Common;
     using Apache.Ignite.Core.Compute;
     using Apache.Ignite.Core.Impl.Binary;
+    using Apache.Ignite.Core.Services;
     using Apache.Ignite.Core.Transactions;
 
     /// <summary>
@@ -103,9 +104,12 @@ static ExceptionUtils()
             Exs["org.apache.ignite.IgniteAuthenticationException"] = (i, m, e) => new SecurityException(m, e);
             Exs["org.apache.ignite.plugin.security.GridSecurityException"] = (i, m, e) => new SecurityException(m, e);
 
-            // Future exceptions
+            // Future exceptions.
             Exs["org.apache.ignite.lang.IgniteFutureCancelledException"] = (i, m, e) => new IgniteFutureCancelledException(m, e);
             Exs["org.apache.ignite.internal.IgniteFutureCancelledCheckedException"] = (i, m, e) => new IgniteFutureCancelledException(m, e);
+
+            // Service exceptions.
+            Exs["org.apache.ignite.services.ServiceDeploymentException"] = (i, m, e) => new ServiceDeploymentException(m, e);
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/ServiceProxySerializer.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/ServiceProxySerializer.cs
index 8e44360ff7b65..bd15a7cabaef0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/ServiceProxySerializer.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/ServiceProxySerializer.cs
@@ -154,6 +154,42 @@ public static object ReadInvocationResult(IBinaryStream stream, Marshaller marsh
                                                  "Examine InnerException for details.", (Exception) err);
         }
 
+        /// <summary>
+        /// Reads service deployment result.
+        /// </summary>
+        /// <param name="stream">Stream.</param>
+        /// <param name="marsh">Marshaller.</param>
+        /// <param name="keepBinary">Binary flag.</param>
+        /// <returns>
+        /// Method invocation result, or exception in case of error.
+        /// </returns>
+        public static void ReadDeploymentResult(IBinaryStream stream, Marshaller marsh, bool keepBinary)
+        {
+            Debug.Assert(stream != null);
+            Debug.Assert(marsh != null);
+
+            var mode = keepBinary ? BinaryMode.ForceBinary : BinaryMode.Deserialize;
+
+            var reader = marsh.StartUnmarshal(stream, mode);
+
+            object err;
+
+            BinaryUtils.ReadInvocationResult(reader, out err);
+
+            if (err == null)
+            {
+                return;
+            }
+
+            var binErr = err as IBinaryObject;
+
+            throw binErr != null
+                ? new ServiceDeploymentException("Service deployment failed with a binary error. " +
+                                                 "Examine BinaryCause for details.", binErr)
+                : new ServiceDeploymentException("Service deployment failed with an exception. " +
+                                                 "Examine InnerException for details.", (Exception) err);
+        }
+
         /// <summary>
         /// Writes the argument in platform-compatible format.
         /// </summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs
index 88d2a76f4dbf9..2f309313f7f9e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs
@@ -25,6 +25,7 @@ namespace Apache.Ignite.Core.Impl.Services
     using Apache.Ignite.Core.Binary;
     using Apache.Ignite.Core.Cluster;
     using Apache.Ignite.Core.Impl.Binary;
+    using Apache.Ignite.Core.Impl.Binary.IO;
     using Apache.Ignite.Core.Impl.Common;
     using Apache.Ignite.Core.Impl.Unmanaged;
     using Apache.Ignite.Core.Services;
@@ -204,13 +205,13 @@ public void DeployMultiple(string name, IService service, int totalCount, int ma
             IgniteArgumentCheck.NotNullOrEmpty(name, "name");
             IgniteArgumentCheck.NotNull(service, "service");
 
-            DoOutOp(OpDeployMultiple, w =>
+            DoOutInOp(OpDeployMultiple, w =>
             {
                 w.WriteString(name);
                 w.WriteObject(service);
                 w.WriteInt(totalCount);
                 w.WriteInt(maxPerNodeCount);
-            });
+            }, s => ReadDeploymentResult(s));
         }
 
         /** <inheritDoc /> */
@@ -225,7 +226,7 @@ public Task DeployMultipleAsync(string name, IService service, int totalCount, i
                 w.WriteObject(service);
                 w.WriteInt(totalCount);
                 w.WriteInt(maxPerNodeCount);
-            });
+            }, _keepBinary, s => ReadDeploymentResult(s));
         }
 
         /** <inheritDoc /> */
@@ -233,7 +234,7 @@ public void Deploy(ServiceConfiguration configuration)
         {
             IgniteArgumentCheck.NotNull(configuration, "configuration");
 
-            DoOutOp(OpDeploy, w => WriteServiceConfiguration(configuration, w));
+            DoOutInOp(OpDeploy, w => WriteServiceConfiguration(configuration, w), s => ReadDeploymentResult(s));
         }
 
         /** <inheritDoc /> */
@@ -241,7 +242,8 @@ public Task DeployAsync(ServiceConfiguration configuration)
         {
             IgniteArgumentCheck.NotNull(configuration, "configuration");
 
-            return DoOutOpAsync(OpDeployAsync, w => WriteServiceConfiguration(configuration, w));
+            return DoOutOpAsync(OpDeployAsync, w => WriteServiceConfiguration(configuration, w), 
+                _keepBinary, ReadDeploymentResult);
         }
 
         /** <inheritDoc /> */
@@ -400,5 +402,22 @@ private static void WriteServiceConfiguration(ServiceConfiguration configuration
             else
                 w.WriteObject<object>(null);
         }
+
+        /// <summary>
+        /// Reads the deployment result.
+        /// </summary>
+        private object ReadDeploymentResult(BinaryReader r)
+        {
+            return r != null ? ReadDeploymentResult(r.Stream) : null;
+        }
+
+        /// <summary>
+        /// Reads the deployment result.
+        /// </summary>
+        private object ReadDeploymentResult(IBinaryStream s)
+        {
+            ServiceProxySerializer.ReadDeploymentResult(s, Marshaller, _keepBinary);
+            return null;
+        }
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs
index cc205e829d476..cd642ad1a27f6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs
@@ -942,22 +942,47 @@ private void EventFilterDestroy(void* target, long ptr)
 
         #region IMPLEMENTATION: SERVICES
 
+        [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes",
+            Justification = "User processor can throw any exception")]
         private long ServiceInit(void* target, long memPtr)
         {
             return SafeCall(() =>
             {
                 using (var stream = IgniteManager.Memory.Get(memPtr).GetStream())
                 {
-                    var reader = _ignite.Marshaller.StartUnmarshal(stream);
+                    try
+                    {
+                        var reader = _ignite.Marshaller.StartUnmarshal(stream);
 
-                    bool srvKeepBinary = reader.ReadBoolean();
-                    var svc = reader.ReadObject<IService>();
+                        var srvKeepBinary = reader.ReadBoolean();
+                        var svc = reader.ReadObject<IService>();
 
-                    ResourceProcessor.Inject(svc, _ignite);
+                        ResourceProcessor.Inject(svc, _ignite);
 
-                    svc.Init(new ServiceContext(_ignite.Marshaller.StartUnmarshal(stream, srvKeepBinary)));
+                        svc.Init(new ServiceContext(_ignite.Marshaller.StartUnmarshal(stream, srvKeepBinary)));
 
-                    return _handleRegistry.Allocate(svc);
+                        stream.Reset();
+
+                        stream.WriteBool(true); // Success.
+
+                        stream.SynchronizeOutput();
+
+                        return _handleRegistry.Allocate(svc);
+                    }
+                    catch (Exception e)
+                    {
+                        stream.Reset();
+
+                        var writer = _ignite.Marshaller.StartMarshal(stream);
+
+                        BinaryUtils.WriteInvocationResult(writer, false, e);
+
+                        _ignite.Marshaller.FinishMarshal(writer);
+
+                        stream.SynchronizeOutput();
+
+                        return 0;
+                    }
                 }
             });
         }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Services/ServiceDeploymentException.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Services/ServiceDeploymentException.cs
new file mode 100644
index 0000000000000..825f91e5b3e5e
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Services/ServiceDeploymentException.cs
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Core.Services
+{
+    using System;
+    using System.Diagnostics.CodeAnalysis;
+    using System.Runtime.Serialization;
+    using Apache.Ignite.Core.Binary;
+    using Apache.Ignite.Core.Common;
+
+    /// <summary>
+    /// Indicates an error during Grid Services deployment.
+    /// </summary>
+    [Serializable]
+    public class ServiceDeploymentException : IgniteException
+    {
+        /** Serializer key. */
+        private const string KeyBinaryCause = "BinaryCause";
+
+        /** Cause. */
+        private readonly IBinaryObject _binaryCause;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ServiceDeploymentException"/> class.
+        /// </summary>
+        public ServiceDeploymentException()
+        {
+            // No-op.
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ServiceDeploymentException"/> class.
+        /// </summary>
+        /// <param name="message">The message that describes the error.</param>
+        public ServiceDeploymentException(string message) : base(message)
+        {
+            // No-op.
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ServiceDeploymentException"/> class.
+        /// </summary>
+        /// <param name="message">The message.</param>
+        /// <param name="cause">The cause.</param>
+        public ServiceDeploymentException(string message, Exception cause) : base(message, cause)
+        {
+            // No-op.
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ServiceDeploymentException"/> class.
+        /// </summary>
+        /// <param name="message">The message.</param>
+        /// <param name="binaryCause">The binary cause.</param>
+        public ServiceDeploymentException(string message, IBinaryObject binaryCause)
+            : base(message)
+        {
+            _binaryCause = binaryCause;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="IgniteException"/> class.
+        /// </summary>
+        /// <param name="info">Serialization information.</param>
+        /// <param name="ctx">Streaming context.</param>
+        protected ServiceDeploymentException(SerializationInfo info, StreamingContext ctx)
+            : base(info, ctx)
+        {
+            _binaryCause = (IBinaryObject)info.GetValue(KeyBinaryCause, typeof(IBinaryObject));
+        }
+
+        /// <summary>
+        /// Gets the binary cause.
+        /// </summary>
+        public IBinaryObject BinaryCause
+        {
+            get { return _binaryCause; }
+        }
+
+        /// <summary>
+        /// When overridden in a derived class, sets the <see cref="SerializationInfo" />
+        /// with information about the exception.
+        /// </summary>
+        /// <param name="info">The <see cref="SerializationInfo" /> that holds the serialized object data
+        /// about the exception being thrown.</param>
+        /// <param name="context">The <see cref="StreamingContext" /> that contains contextual information
+        /// about the source or destination.</param>
+        [SuppressMessage("Microsoft.Design", "CA1062:Validate arguments of public methods")]
+        public override void GetObjectData(SerializationInfo info, StreamingContext context)
+        {
+            info.AddValue(KeyBinaryCause, _binaryCause);
+
+            base.GetObjectData(info, context);
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.sln.DotSettings b/modules/platforms/dotnet/Apache.Ignite.sln.DotSettings
index 078e9fbdb9dbb..9d5b728a9c554 100644
--- a/modules/platforms/dotnet/Apache.Ignite.sln.DotSettings
+++ b/modules/platforms/dotnet/Apache.Ignite.sln.DotSettings
@@ -7,4 +7,5 @@
 	<s:Boolean x:Key="/Default/CodeStyle/CSharpUsing/AddImportsToDeepestScope/@EntryValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeStyle/CSharpUsing/QualifiedUsingAtNestedScope/@EntryValue">True</s:Boolean>
 	<s:String x:Key="/Default/CodeInspection/Highlighting/InspectionSeverities/=ConvertClosureToMethodGroup/@EntryIndexedValue">DO_NOT_SHOW</s:String>
+	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002EXml_002ECodeStyle_002EFormatSettingsUpgrade_002EXmlMoveToCommonFormatterSettingsUpgrade/@EntryIndexedValue">True</s:Boolean>
 </wpf:ResourceDictionary>
\ No newline at end of file

From d3be7960878a302d2b156bba89be440a49b7c4ef Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Thu, 7 Sep 2017 14:47:05 +0300
Subject: [PATCH 409/446] ignite-6289 Remove "allOrNone" flag from
 IgniteServices#deployAll method

---
 .../org/apache/ignite/IgniteServices.java     |  19 +--
 .../ignite/internal/IgniteServicesImpl.java   |   6 +-
 .../GridServiceDeploymentCompoundFuture.java  | 129 ++---------------
 .../service/GridServiceProcessor.java         |  64 +++-----
 .../service/PreparedConfigurations.java       |   8 +-
 ...rviceDeploymentCompoundFutureSelfTest.java | 121 +---------------
 ...idServiceProcessorBatchDeploySelfTest.java | 137 +++++-------------
 7 files changed, 81 insertions(+), 403 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
index 369baebfb6237..edcecd34738ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
@@ -286,24 +286,21 @@ public void deployMultiple(String name, Service svc, int totalCnt, int maxPerNod
 
     /**
      * Deploys multiple services described by provided configurations. Depending on specified parameters, multiple
-     * instances of the same service may be deployed (see {@link ServiceConfiguration}).
-     * Whenever topology changes, Ignite will automatically rebalance
-     * the deployed services within cluster to make sure that each node will end up with
-     * about equal number of deployed instances whenever possible.
+     * instances of the same service may be deployed (see {@link ServiceConfiguration}). Whenever topology changes,
+     * Ignite will automatically rebalance the deployed services within cluster to make sure that each node will end up
+     * with about equal number of deployed instances whenever possible.
      *
-     * If deployment fails, then {@link ServiceDeploymentException} containing a list of failed services will be
-     * thrown. It is guaranteed that all services that were provided to this method and are not present in the list of
-     * failed services are successfully deployed by the moment of the exception being thrown.
+     * If deployment of some of the provided services fails, then {@link ServiceDeploymentException} containing a list
+     * of failed services will be thrown. It is guaranteed that all services that were provided to this method and are
+     * not present in the list of failed services are successfully deployed by the moment of the exception being thrown.
+     * Note that if exception is thrown, then partial deployment may have occurred.
      *
      * @param cfgs {@link Collection} of service configurations to be deployed.
-     * @param allOrNone Specifies behavior in case when errors during deployment occur. If {@code true}, then two
-     * outcomes are possible: either all services will be deployed, or none of them. If {@code false}, then partial
-     * deployments are permitted.
      * @throws ServiceDeploymentException If failed to deploy services.
      * @see IgniteServices#deploy(ServiceConfiguration)
      */
     @IgniteAsyncSupported
-    public void deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) throws ServiceDeploymentException;
+    public void deployAll(Collection<ServiceConfiguration> cfgs) throws ServiceDeploymentException;
 
     /**
      * Cancels service deployment. If a service with specified name was deployed on the grid,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
index a51f3f1ba95fd..58b3a2a96ff1c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
@@ -151,17 +151,17 @@ public IgniteServicesImpl(GridKernalContext ctx, ClusterGroupAdapter prj, boolea
     @Override public void deploy(ServiceConfiguration cfg) {
         A.notNull(cfg, "cfg");
 
-        deployAll(Collections.singleton(cfg), false);
+        deployAll(Collections.singleton(cfg));
     }
 
     /** {@inheritDoc} */
-    @Override public void deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+    @Override public void deployAll(Collection<ServiceConfiguration> cfgs) {
         A.notNull(cfgs, "cfgs");
 
         guard();
 
         try {
-            saveOrGet(ctx.service().deployAll(cfgs, allOrNone));
+            saveOrGet(ctx.service().deployAll(cfgs));
         }
         catch (IgniteCheckedException e) {
             throw U.convertException(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
index bbf03702a226f..45ccc24b9fa86 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
@@ -20,140 +20,51 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.services.ServiceDeploymentException;
 import org.jetbrains.annotations.Nullable;
 
 /**
- * Service deployment compound future, {@code allOrNone} parameter specifies failing policy.
- * <p>
- * If {@code allOrNone} parameter is set to {@code false}, then this future waits for completion of all child futures.
- * If any exceptions are thrown during deployment, then {@link IgniteCheckedException} with {@link
- * ServiceDeploymentException} as a cause will be thrown from {@link IgniteInternalFuture#get get()} method after all
- * futures complete or fail. Inner exception will contain configurations of failed services.
+ * Service deployment compound future. If any exceptions are thrown during deployment, then {@link
+ * IgniteCheckedException} with {@link ServiceDeploymentException} as a cause will be thrown from {@link
+ * IgniteInternalFuture#get get()} method after all futures complete or fail. Inner exception will contain
+ * configurations of failed services.
  */
 public class GridServiceDeploymentCompoundFuture extends GridCompoundFuture<Object, Object> {
-    /** */
-    private final boolean allOrNone;
-
-    /** Kernal context. */
-    private final GridKernalContext ctx;
-
-    /** Logger. */
-    private final IgniteLogger log;
-
     /** Names of services written to cache during current deployment. */
     private Collection<String> svcsToRollback;
 
     /** */
     private volatile ServiceDeploymentException err;
 
-    /**
-     * @param allOrNone Failing policy.
-     * @param ctx Kernal context.
-     */
-    GridServiceDeploymentCompoundFuture(boolean allOrNone, GridKernalContext ctx) {
-        this.allOrNone = allOrNone;
-        this.ctx = ctx;
-        this.log = ctx.log(getClass());
-    }
-
     /** {@inheritDoc} */
     @Override protected boolean processFailure(Throwable err, IgniteInternalFuture<Object> fut) {
         assert fut instanceof GridServiceDeploymentFuture : fut;
 
         GridServiceDeploymentFuture depFut = (GridServiceDeploymentFuture)fut;
 
-        if (allOrNone) {
-            if (initialized()) {
-                onDone(new IgniteCheckedException(
-                    new ServiceDeploymentException("Failed to deploy provided services.", err, getConfigurations())));
-            }
-            else {
-                synchronized (this) {
-                    if (this.err == null) {
-                        this.err = new ServiceDeploymentException("Failed to deploy provided services.", err,
-                            new ArrayList<ServiceConfiguration>());
-                    }
-                    else
-                        this.err.addSuppressed(err);
-                }
+        synchronized (this) {
+            if (this.err == null) {
+                this.err = new ServiceDeploymentException("Failed to deploy some services.",
+                    new ArrayList<ServiceConfiguration>());
             }
-        }
-        else {
-            synchronized (this) {
-                if (this.err == null)
-                    this.err = new ServiceDeploymentException("Failed to deploy some services.",
-                        new ArrayList<ServiceConfiguration>());
 
-                this.err.getFailedConfigurations().add(depFut.configuration());
-                this.err.addSuppressed(err);
-            }
+            this.err.getFailedConfigurations().add(depFut.configuration());
+            this.err.addSuppressed(err);
         }
 
         return true;
     }
 
-    /**
-     * Marks this future as initialized. Will complete with error if failures before initialization occurred and
-     * all-or-none policy is followed.
-     */
-    public void serviceDeploymentMarkInitialized() {
-        if (allOrNone && this.err != null) {
-            this.err.getFailedConfigurations().addAll(getConfigurations());
-
-            onDone(new IgniteCheckedException(this.err));
-        }
-        else
-            super.markInitialized();
-    }
-
     /** {@inheritDoc} */
-    @Override protected boolean onDone(@Nullable final Object res, @Nullable Throwable err, final boolean cancel) {
-        final Throwable resErr;
-
+    @Override protected boolean onDone(@Nullable Object res, @Nullable Throwable err, boolean cancel) {
         if (err == null && this.err != null)
-            resErr = new IgniteCheckedException(this.err);
-        else
-            resErr = err;
-
-        if (allOrNone && this.err != null && svcsToRollback != null) {
-            U.warn(log, "Failed to deploy provided services. The following services will be cancelled:" + svcsToRollback);
-
-            IgniteInternalFuture<?> fut = ctx.service().cancelAll(svcsToRollback);
-
-            /*
-            Can not call fut.get() since it is possible we are in system pool now and
-            fut also should be completed from system pool.
-             */
-            fut.listen(new IgniteInClosure<IgniteInternalFuture>() {
-                @Override public void apply(IgniteInternalFuture fut) {
-                    try {
-                        fut.get();
-                    }
-                    catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to cancel deployed services.", e);
-                    }
-                    finally {
-                        svcsToRollback = null;
-                    }
-
-                    GridServiceDeploymentCompoundFuture.super.onDone(res, resErr, cancel);
-                }
-            });
-
-            return false;
-        }
+            err = new IgniteCheckedException(this.err);
 
-        return super.onDone(res, resErr, cancel);
+        return super.onDone(res, err, cancel);
     }
 
     /**
@@ -180,18 +91,4 @@ public Collection<String> servicesToRollback() {
         else
             return Collections.emptyList();
     }
-
-    /**
-     * @return Collection of configurations, stored in child futures.
-     */
-    private Collection<ServiceConfiguration> getConfigurations() {
-        Collection<IgniteInternalFuture<Object>> futs = futures();
-
-        List<ServiceConfiguration> cfgs = new ArrayList<>(futs.size());
-
-        for (IgniteInternalFuture<Object> fut : futs)
-            cfgs.add(((GridServiceDeploymentFuture)fut).configuration());
-
-        return cfgs;
-    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 2a30bad8d4950..8c9ca053baec1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -305,7 +305,7 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
                     c.setNodeFilter(ctx.cluster().get().forServers().predicate());
             }
 
-            deployAll(Arrays.asList(cfgs), true).get();
+            deployAll(Arrays.asList(cfgs)).get();
         }
 
         if (log.isDebugEnabled())
@@ -487,10 +487,9 @@ public IgniteInternalFuture<?> deployKeyAffinitySingleton(String name, Service s
 
     /**
      * @param cfgs Service configurations.
-     * @param allOrNone Failure processing policy.
      * @return Configurations to deploy.
      */
-    private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+    private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceConfiguration> cfgs) {
         List<ServiceConfiguration> cfgsCp = new ArrayList<>(cfgs.size());
 
         ServicesCompatibilityState state = markCompatibilityStateAsUsed();
@@ -543,40 +542,28 @@ private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceCo
             }
 
             if (err != null) {
-                if (allOrNone) {
-                    return new PreparedConfigurations(null,
-                        null,
-                        new IgniteCheckedException(
-                            new ServiceDeploymentException("None of the provided services were deplyed.", err, cfgs)));
-                }
-                else {
-                    if (failedFuts == null)
-                        failedFuts = new ArrayList<>();
+                if (failedFuts == null)
+                    failedFuts = new ArrayList<>();
 
-                    GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+                GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
 
-                    fut.onDone(err);
+                fut.onDone(err);
 
-                    failedFuts.add(fut);
-                }
+                failedFuts.add(fut);
             }
         }
 
-        return new PreparedConfigurations(cfgsCp, failedFuts, null);
+        return new PreparedConfigurations(cfgsCp, failedFuts);
     }
 
     /**
      * @param cfgs Service configurations.
-     * @param allOrNone Failure processing policy.
      * @return Future for deployment.
      */
-    public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+    public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs) {
         assert cfgs != null;
 
-        PreparedConfigurations srvCfg = prepareServiceConfigurations(cfgs, allOrNone);
-
-        if (srvCfg.err != null)
-            return new GridFinishedFuture<>(srvCfg.err);
+        PreparedConfigurations srvCfg = prepareServiceConfigurations(cfgs);
 
         List<ServiceConfiguration> cfgsCp = srvCfg.cfgs;
 
@@ -591,7 +578,7 @@ public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs,
         GridServiceDeploymentCompoundFuture res;
 
         while (true) {
-            res = new GridServiceDeploymentCompoundFuture(allOrNone, ctx);
+            res = new GridServiceDeploymentCompoundFuture();
 
             if (ctx.deploy().enabled())
                 ctx.cache().context().deploy().ignoreOwnership(true);
@@ -608,16 +595,8 @@ else if (cfgsCp.size() > 1) {
                             catch (IgniteCheckedException e) {
                                 if (X.hasCause(e, ClusterTopologyCheckedException.class))
                                     throw e; // Retry.
-
-                                if (allOrNone) {
-                                    for (String name : res.servicesToRollback())
-                                        depFuts.remove(name).onDone(e);
-
-                                    res.onDone(new IgniteCheckedException(new ServiceDeploymentException(
-                                        "Failed to deploy provided services.", e, cfgs)));
-
-                                    return res;
-                                }
+                                else
+                                    U.error(log, e.getMessage());
                             }
                         }
 
@@ -651,7 +630,7 @@ else if (cfgsCp.size() > 1) {
         if (ctx.clientDisconnected()) {
             IgniteClientDisconnectedCheckedException err =
                 new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
-                "Failed to deploy services, client node disconnected: " + cfgs);
+                    "Failed to deploy services, client node disconnected: " + cfgs);
 
             for (String name : res.servicesToRollback()) {
                 GridServiceDeploymentFuture fut = depFuts.remove(name);
@@ -668,7 +647,7 @@ else if (cfgsCp.size() > 1) {
                 res.add(fut, false);
         }
 
-        res.serviceDeploymentMarkInitialized();
+        res.markInitialized();
 
         return res;
     }
@@ -705,12 +684,8 @@ private void writeServiceToCache(GridServiceDeploymentCompoundFuture res, Servic
 
             if (dep != null) {
                 if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
-                    String err = "Failed to deploy service (service already exists with different " +
-                        "configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']';
-
-                    U.error(log, err);
-
-                    throw new IgniteCheckedException(err);
+                    throw new IgniteCheckedException("Failed to deploy service (service already exists with " +
+                        "different configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']');
                 }
                 else {
                     res.add(fut, false);
@@ -755,7 +730,7 @@ private void writeServiceToCache(GridServiceDeploymentCompoundFuture res, Servic
     public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
         A.notNull(cfg, "cfg");
 
-        return deployAll(Collections.singleton(cfg), false);
+        return deployAll(Collections.singleton(cfg));
     }
 
     /**
@@ -788,7 +763,8 @@ public IgniteInternalFuture<?> cancel(String name) {
                 if (X.hasCause(e, ClusterTopologyCheckedException.class)) {
                     if (log.isDebugEnabled())
                         log.debug("Topology changed while cancelling service (will retry): " + e.getMessage());
-                } else {
+                }
+                else {
                     U.error(log, "Failed to undeploy service: " + name, e);
 
                     return new GridFinishedFuture<>(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
index a581e159348a8..dc41c22311722 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
@@ -31,19 +31,13 @@ class PreparedConfigurations {
     /** */
     final List<GridServiceDeploymentFuture> failedFuts;
 
-    /** */
-    final Exception err;
-
     /**
      * @param cfgs Configurations to deploy.
      * @param failedFuts Finished futures for failed configurations.
-     * @param err Error if need to stop deploy.
      */
-    PreparedConfigurations(List<ServiceConfiguration> cfgs, List<GridServiceDeploymentFuture> failedFuts,
-        Exception err) {
+    PreparedConfigurations(List<ServiceConfiguration> cfgs, List<GridServiceDeploymentFuture> failedFuts) {
         this.cfgs = cfgs;
         this.failedFuts = failedFuts;
-        this.err = err;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
index ca95198cf2fc9..bcc050400835b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
@@ -22,9 +22,7 @@
 import java.util.List;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
-import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.services.ServiceConfiguration;
@@ -33,16 +31,6 @@
 
 /** */
 public class GridServiceDeploymentCompoundFutureSelfTest extends GridCommonAbstractTest {
-    /** */
-    private static GridKernalContext ctx;
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        IgniteKernal kernal = (IgniteKernal)startGrid(0);
-
-        ctx = kernal.context();
-    }
-
     /** {@inheritDoc} */
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
@@ -51,8 +39,8 @@ public class GridServiceDeploymentCompoundFutureSelfTest extends GridCommonAbstr
     /**
      * @throws Exception If failed.
      */
-    public void testWaitForCompletionOnFailingFuturePartial() throws Exception {
-        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(false, ctx);
+    public void testWaitForCompletionOnFailingFuture() throws Exception {
+        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture();
 
         int failingFutsNum = 2;
 
@@ -80,7 +68,7 @@ public void testWaitForCompletionOnFailingFuturePartial() throws Exception {
             compFut.add(fut);
         }
 
-        compFut.serviceDeploymentMarkInitialized();
+        compFut.markInitialized();
 
         List<Exception> causes = new ArrayList<>();
 
@@ -125,109 +113,6 @@ public void testWaitForCompletionOnFailingFuturePartial() throws Exception {
         }
     }
 
-    /**
-     * @throws Exception if failed.
-     */
-    public void testFailAllAfterInitialized() throws Exception {
-        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(true, ctx);
-
-        ServiceConfiguration failingCfg = config("Failed");
-
-        GridServiceDeploymentFuture failingFut = new GridServiceDeploymentFuture(failingCfg);
-
-        compFut.add(failingFut);
-
-        int futsNum = 5;
-
-        List<ServiceConfiguration> cfgs = new ArrayList<>(futsNum + 1);
-
-        cfgs.add(failingCfg);
-
-        for (int i = 0; i < futsNum; i++) {
-            ServiceConfiguration cfg = config(String.valueOf(i));
-
-            cfgs.add(cfg);
-
-            compFut.add(new GridServiceDeploymentFuture(cfg));
-        }
-
-        compFut.serviceDeploymentMarkInitialized();
-
-        Exception expCause = new Exception("Test error");
-
-        failingFut.onDone(expCause);
-
-        assertFailAll(compFut, cfgs, expCause);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testFailAllBeforeInitialized() throws Exception {
-        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(true, ctx);
-
-        ServiceConfiguration failingCfg = config("Failed");
-
-        GridServiceDeploymentFuture failingFut = new GridServiceDeploymentFuture(failingCfg);
-
-        Exception expCause = new Exception("Test error");
-
-        failingFut.onDone(expCause);
-
-        compFut.add(failingFut);
-
-        assertFalse(compFut.isDone());
-
-        int futsNum = 5;
-
-        List<ServiceConfiguration> cfgs = new ArrayList<>(futsNum + 1);
-
-        cfgs.add(failingCfg);
-
-        for (int i = 0; i < futsNum; i++) {
-            ServiceConfiguration cfg = config(String.valueOf(i));
-
-            cfgs.add(cfg);
-
-            compFut.add(new GridServiceDeploymentFuture(cfg));
-        }
-
-        compFut.serviceDeploymentMarkInitialized();
-
-        assertFailAll(compFut, cfgs, expCause);
-    }
-
-    /**
-     * Try waiting for the future completion and check that a proper exception is thrown.
-     *
-     * @param fut Future.
-     * @param expCfgs Expected cfgs.
-     * @param expCause Expected cause.
-     */
-    private void assertFailAll(GridServiceDeploymentCompoundFuture fut, Collection<ServiceConfiguration> expCfgs,
-        Exception expCause) {
-        try {
-            fut.get();
-
-            fail("Should never reach here.");
-        }
-        catch (IgniteCheckedException ce) {
-            log.info("Expected exception: " + ce.getMessage());
-
-            IgniteException e = U.convertException(ce);
-
-            assertTrue(e instanceof ServiceDeploymentException);
-
-            assertEqualsCollections(expCfgs, ((ServiceDeploymentException)e).getFailedConfigurations());
-
-            Throwable actCause = e.getCause();
-
-            assertTrue(actCause instanceof IgniteCheckedException);
-
-            assertEquals(expCause, actCause.getCause());
-        }
-    }
-
     /**
      * @param name Name.
      * @return Dummy configuration with a specified name.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
index 337e69565968b..ab012b37ea551 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
@@ -100,7 +100,7 @@ public void testDeployAll() throws Exception {
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(cfgs, false);
+        client.services().deployAll(cfgs);
 
         assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
 
@@ -121,7 +121,7 @@ public void testDeployAllAsync() throws Exception {
 
         IgniteServices services =  client.services().withAsync();
 
-        services.deployAll(cfgs, false);
+        services.deployAll(cfgs);
 
         services.future().get();
 
@@ -131,7 +131,7 @@ public void testDeployAllAsync() throws Exception {
     }
 
     /**
-     * TODO: enable when IGNITE-6259 is fixed
+     * TODO: enable when IGNITE-6259 is fixed.
      *
      * @throws Exception If failed.
      */
@@ -170,7 +170,7 @@ public void _testDeployAllTopologyChange() throws Exception {
             while (from < numServices) {
                 int to = Math.min(numServices, from + batchSize);
 
-                services.deployAll(cfgs.subList(from, to), false);
+                services.deployAll(cfgs.subList(from, to));
 
                 services.future().get(5000);
 
@@ -189,7 +189,7 @@ public void _testDeployAllTopologyChange() throws Exception {
     }
 
     /**
-     * TODO: enable when IGNITE-6259 is fixed
+     * TODO: enable when IGNITE-6259 is fixed.
      *
      * @throws Exception If failed.
      */
@@ -239,7 +239,7 @@ public void _testDeployAllTopologyChangeFail() throws Exception {
                 failingCfgs.add(failingCfg);
 
                 try {
-                    services.deployAll(cfgsBatch, false);
+                    services.deployAll(cfgsBatch);
 
                     services.future().get(5000);
 
@@ -274,37 +274,22 @@ public void _testDeployAllTopologyChangeFail() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testDeployAllFailAll() throws Exception {
-        deployAllFail(false, true);
+    public void testDeployAllFail() throws Exception {
+        deployAllFail(false);
     }
 
     /**
      * @throws Exception If failed.
      */
-    public void testDeployAllPartial() throws Exception {
-        deployAllFail(false, false);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testDeployAllAsyncFailAll() throws Exception {
-        deployAllFail(true, true);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testDeployAllAsyncFailPartial() throws Exception {
-        deployAllFail(true, false);
+    public void testDeployAllAsyncFail() throws Exception {
+        deployAllFail(true);
     }
 
     /**
      * @param async If {@code true}, then asynchronous method of deployment will be performed.
-     * @param allOrNone Failing strategy.
      * @throws Exception If failed.
      */
-    private void deployAllFail(boolean async, boolean allOrNone) throws Exception {
+    private void deployAllFail(boolean async) throws Exception {
         Ignite client = grid(CLIENT_NODE_NAME);
 
         CountDownLatch latch = new CountDownLatch(NUM_SERVICES - 1);
@@ -317,20 +302,11 @@ private void deployAllFail(boolean async, boolean allOrNone) throws Exception {
 
         failingCfg.setName(null);
 
-        assertFailingDeploy(client, async, allOrNone, cfgs, failingCfg);
+        assertFailingDeploy(client, async, cfgs, failingCfg);
 
-        if (allOrNone) {
-            assertFalse("Some of the services were deployed.", latch.await(2, TimeUnit.SECONDS));
-
-            assertEquals(NUM_SERVICES - 1, latch.getCount());
-
-            assertTrue(client.services().serviceDescriptors().isEmpty());
-        }
-        else {
-            assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
 
-            assertDeployedServices(client, cfgs.subList(0, cfgs.size() - 1));
-        }
+        assertDeployedServices(client, cfgs.subList(0, cfgs.size() - 1));
     }
 
     /**
@@ -351,8 +327,8 @@ public void testClashingNames() throws Exception {
         IgniteServices svcs1 = client.services().withAsync();
         IgniteServices svcs2 = client.services().withAsync();
 
-        svcs1.deployAll(fstBatch, false);
-        svcs2.deployAll(sndBatch, false);
+        svcs1.deployAll(fstBatch);
+        svcs2.deployAll(sndBatch);
 
         svcs1.future().get();
         svcs2.future().get();
@@ -365,32 +341,12 @@ public void testClashingNames() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testClashingNamesFailAll() throws Exception {
-        clashingNamesFail(true);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testClashingNamesPartial() throws Exception {
-        clashingNamesFail(false);
-    }
-
-    /**
-     * @param allOrNone Failing strategy.
-     * @throws Exception If failed.
-     */
-    private void clashingNamesFail(boolean allOrNone) throws Exception {
+    public void testClashingNamesFail() throws Exception {
         Ignite client = grid(CLIENT_NODE_NAME);
 
         List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
 
-        int numDepSvcs;
-
-        if (allOrNone)
-            numDepSvcs = NUM_SERVICES / 2;
-        else
-            numDepSvcs = NUM_SERVICES - 1;
+        int numDepSvcs = NUM_SERVICES - 1;
 
         CountDownLatch latch = new CountDownLatch(numDepSvcs);
 
@@ -401,13 +357,13 @@ private void clashingNamesFail(boolean allOrNone) throws Exception {
 
         IgniteServices services = client.services().withAsync();
 
-        services.deployAll(fstBatch, false);
+        services.deployAll(fstBatch);
 
         ServiceConfiguration failingCfg = cfgs.get(NUM_SERVICES - 1);
 
         failingCfg.setName(null);
 
-        assertFailingDeploy(client, false, allOrNone, sndBatch, failingCfg);
+        assertFailingDeploy(client, false, sndBatch, failingCfg);
 
         services.future().get();
 
@@ -419,33 +375,12 @@ private void clashingNamesFail(boolean allOrNone) throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testClashingNameDifferentConfigFailAll() throws Exception {
-        testClashingNameDifferentConfig(true);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testClashingNameDifferentConfigPartial() throws Exception {
-        testClashingNameDifferentConfig(false);
-    }
-
-    /**
-     * @param allOrNone Failing strategy.
-     * @throws Exception If failed.
-     */
-    private void testClashingNameDifferentConfig(boolean allOrNone) throws Exception {
+    public void testClashingNameDifferentConfig() throws Exception {
         Ignite client = grid(CLIENT_NODE_NAME);
 
         List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
 
-        int numDepSvcs;
-
-        if (allOrNone)
-            numDepSvcs = NUM_SERVICES / 2;
-        else
-            numDepSvcs = NUM_SERVICES - 1;
-
+        int numDepSvcs = NUM_SERVICES - 1;
 
         CountDownLatch latch = new CountDownLatch(numDepSvcs);
 
@@ -454,7 +389,7 @@ private void testClashingNameDifferentConfig(boolean allOrNone) throws Exception
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(fstBatch, false);
+        client.services().deployAll(fstBatch);
 
         ServiceConfiguration failingCfg = copyService(cfgs.get(NUM_SERVICES - 1));
 
@@ -464,7 +399,7 @@ private void testClashingNameDifferentConfig(boolean allOrNone) throws Exception
 
         sndBatch.add(failingCfg);
 
-        assertFailingDeploy(client, false, allOrNone, sndBatch, failingCfg);
+        assertFailingDeploy(client, false, sndBatch, failingCfg);
 
         assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
 
@@ -483,7 +418,7 @@ public void testCancelAll() throws Exception {
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(cfgs, true);
+        client.services().deployAll(cfgs);
 
         latch.await(30, TimeUnit.SECONDS);
 
@@ -504,7 +439,7 @@ public void testCancelAllAsync() throws Exception {
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(cfgs, true);
+        client.services().deployAll(cfgs);
 
         latch.await(30, TimeUnit.SECONDS);
 
@@ -518,7 +453,7 @@ public void testCancelAllAsync() throws Exception {
     }
 
     /**
-     * TODO: enable when IGNITE-6259 is fixed
+     * TODO: enable when IGNITE-6259 is fixed.
      *
      * @throws Exception If failed.
      */
@@ -533,7 +468,7 @@ public void _testCancelAllTopologyChange() throws Exception {
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(cfgs, true);
+        client.services().deployAll(cfgs);
 
         latch.await(30, TimeUnit.SECONDS);
 
@@ -585,7 +520,7 @@ public void testCancelAllClashingNames() throws Exception {
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(cfgs, true);
+        client.services().deployAll(cfgs);
 
         latch.await(30, TimeUnit.SECONDS);
 
@@ -616,11 +551,10 @@ public void testCancelAllClashingNames() throws Exception {
      * @param client Client.
      * @param async If {@code true}, then async version of deploy method will be used.
      * @param cfgs Service configurations.
-     * @param allOrNone Failing policy.
      * @param failingCfg Configuration of the failing service.
      * @throws Exception If failed.
      */
-    private void assertFailingDeploy(Ignite client, boolean async, boolean allOrNone, List<ServiceConfiguration> cfgs,
+    private void assertFailingDeploy(Ignite client, boolean async, List<ServiceConfiguration> cfgs,
         ServiceConfiguration failingCfg) throws Exception {
 
         IgniteServices services = client.services();
@@ -628,26 +562,21 @@ private void assertFailingDeploy(Ignite client, boolean async, boolean allOrNone
         if (async) {
             services = services.withAsync();
 
-            services.deployAll(cfgs, allOrNone);
+            services.deployAll(cfgs);
         }
 
         try {
             if (async)
                 services.future().get();
             else
-                services.deployAll(cfgs, allOrNone);
+                services.deployAll(cfgs);
 
             fail("Should never reach here.");
         }
         catch (ServiceDeploymentException e) {
             info("Expected exception: " + e.getMessage());
 
-            Collection<ServiceConfiguration> expFails;
-
-            if (allOrNone)
-                expFails = cfgs;
-            else
-                expFails = Collections.singleton(failingCfg);
+            Collection<ServiceConfiguration> expFails = Collections.singleton(failingCfg);
 
             Collection<ServiceConfiguration> actFails = e.getFailedConfigurations();
 

From 565ee846490834b74a9cc0f8cd0a28a5494392cd Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 8 Sep 2017 11:20:28 +0300
Subject: [PATCH 410/446] GG-12690 - Deregister local continuous query handlers
 on cache stop.

---
 .../CacheContinuousQueryHandler.java          |  7 ++++++
 .../CacheContinuousQueryManager.java          |  2 +-
 .../continuous/GridContinuousProcessor.java   | 24 +++++++++++++++++++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 0cdbec5013b15..ad11013678db5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -227,6 +227,13 @@ public void internal(boolean internal) {
         this.internal = internal;
     }
 
+    /**
+     * @return Internal query.
+     */
+    public boolean internal() {
+        return internal;
+    }
+
     /**
      * @param notifyExisting Notify existing.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 12b02f0fa9386..b3f7d6e93f7b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -839,7 +839,7 @@ GridContinuousHandler.RegisterStatus registerListener(UUID lsnrId,
      * @param internal Internal flag.
      * @param id Listener ID.
      */
-    void unregisterListener(boolean internal, UUID id) {
+    public void unregisterListener(boolean internal, UUID id) {
         CacheContinuousQueryListener lsnr;
 
         if (internal) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 50d29d1f8de6e..6b1d0beaf02e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -61,6 +61,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -568,6 +569,29 @@ public void onCacheStop(GridCacheContext ctx) {
             if (hnd.isQuery() && F.eq(ctx.name(), hnd.cacheName()))
                 it.remove();
         }
+
+        // Deregister local handlers.
+        Iterator<Map.Entry<UUID, LocalRoutineInfo>> it2 = locInfos.entrySet().iterator();
+
+        while (it2.hasNext()) {
+            Map.Entry<UUID, LocalRoutineInfo> entry = it2.next();
+
+            GridContinuousHandler hnd = entry.getValue().hnd;
+
+            if (hnd.isQuery() && F.eq(ctx.name(), hnd.cacheName())) {
+                it2.remove();
+
+                assert hnd instanceof CacheContinuousQueryHandler : hnd;
+
+                CacheContinuousQueryHandler hnd0 = (CacheContinuousQueryHandler)hnd;
+
+                unregisterHandler(entry.getKey(), hnd, true);
+
+                CacheContinuousQueryManager qryMgr = ctx.continuousQueries();
+
+                qryMgr.unregisterListener(hnd0.internal(), entry.getKey());
+            }
+        }
     }
 
     /**

From 1c07cc9babf2e77736cd1caa44ddf944eeea3cf1 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 8 Sep 2017 14:37:03 +0300
Subject: [PATCH 411/446] ignite-12754 fix for
 IgniteCacheCreatePutTest.testUpdatesAndCacheStart.

fix is based on the following commit: 05dd08b993e2d7f88176c051463b178431714f85
---
 .../processors/cache/distributed/dht/GridDhtCacheAdapter.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 2fa934b79db71..417eb3536536f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -219,7 +219,8 @@ protected GridDhtCacheAdapter(GridCacheContext<K, V> ctx, GridCacheConcurrentMap
     @Override public void onKernalStart() throws IgniteCheckedException {
         super.onKernalStart();
 
-        preldr.onKernalStart();
+        if (preldr != null)
+            preldr.onKernalStart();
     }
 
     /** {@inheritDoc} */

From 9e3e1149173787f7e5501f937907f3fc6ffd6ffe Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 5 Sep 2016 13:43:54 +0300
Subject: [PATCH 412/446] Minor fix for IGNITE-6256: Add localnode to topology
 snapshot.

(cherry picked from commit bb40e57)
---
 .../ignite/spi/discovery/tcp/ServerImpl.java   | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index cfa36f1599590..b1fd7e23f308f 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -458,34 +458,36 @@ else if (log.isInfoEnabled()) {
         U.interrupt(statsPrinter);
         U.join(statsPrinter, log);
 
-        Collection<TcpDiscoveryNode> rmts = null;
+        Collection<TcpDiscoveryNode> nodes = null;
 
         if (!disconnect)
             spi.printStopInfo();
         else {
             spi.getSpiContext().deregisterPorts();
 
-            rmts = ring.visibleRemoteNodes();
+            nodes = ring.visibleNodes();
         }
 
         long topVer = ring.topologyVersion();
 
         ring.clear();
 
-        if (rmts != null && !rmts.isEmpty()) {
-            // This is restart/disconnection and remote nodes are not empty.
-            // We need to fire FAIL event for each.
+        if (nodes != null) {
+            // This is restart/disconnection and we need to fire FAIL event for each remote node.
             DiscoverySpiListener lsnr = spi.lsnr;
 
             if (lsnr != null) {
-                Collection<ClusterNode> processed = new HashSet<>();
+                Collection<ClusterNode> processed = new HashSet<>(nodes.size());
+
+                for (TcpDiscoveryNode n : nodes) {
+                    if(n.isLocal())
+                        continue;
 
-                for (TcpDiscoveryNode n : rmts) {
                     assert n.visible();
 
                     processed.add(n);
 
-                    List<ClusterNode> top = U.arrayList(rmts, F.notIn(processed));
+                    List<ClusterNode> top = U.arrayList(nodes, F.notIn(processed));
 
                     topVer++;
 

From 168655820615765817b05d8e8dbe9cbae5fd41dd Mon Sep 17 00:00:00 2001
From: Konstantin Dudkov <kdudkov@ya.ru>
Date: Fri, 7 Apr 2017 17:17:04 +0300
Subject: [PATCH 413/446] IGNITE-4876 - Tests should wait for topology change
 on all nodes

(cherry picked from commit 4e80ddf)
---
 .../junits/GridAbstractTest.java              | 53 ++++++++++++++++++-
 1 file changed, 51 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 5c43ecbfc9bb2..a9815483ee453 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -57,16 +57,20 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgnitionEx;
 import org.apache.ignite.internal.binary.BinaryEnumCache;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
 import org.apache.ignite.internal.util.GridClassLoaderCache;
 import org.apache.ignite.internal.util.GridTestClockTimer;
 import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -135,6 +139,9 @@ public abstract class GridAbstractTest extends TestCase {
     /** */
     private static final long DFLT_TEST_TIMEOUT = 5 * 60 * 1000;
 
+    /** */
+    private static final int DFLT_TOP_WAIT_TIMEOUT = 2000;
+
     /** */
     private static final transient Map<Class<?>, TestCounters> tests = new ConcurrentHashMap<>();
 
@@ -862,6 +869,16 @@ protected void stopGrid(@Nullable String gridName) {
      */
     @SuppressWarnings({"deprecation"})
     protected void stopGrid(@Nullable String gridName, boolean cancel) {
+        stopGrid(gridName, cancel, true);
+    }
+
+    /**
+     * @param gridName Ignite instance name.
+     * @param cancel Cancel flag.
+     * @param awaitTop Await topology change flag.
+     */
+    @SuppressWarnings({"deprecation"})
+    protected void stopGrid(@Nullable String gridName, boolean cancel, boolean awaitTop) {
         try {
             Ignite ignite = grid(gridName);
 
@@ -874,6 +891,9 @@ protected void stopGrid(@Nullable String gridName, boolean cancel) {
                 G.stop(gridName, cancel);
             else
                 IgniteProcessProxy.stop(gridName, cancel);
+
+            if (awaitTop)
+                awaitTopologyChange();
         }
         catch (IllegalStateException ignored) {
             // Ignore error if grid already stopped.
@@ -908,10 +928,10 @@ protected void stopAllGrids(boolean cancel) {
             }
 
             for (Ignite g : clients)
-                stopGrid(g.name(), cancel);
+                stopGrid(g.name(), cancel, false);
 
             for (Ignite g : srvs)
-                stopGrid(g.name(), cancel);
+                stopGrid(g.name(), cancel, false);
 
             assert G.allGrids().isEmpty();
         }
@@ -1894,6 +1914,35 @@ public static IgniteEx grid(String name, boolean remote, boolean thisRemote) {
         }
     }
 
+    /**
+     *
+     * @throws IgniteInterruptedCheckedException
+     */
+    public void awaitTopologyChange() throws IgniteInterruptedCheckedException {
+        for (Ignite g : G.allGrids()) {
+            final GridKernalContext ctx = ((IgniteKernal)g).context();
+
+            if (ctx.isStopping())
+                continue;
+
+            AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx();
+            AffinityTopologyVersion exchVer = ctx.cache().context().exchange().readyAffinityVersion();
+
+            if (! topVer.equals(exchVer)) {
+                info("topology version mismatch: node "  + g.name() + " " + exchVer + ", " + topVer);
+
+                GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                    @Override public boolean apply() {
+                        AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx();
+                        AffinityTopologyVersion exchVer = ctx.cache().context().exchange().readyAffinityVersion();
+
+                        return exchVer.equals(topVer);
+                    }
+                }, DFLT_TOP_WAIT_TIMEOUT);
+            }
+        }
+    }
+
     /**
      *
      */

From 8ca38cdfb6edfb84ef4e0b2c83853776d8bf3b31 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 12 Sep 2017 16:53:46 +0300
Subject: [PATCH 414/446] GG-12695: Fixed test
 IgniteCacheNearRestartRollbackSelfTest.testRestarts().

---
 .../apache/ignite/internal/IgniteKernal.java  | 66 +++++++++++--------
 ...gniteCacheNearRestartRollbackSelfTest.java |  2 -
 2 files changed, 37 insertions(+), 31 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index a1997b1adfed7..d41075f012315 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -887,37 +887,45 @@ public void start(final IgniteConfiguration cfg,
 
             ctx.add(discoMgr, false);
 
-            // Start processors before discovery manager, so they will
-            // be able to start receiving messages once discovery completes.
-            startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx));
-            startProcessor(new GridClockSyncProcessor(ctx));
-            startProcessor(new GridAffinityProcessor(ctx));
-            startProcessor(createComponent(GridSegmentationProcessor.class, ctx));
-            startProcessor(createComponent(IgniteCacheObjectProcessor.class, ctx));
-            startProcessor(new GridCacheProcessor(ctx));
-            startProcessor(new GridQueryProcessor(ctx));
-            startProcessor(new OdbcProcessor(ctx));
-            startProcessor(new GridServiceProcessor(ctx));
-            startProcessor(new GridTaskSessionProcessor(ctx));
-            startProcessor(new GridJobProcessor(ctx));
-            startProcessor(new GridTaskProcessor(ctx));
-            startProcessor((GridProcessor)SCHEDULE.createOptional(ctx));
-            startProcessor(new GridRestProcessor(ctx));
-            startProcessor(new DataStreamProcessor(ctx));
-            startProcessor((GridProcessor)IGFS.create(ctx, F.isEmpty(cfg.getFileSystemConfiguration())));
-            startProcessor(new GridContinuousProcessor(ctx));
-            startProcessor(createHadoopComponent());
-            startProcessor(new DataStructuresProcessor(ctx));
-            startProcessor(createComponent(PlatformProcessor.class, ctx));
-
-            // Start plugins.
-            for (PluginProvider provider : ctx.plugins().allProviders()) {
-                ctx.add(new GridPluginComponent(provider));
-
-                provider.start(ctx.plugins().pluginContextForProvider(provider));
+                // Start processors before discovery manager, so they will
+                // be able to start receiving messages once discovery completes.
+            try {
+                startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx));
+                startProcessor(new GridClockSyncProcessor(ctx));
+                startProcessor(new GridAffinityProcessor(ctx));
+                startProcessor(createComponent(GridSegmentationProcessor.class, ctx));
+                startProcessor(createComponent(IgniteCacheObjectProcessor.class, ctx));
+                startProcessor(new GridCacheProcessor(ctx));
+                startProcessor(new GridQueryProcessor(ctx));
+                startProcessor(new OdbcProcessor(ctx));
+                startProcessor(new GridServiceProcessor(ctx));
+                startProcessor(new GridTaskSessionProcessor(ctx));
+                startProcessor(new GridJobProcessor(ctx));
+                startProcessor(new GridTaskProcessor(ctx));
+                startProcessor((GridProcessor)SCHEDULE.createOptional(ctx));
+                startProcessor(new GridRestProcessor(ctx));
+                startProcessor(new DataStreamProcessor(ctx));
+                startProcessor((GridProcessor)IGFS.create(ctx, F.isEmpty(cfg.getFileSystemConfiguration())));
+                startProcessor(new GridContinuousProcessor(ctx));
+                startProcessor(createHadoopComponent());
+                startProcessor(new DataStructuresProcessor(ctx));
+                startProcessor(createComponent(PlatformProcessor.class, ctx));
+
+                // Start plugins.
+                for (PluginProvider provider : ctx.plugins().allProviders()) {
+                    ctx.add(new GridPluginComponent(provider));
+
+                    provider.start(ctx.plugins().pluginContextForProvider(provider));
+                }
+
+                fillNodeAttributes(clusterProc.updateNotifierEnabled());
             }
+            catch (Throwable e) {
+                // Stop discovery spi to close tcp socket.
+                ctx.discovery().stop(true);
 
-            fillNodeAttributes(clusterProc.updateNotifierEnabled());
+                throw e;
+            }
 
             gw.writeLock();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index aea4d7782d858..3f242b5726867 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -132,8 +132,6 @@ protected CacheConfiguration<Object, Object> cacheConfiguration(String gridName)
      */
     @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
     public void testRestarts() throws Exception {
-        fail("https://ggsystems.atlassian.net/browse/GG-12398");
-
         startGrids(4);
 
         Ignite tester = ignite(3);

From b954d26ee388416baadeeae597efd2d6060038f7 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Mon, 1 Aug 2016 10:10:51 +0300
Subject: [PATCH 415/446] IGNITE-1690: Re-enabled
 IgniteCacheCreateRestartSelfTest.testStopOriginatingNode. This closes #753.

(cherry picked from commit 0763de8)
---
 .../processors/cache/IgniteCacheCreateRestartSelfTest.java      | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheCreateRestartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheCreateRestartSelfTest.java
index e8e66c4155391..681636a34860e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheCreateRestartSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheCreateRestartSelfTest.java
@@ -71,8 +71,6 @@ public class IgniteCacheCreateRestartSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testStopOriginatingNode() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-1690");
-
         startGrids(NODES);
 
         ThreadLocalRandom rnd = ThreadLocalRandom.current();

From 88a7c0239bffcc2a3a47e67d11df360542dd0be5 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Wed, 14 Jun 2017 14:55:48 +0300
Subject: [PATCH 416/446] Fixed missing Apache header in
 HttpIgniteUpdatesChecker.

(cherry picked from commit 69aa299)
---
 .../cluster/HttpIgniteUpdatesChecker.java       | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
index c052c0997fe2a..2b93ceb1e44e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.ignite.internal.processors.cluster;
 
 import java.io.BufferedReader;

From 77723e433561fd908efda9abd067536494137ac4 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 13 Sep 2017 12:07:55 +0300
Subject: [PATCH 417/446] GG-12767: fixed flaky test IgniteCacheGetRestartTest.

Try avoid timeout in IgniteCacheGetRestartTest.

(cherry picked from commit b41ecd1)
---
 .../cache/distributed/IgniteCacheGetRestartTest.java  | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGetRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGetRestartTest.java
index 3b0c2fac32947..eef7750b05bf0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGetRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGetRestartTest.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -90,6 +91,8 @@ public class IgniteCacheGetRestartTest extends GridCommonAbstractTest {
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL, "true");
+
         super.beforeTestsStarted();
 
         startGrids(SRVS);
@@ -108,11 +111,13 @@ public class IgniteCacheGetRestartTest extends GridCommonAbstractTest {
         super.afterTestsStopped();
 
         stopAllGrids();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
     }
 
     /** {@inheritDoc} */
     @Override protected long getTestTimeout() {
-        return TEST_TIME + 60_000;
+        return TEST_TIME + 3 * 60_000;
     }
 
     /**
@@ -198,6 +203,8 @@ private void checkRestart(final CacheConfiguration ccfg, final int restartCnt) t
                 @Override public Void call() throws Exception {
                     int nodeIdx = restartNodeIdx.getAndIncrement();
 
+                    Thread.currentThread().setName("restart-thread-" + nodeIdx);
+
                     boolean clientMode = clientNode.compareAndSet(false, true);
 
                     while (U.currentTimeMillis() < stopTime) {
@@ -219,7 +226,7 @@ private void checkRestart(final CacheConfiguration ccfg, final int restartCnt) t
 
                         IgniteInternalFuture<?> syncFut = ((IgniteCacheProxy)cache).context().preloader().syncFuture();
 
-                        while (!syncFut.isDone())
+                        while (!syncFut.isDone() && U.currentTimeMillis() < stopTime)
                             checkGet(cache);
 
                         checkGet(cache);

From 395e78cf92621df23c8173323031f655cfe53518 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 13 Apr 2017 16:36:54 +0300
Subject: [PATCH 418/446] IGNITE-3477 - Fixing flaky full API suite

(cherry picked from commit 8d2b020)
---
 .../cache/H2CacheStoreStrategy.java           | 35 +++++++++++++++----
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/H2CacheStoreStrategy.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/H2CacheStoreStrategy.java
index ccb299438ce86..6261366e10569 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/H2CacheStoreStrategy.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/H2CacheStoreStrategy.java
@@ -54,6 +54,9 @@ public class H2CacheStoreStrategy implements TestCacheStoreStrategy {
     /** Pool to get {@link Connection}s from. */
     private final JdbcConnectionPool dataSrc;
 
+    /** */
+    private final int port;
+
     /** Script that creates CACHE table. */
     private static final String CREATE_CACHE_TABLE =
         "create table if not exists CACHE(k binary not null, v binary not null, PRIMARY KEY(k));";
@@ -75,9 +78,14 @@ public class H2CacheStoreStrategy implements TestCacheStoreStrategy {
      * @throws IgniteCheckedException If failed.
      */
     public H2CacheStoreStrategy() throws IgniteCheckedException {
+        Server srv = null;
+
         try {
-            Server.createTcpServer().start();
-            dataSrc = H2CacheStoreSessionListenerFactory.createDataSource();
+            srv = Server.createTcpServer().start();
+
+            port = srv.getPort();
+
+            dataSrc = H2CacheStoreSessionListenerFactory.createDataSource(port);
 
             try (Connection conn = connection()) {
                 RunScript.execute(conn, new StringReader(CREATE_CACHE_TABLE));
@@ -86,7 +94,8 @@ public H2CacheStoreStrategy() throws IgniteCheckedException {
             }
         }
         catch (SQLException e) {
-            throw new IgniteCheckedException(e);
+            throw new IgniteCheckedException("Failed to set up cache store strategy" +
+                (srv == null ? "" : ": " + srv.getStatus()), e);
         }
     }
 
@@ -242,7 +251,7 @@ private int querySingleInt(String qry, String errorMsg) {
 
     /** {@inheritDoc} */
     @Override public void updateCacheConfiguration(CacheConfiguration<Object, Object> cfg) {
-        cfg.setCacheStoreSessionListenerFactories(new H2CacheStoreSessionListenerFactory());
+        cfg.setCacheStoreSessionListenerFactories(new H2CacheStoreSessionListenerFactory(port));
     }
 
     /** {@inheritDoc} */
@@ -260,11 +269,23 @@ public static class H2StoreFactory implements Factory<CacheStore<Object, Object>
 
     /** Serializable {@link Factory} producing H2 backed {@link CacheStoreSessionListener}s. */
     public static class H2CacheStoreSessionListenerFactory implements Factory<CacheStoreSessionListener> {
+        /** */
+        private int port;
+
+        /**
+         * @param port Port.
+         */
+        public H2CacheStoreSessionListenerFactory(int port) {
+            this.port = port;
+        }
+
         /**
          * @return Connection pool
          */
-        static JdbcConnectionPool createDataSource() {
-            JdbcConnectionPool pool = JdbcConnectionPool.create("jdbc:h2:tcp://localhost/mem:TestDb;LOCK_MODE=0", "sa", "");
+        static JdbcConnectionPool createDataSource(int port) {
+            JdbcConnectionPool pool = JdbcConnectionPool.create("jdbc:h2:tcp://localhost:" + port +
+                "/mem:TestDb;LOCK_MODE=0", "sa", "");
+
             pool.setMaxConnections(100);
             return pool;
         }
@@ -272,7 +293,7 @@ static JdbcConnectionPool createDataSource() {
         /** {@inheritDoc} */
         @Override public CacheStoreSessionListener create() {
             CacheJdbcStoreSessionListener lsnr = new CacheJdbcStoreSessionListener();
-            lsnr.setDataSource(createDataSource());
+            lsnr.setDataSource(createDataSource(port));
             return lsnr;
         }
     }

From 9ad68c735f76c6f5ce16c787e5cc07d2638d116d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 13 Apr 2017 16:36:54 +0300
Subject: [PATCH 419/446] GG-12773: Fixed potential deadlock when service is
 being deployed while node is stopping.

---
 .../service/GridServiceProcessor.java         | 171 ++++++++++--------
 .../GridServiceProcessorStopSelfTest.java     | 137 +++++++++++++-
 2 files changed, 228 insertions(+), 80 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index c452da31086d0..106aa7f678010 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -493,124 +493,137 @@ public IgniteInternalFuture<?> deployKeyAffinitySingleton(String name, Service s
     public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
         A.notNull(cfg, "cfg");
 
-        ServicesCompatibilityState state = markCompatibilityStateAsUsed();
+        if (!busyLock.enterBusy()) {
+            IgniteCheckedException e = new IgniteCheckedException("Service deployment has been cancelled (node is stopping): " +
+                cfg.getName());
 
-        validate(cfg);
-
-        ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
+            return new GridFinishedFuture<>(e);
+        }
 
-        if (!state.srvcCompatibility) {
-            Marshaller marsh = ctx.config().getMarshaller();
+        try {
 
-            LazyServiceConfiguration cfg0;
+            ServicesCompatibilityState state = markCompatibilityStateAsUsed();
 
-            try {
-                byte[] srvcBytes = U.marshal(marsh, cfg.getService());
+            validate(cfg);
 
-                cfg0 = new LazyServiceConfiguration(cfg, srvcBytes);
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to marshal service with configured marshaller [srvc=" + cfg.getService()
-                    + ", marsh=" + marsh + "]", e);
+            ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
 
-                return new GridFinishedFuture<>(e);
-            }
+            if (!state.srvcCompatibility) {
+                Marshaller marsh = ctx.config().getMarshaller();
 
-            cfg = cfg0;
-        }
+                LazyServiceConfiguration cfg0;
 
-        GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+                try {
+                    byte[] srvcBytes = U.marshal(marsh, cfg.getService());
 
-        GridServiceDeploymentFuture old = depFuts.putIfAbsent(cfg.getName(), fut);
+                    cfg0 = new LazyServiceConfiguration(cfg, srvcBytes);
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to marshal service with configured marshaller [srvc=" + cfg.getService()
+                        + ", marsh=" + marsh + "]", e);
 
-        if (old != null) {
-            if (!old.configuration().equalsIgnoreNodeFilter(cfg)) {
-                fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
-                    "different configuration) [deployed=" + old.configuration() + ", new=" + cfg + ']'));
+                    return new GridFinishedFuture<>(e);
+                }
 
-                return fut;
+                cfg = cfg0;
             }
 
-            return old;
-        }
+            GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
 
-        if (ctx.clientDisconnected()) {
-            fut.onDone(new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
-                "Failed to deploy service, client node disconnected."));
+            GridServiceDeploymentFuture old = depFuts.putIfAbsent(cfg.getName(), fut);
 
-            depFuts.remove(cfg.getName(), fut);
-        }
+            if (old != null) {
+                if (!old.configuration().equalsIgnoreNodeFilter(cfg)) {
+                    fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
+                        "different configuration) [deployed=" + old.configuration() + ", new=" + cfg + ']'));
 
-        while (true) {
-            try {
-                GridServiceDeploymentKey key = new GridServiceDeploymentKey(cfg.getName());
+                    return fut;
+                }
+
+                return old;
+            }
 
-                if (ctx.deploy().enabled())
-                    ctx.cache().context().deploy().ignoreOwnership(true);
+            if (ctx.clientDisconnected()) {
+                fut.onDone(new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
+                    "Failed to deploy service, client node disconnected."));
 
+                depFuts.remove(cfg.getName(), fut);
+            }
+
+            while (true) {
                 try {
-                    GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
-                        new GridServiceDeployment(ctx.localNodeId(), cfg));
+                    GridServiceDeploymentKey key = new GridServiceDeploymentKey(cfg.getName());
 
-                    if (dep != null) {
-                        if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
-                            // Remove future from local map.
-                            depFuts.remove(cfg.getName(), fut);
+                    if (ctx.deploy().enabled())
+                        ctx.cache().context().deploy().ignoreOwnership(true);
 
-                            fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
-                                "different configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']'));
-                        }
-                        else {
-                            Iterator<Cache.Entry<Object, Object>> it = serviceEntries(
-                                ServiceAssignmentsPredicate.INSTANCE);
+                    try {
+                        GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
+                            new GridServiceDeployment(ctx.localNodeId(), cfg));
+
+                        if (dep != null) {
+                            if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
+                                // Remove future from local map.
+                                depFuts.remove(cfg.getName(), fut);
+
+                                fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
+                                    "different configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']'));
+                            }
+                            else {
+                                Iterator<Cache.Entry<Object, Object>> it = serviceEntries(
+                                    ServiceAssignmentsPredicate.INSTANCE);
 
-                            while (it.hasNext()) {
-                                Cache.Entry<Object, Object> e = it.next();
+                                while (it.hasNext()) {
+                                    Cache.Entry<Object, Object> e = it.next();
 
-                                if (e.getKey() instanceof GridServiceAssignmentsKey) {
-                                    GridServiceAssignments assigns = (GridServiceAssignments)e.getValue();
+                                    if (e.getKey() instanceof GridServiceAssignmentsKey) {
+                                        GridServiceAssignments assigns = (GridServiceAssignments)e.getValue();
 
-                                    if (assigns.name().equals(cfg.getName())) {
-                                        // Remove future from local map.
-                                        depFuts.remove(cfg.getName(), fut);
+                                        if (assigns.name().equals(cfg.getName())) {
+                                            // Remove future from local map.
+                                            depFuts.remove(cfg.getName(), fut);
 
-                                        fut.onDone();
+                                            fut.onDone();
 
-                                        break;
+                                            break;
+                                        }
                                     }
                                 }
-                            }
 
-                            if (!dep.configuration().equalsIgnoreNodeFilter(cfg))
-                                U.warn(log, "Service already deployed with different configuration (will ignore) " +
-                                    "[deployed=" + dep.configuration() + ", new=" + cfg + ']');
+                                if (!dep.configuration().equalsIgnoreNodeFilter(cfg))
+                                    U.warn(log, "Service already deployed with different configuration (will ignore) " +
+                                        "[deployed=" + dep.configuration() + ", new=" + cfg + ']');
+                            }
                         }
                     }
-                }
-                finally {
-                    if (ctx.deploy().enabled())
-                        ctx.cache().context().deploy().ignoreOwnership(false);
-                }
+                    finally {
+                        if (ctx.deploy().enabled())
+                            ctx.cache().context().deploy().ignoreOwnership(false);
+                    }
 
-                return fut;
-            }
-            catch (ClusterTopologyCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
-            }
-            catch (IgniteCheckedException e) {
-                if (e.hasCause(ClusterTopologyCheckedException.class)) {
+                    return fut;
+                }
+                catch (ClusterTopologyCheckedException e) {
                     if (log.isDebugEnabled())
                         log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
-
-                    continue;
                 }
+                catch (IgniteCheckedException e) {
+                    if (e.hasCause(ClusterTopologyCheckedException.class)) {
+                        if (log.isDebugEnabled())
+                            log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
 
-                U.error(log, "Failed to deploy service: " + cfg.getName(), e);
+                        continue;
+                    }
 
-                return new GridFinishedFuture<>(e);
+                    U.error(log, "Failed to deploy service: " + cfg.getName(), e);
+
+                    return new GridFinishedFuture<>(e);
+                }
             }
         }
+        finally {
+            busyLock.leaveBusy();
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
index ea0ba51b60954..563c3219f28da 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
@@ -21,6 +21,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
@@ -31,8 +32,12 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.lang.GridPlainCallable;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.services.ServiceContext;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -161,8 +166,138 @@ public void testStopDuringHangedDeployment() throws Exception {
         assertTrue("Deploy future isn't completed", wait);
 
         fut.get();
+    }
 
-        Ignition.stopAll(true);
+    /**
+     * @throws Exception If fails
+     */
+    public void testServiceDeploymentCancelationOnStop() throws Exception {
+        final Ignite node = startGrid(0);
+
+        final IgniteServices services = node.services();
+        // Deploy some service.
+        services.deploy(getServiceConfiguration("myService1"));
+
+        //Stop node async, this will cancel the service #1.
+        final IgniteInternalFuture<Boolean> stopAsync = GridTestUtils.runAsync(new GridPlainCallable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                node.close();
+
+                return true;
+            }
+        }, "node-stopping-thread");
+
+        // Wait for the service #1 cancellation during node stopping.
+        // At this point node.stop process will be paused until svcCancelFinishLatch released.
+        ServiceImpl.svcCancelStartLatch.await();
+
+        final AtomicReference<IgniteFuture> queuedFuture = new AtomicReference<>();
+
+        // Try to deploy another service.
+        final IgniteInternalFuture<?> deployAsync = GridTestUtils.runAsync(new GridPlainCallable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                IgniteServices async = services.withAsync();
+
+                async.deploy(getServiceConfiguration("myService2"));
+
+                IgniteFuture<Object> future = async.future();
+
+                queuedFuture.set(future);
+
+                // Here, deployment future is added to queue and
+                // then it will be cancelled when svcCancelFinishLatch be released.
+                // So, we'll wait for queue cleaning and try to deploy one more service.
+                try {
+                    future.get();
+                }
+                catch (Exception ignore) {
+                    // Noop.
+                }
+
+                // Normally, this should fail with some Exception as node is stopping right now.
+                // But we face a deadlock here.
+                for (int i = 0; i < 5; i++) {
+                    try {
+                        services.deploy(getServiceConfiguration("service3"));
+                    }
+                    catch (Exception ignore) {
+                        // Noop.
+                    }
+                }
+
+                return true;
+            }
+        }, "svc-deploy-thread");
+
+        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return queuedFuture.get() != null;
+            }
+        }, 3000);
+
+        // Allow node to be stopped.
+        ServiceImpl.svcCancelFinishLatch.countDown();
+
+        // Wait for all service deployments have finished.
+        boolean deployDone = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                IgniteFuture fut = queuedFuture.get();
+
+                return fut != null && fut.isDone() && deployAsync.isDone();
+
+            }
+        }, 5000);
+
+        assertTrue("Node stopping and service deployment processes falls into a deadlock.", deployDone);
+
+        if (!deployDone)
+            deployAsync.cancel();
+
+        if (!stopAsync.isDone())
+            stopAsync.cancel();
+    }
+
+    /** */
+    private ServiceConfiguration getServiceConfiguration(String svcName) {
+        ServiceConfiguration svcCfg = new ServiceConfiguration();
+        svcCfg.setName(svcName);
+        svcCfg.setService(new ServiceImpl());
+        svcCfg.setTotalCount(1);
+
+        return svcCfg;
+    }
+
+    /** Dummy Implementation. */
+    static class ServiceImpl implements Service {
+        /** */
+        static final CountDownLatch svcCancelStartLatch = new CountDownLatch(1);
+
+        /** */
+        static final CountDownLatch svcCancelFinishLatch = new CountDownLatch(1);
+
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+            System.out.println("cancel service: " + ctx.executionId());
+            try {
+                svcCancelStartLatch.countDown();
+
+                svcCancelFinishLatch.await();
+            }
+            catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(ServiceContext ctx) throws Exception {
+            System.out.println("init service: " + ctx.executionId());
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+            // No-op
+        }
     }
 
     /**

From 64c337c58501f2786298cc5b92a76b8d85f6f030 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 14 Sep 2017 20:05:43 +0300
Subject: [PATCH 420/446] Fixed flaky test
 IgniteCachePeekModesAbstractTest.testNonLocalPartitionSize. Fixed wrong swap
 entries counting.

---
 .../IgniteCachePeekModesAbstractTest.java     | 37 ++++++++++++++-----
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
index fac24ccd00045..75485cdeb0d5b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
@@ -996,23 +996,40 @@ private T2<Integer, Integer> swapKeysCount(int nodeIdx) {
      * @param part Cache partition
      * @return Tuple with number of primary and backup keys (one or both will be zero).
      */
-    private T2<Integer, Integer> swapKeysCount(int nodeIdx, int part) throws IgniteCheckedException {
+    private T2<Integer, Integer> swapKeysCount(final int nodeIdx, final int part) throws IgniteCheckedException {
         GridCacheContext ctx = ((IgniteEx)ignite(nodeIdx)).context().cache().internalCache().context();
         // Swap and offheap are disabled for near cache.
         GridCacheSwapManager swapMgr = ctx.isNear() ? ctx.near().dht().context().swap() : ctx.swap();
-        //First count entries...
-        int cnt = (int)swapMgr.swapEntriesCount(part);
 
-        GridCacheAffinityManager affinity = ctx.affinity();
-        AffinityTopologyVersion topVer = affinity.affinityTopologyVersion();
+        AffinityTopologyVersion topVer = ctx.affinity().affinityTopologyVersion();
+
+        Affinity aff = ignite(nodeIdx).affinity(null);
+
+        ClusterNode node = ignite(nodeIdx).cluster().localNode();
+
+        Iterator<KeyCacheObject> it = swapMgr.swapKeyIterator(true, true, topVer);
+
+        CacheObjectContext coctx = ((IgniteEx)ignite(nodeIdx)).context().cache().internalCache()
+            .context().cacheObjectContext();
 
-        //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
-            primaryCnt = cnt;
-        else if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
-            backupCnt = cnt;
+
+        while (it.hasNext()) {
+            Integer key = it.next().value(coctx, false);
+
+            if (part != aff.partition(key))
+                continue;
+
+            if (aff.isPrimary(node, key))
+                primaryCnt++;
+            else {
+                assertTrue(aff.isBackup(node, key));
+
+                backupCnt++;
+            }
+        }
+
         return new T2<>(primaryCnt, backupCnt);
     }
 

From 46296cf8150987c40346474f352c87d1e0fb1be0 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 15 Sep 2017 11:17:53 +0700
Subject: [PATCH 421/446] IGNITE-6367 Restrict files search by IGNITE_HOME.

---
 .../visor/file/VisorLatestTextFilesTask.java  | 15 +++---
 .../visor/log/VisorLogSearchTask.java         | 28 +++++------
 .../internal/visor/util/VisorTaskUtils.java   | 47 ++++++++++++++++---
 3 files changed, 61 insertions(+), 29 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
index 55d358c78dd6e..63b526e1a1b77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
@@ -18,13 +18,11 @@
 package org.apache.ignite.internal.visor.file;
 
 import java.io.File;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.internal.visor.log.VisorLogFile;
@@ -33,6 +31,7 @@
 
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.LOG_FILES_COUNT_LIMIT;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.matchedFiles;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveIgnitePath;
 
 /**
  * Get list files matching filter.
@@ -70,13 +69,11 @@ private VisorLatestTextFilesJob(IgniteBiTuple<String, String> arg, boolean debug
             assert path != null;
             assert regexp != null;
 
-            URL url = U.resolveIgniteUrl(path);
-
-            if (url == null)
-                return null;
-
             try {
-                File folder = new File(url.toURI());
+                File folder = resolveIgnitePath(path);
+
+                if (folder == null)
+                    return null;
 
                 List<VisorLogFile> files = matchedFiles(folder, regexp);
 
@@ -98,4 +95,4 @@ private VisorLatestTextFilesJob(IgniteBiTuple<String, String> arg, boolean debug
             return S.toString(VisorLatestTextFilesJob.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
index b6552b2fca5f6..44fb0a90a5b0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
@@ -18,10 +18,8 @@
 package org.apache.ignite.internal.visor.log;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
-import java.net.URL;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -35,7 +33,6 @@
 import org.apache.ignite.internal.util.io.GridReversedLinesFileReader;
 import org.apache.ignite.internal.util.lang.GridTuple3;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorMultiNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -43,6 +40,8 @@
 
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.decode;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.matchedFiles;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveIgnitePath;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveSymbolicLink;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.textFile;
 
 /**
@@ -216,21 +215,22 @@ private List<GridTuple3<String[], Integer, Integer>> searchInFile(File f, Charse
 
         /** {@inheritDoc} */
         @Override protected Collection<VisorLogSearchResult> run(VisorLogSearchArg arg) {
-            URL url = U.resolveIgniteUrl(arg.folder);
+            try {
+                File folder = resolveIgnitePath(arg.folder);
 
-            if (url == null)
-                throw new IgniteException(new FileNotFoundException("Log folder not found: " + arg.folder));
+                if (folder == null)
+                    return null;
 
-            UUID uuid = ignite.localNode().id();
-            String nid = uuid.toString().toLowerCase();
+                folder = resolveSymbolicLink(folder);
 
-            String filePtrn = arg.filePtrn.replace("@nid8", nid.substring(0, 8)).replace("@nid", nid);
+                UUID uuid = ignite.localNode().id();
+                String nid = uuid.toString().toLowerCase();
 
-            try {
-                File fld = new File(url.toURI());
-                int pathIdx = (fld.isDirectory() ? fld : fld.getParentFile()).getAbsolutePath().length() + 1;
+                String filePtrn = arg.filePtrn.replace("@nid8", nid.substring(0, 8)).replace("@nid", nid);
+
+                int pathIdx = (folder.isDirectory() ? folder : folder.getParentFile()).getAbsolutePath().length() + 1;
 
-                List<VisorLogFile> matchingFiles = matchedFiles(fld, filePtrn);
+                List<VisorLogFile> matchingFiles = matchedFiles(folder, filePtrn);
 
                 Collection<VisorLogSearchResult> results = new ArrayList<>(arg.limit);
 
@@ -276,4 +276,4 @@ private List<GridTuple3<String[], Integer, Integer>> searchInFile(File f, Charse
             return S.toString(VisorLogSearchJob.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
index 3f5003a03d401..7518d93e1aa8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
@@ -30,7 +30,10 @@
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -494,6 +497,34 @@ public static Collection<VisorGridEvent> collectEvents(Ignite ignite, String evt
         return res;
     }
 
+    /**
+     * @param path Path to resolve only relative to IGNITE_HOME.
+     * @return Resolved path as file, or {@code null} if path cannot be resolved.
+     * @throws IOException If failed to resolve path.
+     */
+    public static File resolveIgnitePath(String path) throws IOException {
+        File folder = U.resolveIgnitePath(path);
+
+        if (folder == null)
+            return null;
+
+        if (!folder.toPath().toRealPath(LinkOption.NOFOLLOW_LINKS).startsWith(Paths.get(U.getIgniteHome())))
+            return null;
+
+        return folder;
+    }
+
+    /**
+     * @param file File to resolve.
+     * @return Resolved file if it is a symbolic link or original file.
+     * @throws IOException If failed to resolve symlink.
+     */
+    public static File resolveSymbolicLink(File file) throws IOException {
+        Path path = file.toPath();
+
+        return Files.isSymbolicLink(path) ? Files.readSymbolicLink(path).toFile() : file;
+    }
+
     /**
      * Finds all files in folder and in it's sub-tree of specified depth.
      *
@@ -501,8 +532,11 @@ public static Collection<VisorGridEvent> collectEvents(Ignite ignite, String evt
      * @param maxDepth Depth of the tree. If 1 - just look in the folder, no sub-folders.
      * @param filter file filter.
      * @return List of found files.
+     * @throws IOException If failed to list files.
      */
-    public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) {
+    public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) throws IOException {
+        file = resolveSymbolicLink(file);
+
         if (file.isDirectory()) {
             File[] files = (filter == null) ? file.listFiles() : file.listFiles(filter);
 
@@ -525,12 +559,13 @@ else if (maxDepth > 1)
     }
 
     /**
-     * @param fld Folder with files to match.
+     * @param file Folder with files to match.
      * @param ptrn Pattern to match against file name.
      * @return Collection of matched files.
+     * @throws IOException If failed to filter files.
      */
-    public static List<VisorLogFile> matchedFiles(File fld, final String ptrn) {
-        List<VisorLogFile> files = fileTree(fld, MAX_FOLDER_DEPTH,
+    public static List<VisorLogFile> matchedFiles(File file, final String ptrn) throws IOException {
+        List<VisorLogFile> files = fileTree(file, MAX_FOLDER_DEPTH,
             new FileFilter() {
                 @Override public boolean accept(File f) {
                     return !f.isHidden() && (f.isDirectory() || f.isFile() && f.getName().matches(ptrn));
@@ -873,8 +908,6 @@ public static List<Process> startLocalNode(@Nullable IgniteLogger log, String cf
         if (cmdFilePath == null || !cmdFilePath.exists())
             throw new FileNotFoundException(String.format("File not found: %s", cmdFile));
 
-        String ignite = cmdFilePath.getCanonicalPath();
-
         File nodesCfgPath = U.resolveIgnitePath(cfgPath);
 
         if (nodesCfgPath == null || !nodesCfgPath.exists())
@@ -887,6 +920,8 @@ public static List<Process> startLocalNode(@Nullable IgniteLogger log, String cf
         List<Process> run = new ArrayList<>();
 
         try {
+            String ignite = cmdFilePath.getCanonicalPath();
+
             for (int i = 0; i < nodesToStart; i++) {
                 if (U.isMacOs()) {
                     Map<String, String> macEnv = new HashMap<>(System.getenv());

From 1d165d9c07fc8e13d6ee8d4e7b9cf1a83144e75e Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 15 Sep 2017 11:17:53 +0700
Subject: [PATCH 422/446] IGNITE-6367 Restrict files search by IGNITE_HOME.
 (cherry picked from commit 46296cf)

---
 .../visor/file/VisorLatestTextFilesTask.java  | 15 +++---
 .../visor/log/VisorLogSearchTask.java         | 28 +++++------
 .../internal/visor/util/VisorTaskUtils.java   | 47 ++++++++++++++++---
 3 files changed, 61 insertions(+), 29 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
index 55d358c78dd6e..63b526e1a1b77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
@@ -18,13 +18,11 @@
 package org.apache.ignite.internal.visor.file;
 
 import java.io.File;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.internal.visor.log.VisorLogFile;
@@ -33,6 +31,7 @@
 
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.LOG_FILES_COUNT_LIMIT;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.matchedFiles;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveIgnitePath;
 
 /**
  * Get list files matching filter.
@@ -70,13 +69,11 @@ private VisorLatestTextFilesJob(IgniteBiTuple<String, String> arg, boolean debug
             assert path != null;
             assert regexp != null;
 
-            URL url = U.resolveIgniteUrl(path);
-
-            if (url == null)
-                return null;
-
             try {
-                File folder = new File(url.toURI());
+                File folder = resolveIgnitePath(path);
+
+                if (folder == null)
+                    return null;
 
                 List<VisorLogFile> files = matchedFiles(folder, regexp);
 
@@ -98,4 +95,4 @@ private VisorLatestTextFilesJob(IgniteBiTuple<String, String> arg, boolean debug
             return S.toString(VisorLatestTextFilesJob.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
index b6552b2fca5f6..44fb0a90a5b0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
@@ -18,10 +18,8 @@
 package org.apache.ignite.internal.visor.log;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
-import java.net.URL;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -35,7 +33,6 @@
 import org.apache.ignite.internal.util.io.GridReversedLinesFileReader;
 import org.apache.ignite.internal.util.lang.GridTuple3;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorMultiNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -43,6 +40,8 @@
 
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.decode;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.matchedFiles;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveIgnitePath;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveSymbolicLink;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.textFile;
 
 /**
@@ -216,21 +215,22 @@ private List<GridTuple3<String[], Integer, Integer>> searchInFile(File f, Charse
 
         /** {@inheritDoc} */
         @Override protected Collection<VisorLogSearchResult> run(VisorLogSearchArg arg) {
-            URL url = U.resolveIgniteUrl(arg.folder);
+            try {
+                File folder = resolveIgnitePath(arg.folder);
 
-            if (url == null)
-                throw new IgniteException(new FileNotFoundException("Log folder not found: " + arg.folder));
+                if (folder == null)
+                    return null;
 
-            UUID uuid = ignite.localNode().id();
-            String nid = uuid.toString().toLowerCase();
+                folder = resolveSymbolicLink(folder);
 
-            String filePtrn = arg.filePtrn.replace("@nid8", nid.substring(0, 8)).replace("@nid", nid);
+                UUID uuid = ignite.localNode().id();
+                String nid = uuid.toString().toLowerCase();
 
-            try {
-                File fld = new File(url.toURI());
-                int pathIdx = (fld.isDirectory() ? fld : fld.getParentFile()).getAbsolutePath().length() + 1;
+                String filePtrn = arg.filePtrn.replace("@nid8", nid.substring(0, 8)).replace("@nid", nid);
+
+                int pathIdx = (folder.isDirectory() ? folder : folder.getParentFile()).getAbsolutePath().length() + 1;
 
-                List<VisorLogFile> matchingFiles = matchedFiles(fld, filePtrn);
+                List<VisorLogFile> matchingFiles = matchedFiles(folder, filePtrn);
 
                 Collection<VisorLogSearchResult> results = new ArrayList<>(arg.limit);
 
@@ -276,4 +276,4 @@ private List<GridTuple3<String[], Integer, Integer>> searchInFile(File f, Charse
             return S.toString(VisorLogSearchJob.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
index 3f5003a03d401..7518d93e1aa8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
@@ -30,7 +30,10 @@
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -494,6 +497,34 @@ public static Collection<VisorGridEvent> collectEvents(Ignite ignite, String evt
         return res;
     }
 
+    /**
+     * @param path Path to resolve only relative to IGNITE_HOME.
+     * @return Resolved path as file, or {@code null} if path cannot be resolved.
+     * @throws IOException If failed to resolve path.
+     */
+    public static File resolveIgnitePath(String path) throws IOException {
+        File folder = U.resolveIgnitePath(path);
+
+        if (folder == null)
+            return null;
+
+        if (!folder.toPath().toRealPath(LinkOption.NOFOLLOW_LINKS).startsWith(Paths.get(U.getIgniteHome())))
+            return null;
+
+        return folder;
+    }
+
+    /**
+     * @param file File to resolve.
+     * @return Resolved file if it is a symbolic link or original file.
+     * @throws IOException If failed to resolve symlink.
+     */
+    public static File resolveSymbolicLink(File file) throws IOException {
+        Path path = file.toPath();
+
+        return Files.isSymbolicLink(path) ? Files.readSymbolicLink(path).toFile() : file;
+    }
+
     /**
      * Finds all files in folder and in it's sub-tree of specified depth.
      *
@@ -501,8 +532,11 @@ public static Collection<VisorGridEvent> collectEvents(Ignite ignite, String evt
      * @param maxDepth Depth of the tree. If 1 - just look in the folder, no sub-folders.
      * @param filter file filter.
      * @return List of found files.
+     * @throws IOException If failed to list files.
      */
-    public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) {
+    public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) throws IOException {
+        file = resolveSymbolicLink(file);
+
         if (file.isDirectory()) {
             File[] files = (filter == null) ? file.listFiles() : file.listFiles(filter);
 
@@ -525,12 +559,13 @@ else if (maxDepth > 1)
     }
 
     /**
-     * @param fld Folder with files to match.
+     * @param file Folder with files to match.
      * @param ptrn Pattern to match against file name.
      * @return Collection of matched files.
+     * @throws IOException If failed to filter files.
      */
-    public static List<VisorLogFile> matchedFiles(File fld, final String ptrn) {
-        List<VisorLogFile> files = fileTree(fld, MAX_FOLDER_DEPTH,
+    public static List<VisorLogFile> matchedFiles(File file, final String ptrn) throws IOException {
+        List<VisorLogFile> files = fileTree(file, MAX_FOLDER_DEPTH,
             new FileFilter() {
                 @Override public boolean accept(File f) {
                     return !f.isHidden() && (f.isDirectory() || f.isFile() && f.getName().matches(ptrn));
@@ -873,8 +908,6 @@ public static List<Process> startLocalNode(@Nullable IgniteLogger log, String cf
         if (cmdFilePath == null || !cmdFilePath.exists())
             throw new FileNotFoundException(String.format("File not found: %s", cmdFile));
 
-        String ignite = cmdFilePath.getCanonicalPath();
-
         File nodesCfgPath = U.resolveIgnitePath(cfgPath);
 
         if (nodesCfgPath == null || !nodesCfgPath.exists())
@@ -887,6 +920,8 @@ public static List<Process> startLocalNode(@Nullable IgniteLogger log, String cf
         List<Process> run = new ArrayList<>();
 
         try {
+            String ignite = cmdFilePath.getCanonicalPath();
+
             for (int i = 0; i < nodesToStart; i++) {
                 if (U.isMacOs()) {
                     Map<String, String> macEnv = new HashMap<>(System.getenv());

From a85a42d495bf449b7786fd9a8362be54b1fd2eea Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dgovorukhin@gridgain.com>
Date: Mon, 23 Jan 2017 20:35:02 +0300
Subject: [PATCH 423/446] Bakport fix for flacky test
 GridServiceProcessorProxySelfTest.testLocalProxyInvocation.

(cherry picked from commit 3587ac6)
---
 .../GridServiceProcessorProxySelfTest.java    | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
index 850cb5702b5b0..1418973940b41 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
@@ -20,9 +20,11 @@
 import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceContext;
@@ -240,12 +242,27 @@ public void testLocalProxyInvocation() throws Exception {
         ignite.services().deployNodeSingleton(name, new MapServiceImpl<String, Integer>());
 
         for (int i = 0; i < nodeCount(); i++) {
-            MapService<Integer, String> svc =  grid(i).services().serviceProxy(name, MapService.class, false);
+            final int idx = i;
+
+            final AtomicReference< MapService<Integer, String>> ref = new AtomicReference<>();
+
+            //wait because after deployNodeSingleton we don't have guarantees what service was deploy.
+            boolean wait = GridTestUtils.waitForCondition(new PA() {
+                @Override public boolean apply() {
+                    MapService<Integer, String> svc = grid(idx)
+                        .services()
+                        .serviceProxy(name, MapService.class, false);
+
+                    ref.set(svc);
+
+                    return svc instanceof Service;
+                }
+            }, 2000);
 
             // Make sure service is a local instance.
-            assertTrue("Invalid service instance [srv=" + svc + ", node=" + i + ']', svc instanceof Service);
+            assertTrue("Invalid service instance [srv=" + ref.get() + ", node=" + i + ']', wait);
 
-            svc.put(i, Integer.toString(i));
+            ref.get().put(i, Integer.toString(i));
         }
 
         MapService<Integer, String> map = ignite.services().serviceProxy(name, MapService.class, false);

From 238c7723e95dfed8bbd69bf26c98011490780a68 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 18 Sep 2017 12:54:33 +0300
Subject: [PATCH 424/446] GG-12717: Backport fix CacheJdbcPojoStoreTest from
 master.

---
 .../store/jdbc/CacheAbstractJdbcStore.java    |  67 +++++----
 .../cache/store/CacheOsStoreManager.java      |   3 +
 .../store/GridCacheStoreManagerAdapter.java   |   8 ++
 .../CacheJdbcPojoStoreAbstractSelfTest.java   |  19 ++-
 ...naryMarshallerStoreKeepBinarySelfTest.java |  28 ++++
 ...rStoreKeepBinaryWithSqlEscapeSelfTest.java |  28 ++++
 .../store/jdbc/CacheJdbcPojoStoreTest.java    | 136 ++++++++++++++----
 .../testframework/junits/IgniteMock.java      |   4 +
 .../testsuites/IgniteCacheTestSuite.java      |   4 +
 9 files changed, 237 insertions(+), 60 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
index 817b1a5da8531..64bdec90f5296 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
@@ -61,6 +61,7 @@
 import org.apache.ignite.cache.store.jdbc.dialect.OracleDialect;
 import org.apache.ignite.cache.store.jdbc.dialect.SQLServerDialect;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.binary.BinaryEnumObjectImpl;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
@@ -560,22 +561,35 @@ private JdbcTypeField[] translateFields(Collection<CacheTypeFieldMetadata> oldFl
 
     /**
      * @param type Type name to check.
+     * @param binarySupported True if binary marshaller enable.
      * @return {@code True} if class not found.
      */
-    protected TypeKind kindForName(String type) {
+    protected TypeKind kindForName(String type, boolean binarySupported) {
         if (BUILT_IN_TYPES.contains(type))
             return TypeKind.BUILT_IN;
 
+        if (binarySupported)
+            return TypeKind.BINARY;
+
         try {
             Class.forName(type);
 
             return TypeKind.POJO;
         }
-        catch(ClassNotFoundException ignored) {
-            return TypeKind.BINARY;
+        catch (ClassNotFoundException e) {
+            throw new CacheException("Failed to find class " + type +
+                " (make sure the class is present in classPath or use BinaryMarshaller)", e);
         }
     }
 
+    /**
+     * @param type Type name to check.
+     * @return {@code True} if class not found.
+     */
+    protected TypeKind kindForName(String type) {
+        return kindForName(type, ignite.configuration().getMarshaller() instanceof BinaryMarshaller);
+    }
+
     /**
      * @param cacheName Cache name to check mappings for.
      * @return Type mappings for specified cache name.
@@ -640,11 +654,7 @@ private Map<Object, EntryMapping> getOrCreateCacheMappings(@Nullable String cach
                     String keyType = type.getKeyType();
                     String valType = type.getValueType();
 
-                    TypeKind keyKind = kindForName(keyType);
-
-                    if (!binarySupported && keyKind == TypeKind.BINARY)
-                        throw new CacheException("Key type has no class [cache=" + U.maskName(cacheName) +
-                            ", type=" + keyType + "]");
+                    TypeKind keyKind = kindForName(keyType, binarySupported);
 
                     checkTypeConfiguration(cacheName, keyKind, keyType, type.getKeyFields());
 
@@ -654,21 +664,11 @@ private Map<Object, EntryMapping> getOrCreateCacheMappings(@Nullable String cach
                         throw new CacheException("Key type must be unique in type metadata [cache=" +
                             U.maskName(cacheName) + ", type=" + keyType + "]");
 
-                    TypeKind valKind = kindForName(valType);
+                    TypeKind valKind = kindForName(valType, binarySupported);
 
                     checkTypeConfiguration(cacheName, valKind, valType, type.getValueFields());
 
                     entryMappings.put(keyTypeId, new EntryMapping(cacheName, dialect, type, keyKind, valKind, sqlEscapeAll));
-
-                    // Add one more binding to binary typeId for POJOs,
-                    // because object could be passed to store in binary format.
-                    if (binarySupported && keyKind == TypeKind.POJO) {
-                        keyTypeId = typeIdForTypeName(TypeKind.BINARY, keyType);
-
-                        valKind = valKind == TypeKind.POJO ? TypeKind.BINARY : valKind;
-
-                        entryMappings.put(keyTypeId, new EntryMapping(cacheName, dialect, type, TypeKind.BINARY, valKind, sqlEscapeAll));
-                    }
                 }
 
                 Map<String, Map<Object, EntryMapping>> mappings = new HashMap<>(cacheMappings);
@@ -1419,10 +1419,17 @@ protected void fillParameter(PreparedStatement stmt, int idx, JdbcTypeField fiel
                             // No-op.
                     }
                 }
-                else if (field.getJavaFieldType().isEnum() && fieldVal instanceof Enum) {
-                    Enum val = (Enum)fieldVal;
-                    
-                    fieldVal = NUMERIC_TYPES.contains(field.getDatabaseFieldType()) ? val.ordinal() : val.name();
+                else if (field.getJavaFieldType().isEnum()) {
+                    if (fieldVal instanceof Enum) {
+                        Enum val = (Enum)fieldVal;
+
+                        fieldVal = NUMERIC_TYPES.contains(field.getDatabaseFieldType()) ? val.ordinal() : val.name();
+                    }
+                    else if (fieldVal instanceof BinaryEnumObjectImpl) {
+                        BinaryEnumObjectImpl val = (BinaryEnumObjectImpl)fieldVal;
+
+                        fieldVal = val.enumOrdinal();
+                    }
                 }
 
                 stmt.setObject(idx, fieldVal);
@@ -1474,14 +1481,14 @@ protected int fillKeyParameters(PreparedStatement stmt, EntryMapping m, Object k
      */
     protected int fillValueParameters(PreparedStatement stmt, int idx, EntryMapping em, Object val)
         throws CacheWriterException {
-        TypeKind valKind = em.valueKind();
-
-        // Object could be passed by cache in binary format in case of cache configured with setStoreKeepBinary(true).
-        if (valKind == TypeKind.POJO && val instanceof BinaryObject)
-            valKind = TypeKind.BINARY;
-
         for (JdbcTypeField field : em.uniqValFlds) {
-            Object fieldVal = extractParameter(em.cacheName, em.valueType(), valKind, field.getJavaFieldName(), val);
+            Object fieldVal = extractParameter(
+                em.cacheName,
+                em.valueType(),
+                em.valueKind(),
+                field.getJavaFieldName(),
+                val
+            );
 
             fillParameter(stmt, idx++, field, fieldVal);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java
index 27771ff3e629a..2e23d04f735d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java
@@ -77,6 +77,9 @@ public CacheOsStoreManager(GridKernalContext ctx, CacheConfiguration cfg) {
 
     /** {@inheritDoc} */
     @Override public boolean convertBinary() {
+        if (alwaysKeepBinary)
+            return false;
+
         return configuredConvertBinary() && !(cfgStore instanceof PlatformCacheStore);
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 6ab8c56578593..5d5c7d667c94e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -33,9 +33,11 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cache.store.CacheStoreSession;
 import org.apache.ignite.cache.store.CacheStoreSessionListener;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
@@ -113,6 +115,9 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
     /** */
     private boolean globalSesLsnrs;
 
+    /** Always keep binary. */
+    protected boolean alwaysKeepBinary;
+
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public void initialize(@Nullable CacheStore cfgStore, Map sesHolders) throws IgniteCheckedException {
@@ -148,6 +153,9 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
         sesHolder = sesHolder0;
 
         locStore = U.hasAnnotation(cfgStore, CacheLocalStore.class);
+
+        if (cfgStore instanceof CacheJdbcPojoStore)
+            alwaysKeepBinary = true;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
index 9e59769503f53..5ca6ca45364f8 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ConnectorConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -221,6 +222,7 @@ protected CacheConfiguration cacheConfiguration() throws Exception {
         cc.setAtomicityMode(transactional ? TRANSACTIONAL : ATOMIC);
         cc.setSwapEnabled(false);
         cc.setWriteBehindEnabled(false);
+        cc.setStoreKeepBinary(storeKeepBinary());
 
         CacheJdbcPojoStoreFactory<Object, Object> storeFactory = new CacheJdbcPojoStoreFactory<>();
         storeFactory.setDialect(new H2Dialect());
@@ -237,6 +239,13 @@ protected CacheConfiguration cacheConfiguration() throws Exception {
         return cc;
     }
 
+    /**
+     * @return Flag indicate keep value in binary format or not.
+     */
+    protected boolean storeKeepBinary(){
+        return false;
+    }
+
     /**
      * Fill in-memory database with sample data.
      *
@@ -446,6 +455,8 @@ public void testLoadCachePrimitiveKeysTx() throws Exception {
      * @throws Exception If failed.
      */
     private void checkPutRemove() throws Exception {
+        boolean binaryMarshaller = marshaller() instanceof BinaryMarshaller || marshaller() == null;
+
         IgniteCache<Object, Person> c1 = grid().cache(CACHE_NAME);
 
         Connection conn = getConnection();
@@ -480,7 +491,9 @@ private void checkPutRemove() throws Exception {
             assertEquals(-2, rs.getInt(2));
             assertEquals(testDate, rs.getDate(3));
             assertEquals("Person-to-test-put-insert", rs.getString(4));
-            assertEquals(testGender.toString(), rs.getString(5));
+
+            assertEquals(testGender.toString(),
+                binaryMarshaller ? Gender.values()[rs.getInt(5)].toString(): rs.getString(5));
 
             assertFalse("Unexpected more data in result set", rs.next());
 
@@ -499,7 +512,9 @@ private void checkPutRemove() throws Exception {
             assertEquals(-3, rs.getInt(2));
             assertEquals(testDate, rs.getDate(3));
             assertEquals("Person-to-test-put-update", rs.getString(4));
-            assertEquals(testGender.toString(), rs.getString(5));
+
+            assertEquals(testGender.toString(),
+                binaryMarshaller ? Gender.values()[rs.getInt(5)].toString(): rs.getString(5));
 
             assertFalse("Unexpected more data in result set", rs.next());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java
new file mode 100644
index 0000000000000..dfca8647e2840
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.jdbc;
+
+/**
+ *
+ */
+public class CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest extends CacheJdbcPojoStoreBinaryMarshallerSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean storeKeepBinary() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java
new file mode 100644
index 0000000000000..c7e1f794ab208
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.jdbc;
+
+/**
+ *
+ */
+public class CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest extends CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean storeKeepBinary() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
index 849cab79a03be..f17ece4e59686 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.cache.store.jdbc;
 
+import java.lang.reflect.Field;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
@@ -29,18 +30,22 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import javax.cache.integration.CacheWriterException;
-
+import org.apache.ignite.Ignite;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
 import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect;
 import org.apache.ignite.cache.store.jdbc.model.Organization;
 import org.apache.ignite.cache.store.jdbc.model.OrganizationKey;
 import org.apache.ignite.cache.store.jdbc.model.Person;
 import org.apache.ignite.cache.store.jdbc.model.PersonComplexKey;
 import org.apache.ignite.cache.store.jdbc.model.PersonKey;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.config.GridTestProperties;
 import org.apache.ignite.testframework.junits.cache.GridAbstractCacheStoreSelfTest;
 import org.h2.jdbcx.JdbcConnectionPool;
 
@@ -57,6 +62,12 @@ public class CacheJdbcPojoStoreTest extends GridAbstractCacheStoreSelfTest<Cache
     /** Person count. */
     protected static final int PERSON_CNT = 100000;
 
+    /** Ignite. */
+    private Ignite ig;
+
+    /** Binary enable. */
+    private boolean binaryEnable;
+
     /**
      * @throws Exception If failed.
      */
@@ -226,6 +237,12 @@ public CacheJdbcPojoStoreTest() throws Exception {
         U.closeQuiet(conn);
 
         super.beforeTest();
+
+        Ignite ig = U.field(store, "ignite");
+
+        this.ig = ig;
+
+        binaryEnable = ig.configuration().getMarshaller() instanceof BinaryMarshaller;
     }
 
     /**
@@ -292,26 +309,48 @@ public void testLoadCache() throws Exception {
 
         U.closeQuiet(conn);
 
-        final Collection<OrganizationKey> orgKeys = new ConcurrentLinkedQueue<>();
-        final Collection<PersonKey> prnKeys = new ConcurrentLinkedQueue<>();
-        final Collection<PersonComplexKey> prnComplexKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> orgKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> prnKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> prnComplexKeys = new ConcurrentLinkedQueue<>();
 
         IgniteBiInClosure<Object, Object> c = new CI2<Object, Object>() {
             @Override public void apply(Object k, Object v) {
-                if (k instanceof OrganizationKey && v instanceof Organization)
-                    orgKeys.add((OrganizationKey)k);
-                else if (k instanceof PersonKey && v instanceof Person)
-                    prnKeys.add((PersonKey)k);
-                else if (k instanceof PersonComplexKey && v instanceof Person) {
-                    PersonComplexKey key = (PersonComplexKey)k;
-
-                    Person val = (Person)v;
-
-                    assertTrue("Key ID should be the same as value ID", key.getId() == val.getId());
-                    assertTrue("Key orgID should be the same as value orgID", key.getOrgId() == val.getOrgId());
-                    assertEquals("name" + key.getId(), val.getName());
-
-                    prnComplexKeys.add((PersonComplexKey)k);
+                if (binaryEnable){
+                    if (k instanceof BinaryObject && v instanceof BinaryObject) {
+                        BinaryObject key = (BinaryObject)k;
+                        BinaryObject val = (BinaryObject)v;
+
+                        String keyType = key.type().typeName();
+                        String valType = val.type().typeName();
+
+                        if (OrganizationKey.class.getName().equals(keyType)
+                            && Organization.class.getName().equals(valType))
+                            orgKeys.add(key);
+
+                        if (PersonKey.class.getName().equals(keyType)
+                            && Person.class.getName().equals(valType))
+                            prnKeys.add(key);
+
+                        if (PersonComplexKey.class.getName().equals(keyType)
+                            && Person.class.getName().equals(valType))
+                            prnComplexKeys.add(key);
+                    }
+                }else {
+                    if (k instanceof OrganizationKey && v instanceof Organization)
+                        orgKeys.add(k);
+                    else if (k instanceof PersonKey && v instanceof Person)
+                        prnKeys.add(k);
+                    else if (k instanceof PersonComplexKey && v instanceof Person) {
+                        PersonComplexKey key = (PersonComplexKey)k;
+
+                        Person val = (Person)v;
+
+                        assertTrue("Key ID should be the same as value ID", key.getId() == val.getId());
+                        assertTrue("Key orgID should be the same as value orgID", key.getOrgId() == val.getOrgId());
+                        assertEquals("name" + key.getId(), val.getName());
+
+                        prnComplexKeys.add(k);
+                    }
                 }
             }
         };
@@ -322,15 +361,16 @@ else if (k instanceof PersonComplexKey && v instanceof Person) {
         assertEquals(PERSON_CNT, prnKeys.size());
         assertEquals(PERSON_CNT, prnComplexKeys.size());
 
-        Collection<OrganizationKey> tmpOrgKeys = new ArrayList<>(orgKeys);
-        Collection<PersonKey> tmpPrnKeys = new ArrayList<>(prnKeys);
-        Collection<PersonComplexKey> tmpPrnComplexKeys = new ArrayList<>(prnComplexKeys);
+        Collection<Object> tmpOrgKeys = new ArrayList<>(orgKeys);
+        Collection<Object> tmpPrnKeys = new ArrayList<>(prnKeys);
+        Collection<Object> tmpPrnComplexKeys = new ArrayList<>(prnComplexKeys);
 
         orgKeys.clear();
         prnKeys.clear();
         prnComplexKeys.clear();
 
-        store.loadCache(c, OrganizationKey.class.getName(), "SELECT name, city, id FROM ORGANIZATION",
+        store.loadCache(
+            c, OrganizationKey.class.getName(), "SELECT name, city, id FROM ORGANIZATION",
             PersonKey.class.getName(), "SELECT org_id, id, name FROM Person WHERE id < 1000");
 
         assertEquals(ORGANIZATION_CNT, orgKeys.size());
@@ -380,14 +420,29 @@ public void testParallelLoad() throws Exception {
 
         U.closeQuiet(conn);
 
-        final Collection<PersonComplexKey> prnComplexKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> prnComplexKeys = new ConcurrentLinkedQueue<>();
 
         IgniteBiInClosure<Object, Object> c = new CI2<Object, Object>() {
             @Override public void apply(Object k, Object v) {
-                if (k instanceof PersonComplexKey && v instanceof Person)
-                    prnComplexKeys.add((PersonComplexKey)k);
-                else
-                    fail("Unexpected entry [key=" + k + ", value=" + v + "]");
+                if (binaryEnable) {
+                    if (k instanceof BinaryObject && v instanceof BinaryObject) {
+                        BinaryObject key = (BinaryObject)k;
+                        BinaryObject val = (BinaryObject)v;
+
+                        String keyType = key.type().typeName();
+                        String valType = val.type().typeName();
+
+                        if (PersonComplexKey.class.getName().equals(keyType)
+                            && Person.class.getName().equals(valType))
+                            prnComplexKeys.add(key);
+                    }
+                }
+                else {
+                    if (k instanceof PersonComplexKey && v instanceof Person)
+                        prnComplexKeys.add(k);
+                    else
+                        fail("Unexpected entry [key=" + k + ", value=" + v + "]");
+                }
             }
         };
 
@@ -440,7 +495,7 @@ public void testWriteRetry() throws Exception {
         ses.newSession(null);
 
         try {
-            store.write(new CacheEntryImpl<>(k1, v1));
+            store.write(new CacheEntryImpl<>(wrap(k1), wrap(v1)));
 
             fail("CacheWriterException wasn't thrown.");
         }
@@ -469,4 +524,29 @@ public void testTimestamp() throws Exception {
 
         assertNull(store.load(k));
     }
+
+    /**
+     * @param obj Object.
+     */
+    private Object wrap(Object obj) throws IllegalAccessException {
+        if (binaryEnable) {
+            Class<?> cls = obj.getClass();
+
+            BinaryObjectBuilder builder = ig.binary().builder(cls.getName());
+
+            for (Field f : cls.getDeclaredFields()) {
+                if (f.getName().contains("serialVersionUID"))
+                    continue;
+
+                f.setAccessible(true);
+
+                builder.setField(f.getName(), f.get(obj));
+            }
+
+            return builder.build();
+        }
+
+        return obj;
+    }
+
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
index 5722fa3d6056a..14ba030e24590 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.binary.BinaryCachingMetadataHandler;
 import org.apache.ignite.internal.binary.BinaryContext;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl;
 import org.apache.ignite.internal.processors.cacheobject.NoOpBinary;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -327,6 +328,9 @@ public IgniteMock(
                     return typeName.hashCode();
                 }
             };
+
+            if (marshaller instanceof BinaryMarshaller)
+                ctx.configure((BinaryMarshaller)marshaller, configuration());
         }
 
         binaryMock = new NoOpBinary() {
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index deec72af95cd8..dce93747f12bd 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -32,6 +32,8 @@
 import org.apache.ignite.cache.store.GridCacheLoadOnlyStoreAdapterSelfTest;
 import org.apache.ignite.cache.store.StoreResourceInjectionSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerSelfTest;
+import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest;
+import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreMultitreadedSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreOptimizedMarshallerSelfTest;
@@ -252,7 +254,9 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(CacheJdbcPojoStoreOptimizedMarshallerSelfTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreOptimizedMarshallerWithSqlEscapeSelfTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerSelfTest.class);
+        suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest.class);
+        suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreMultitreadedSelfTest.class);
         suite.addTestSuite(GridCacheBalancingStoreSelfTest.class);
         suite.addTestSuite(GridCacheAffinityApiSelfTest.class);

From ec6c1194f941c7818656a777d9fe99bb70db58d6 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 15 Sep 2017 14:54:07 +0300
Subject: [PATCH 425/446] ignite-3495: metrics for atomic caches

---
 .../processors/cache/GridCacheAdapter.java    | 258 ++++------
 .../distributed/dht/GridDhtCacheAdapter.java  |  26 +
 .../dht/GridPartitionedGetFuture.java         |  21 +-
 .../dht/GridPartitionedSingleGetFuture.java   |   8 +
 .../dht/atomic/GridDhtAtomicCache.java        |  33 +-
 .../GridNearAtomicAbstractUpdateFuture.java   |  25 +
 .../atomic/GridNearAtomicUpdateFuture.java    |  25 +
 .../dht/colocated/GridDhtColocatedCache.java  |  19 +-
 .../distributed/near/GridNearGetFuture.java   |   8 +
 .../cache/local/GridLocalCache.java           |  42 ++
 .../local/atomic/GridLocalAtomicCache.java    |  78 ++-
 ...acheAtomicClientServerMetricsSelfTest.java | 452 ++++++++++++++++++
 .../IgniteCacheMetricsSelfTestSuite.java      |   4 +-
 13 files changed, 816 insertions(+), 183 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicClientServerMetricsSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index ff1850dbcfe39..74415d2733f43 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1406,10 +1406,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Nullable @Override public V get(K key) throws IgniteCheckedException {
         A.notNull(key, "key");
 
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         boolean keepBinary = ctx.keepBinary();
 
         if (keepBinary)
@@ -1423,9 +1419,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
             val = (V)ctx.config().getInterceptor().onGet(key, val);
         }
 
-        if (statsEnabled)
-            metrics0().addGetTimeNanos(System.nanoTime() - start);
-
         return val;
     }
 
@@ -1433,10 +1426,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Nullable @Override public CacheEntry<K, V> getEntry(K key) throws IgniteCheckedException {
         A.notNull(key, "key");
 
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         boolean keepBinary = ctx.keepBinary();
 
         if (keepBinary)
@@ -1459,9 +1448,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
             val = (val0 != null) ? new CacheEntryImplEx<>(key, val0, t != null ? t.version() : null) : null;
         }
 
-        if (statsEnabled)
-            metrics0().addGetTimeNanos(System.nanoTime() - start);
-
         return val;
     }
 
@@ -1469,10 +1455,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Override public IgniteInternalFuture<V> getAsync(final K key) {
         A.notNull(key, "key");
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         final boolean keepBinary = ctx.keepBinary();
 
         final K key0 = keepBinary ? (K)ctx.toCacheKeyObject(key) : key;
@@ -1488,9 +1470,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
                 }
             });
 
-        if (statsEnabled)
-            fut.listen(new UpdateGetTimeStatClosure<V>(metrics0(), start));
-
         return fut;
     }
 
@@ -1498,10 +1477,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Override public IgniteInternalFuture<CacheEntry<K, V>> getEntryAsync(final K key) {
         A.notNull(key, "key");
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         final boolean keepBinary = ctx.keepBinary();
 
         final K key0 = keepBinary ? (K)ctx.toCacheKeyObject(key) : key;
@@ -1535,9 +1510,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
             }
         });
 
-        if (statsEnabled)
-            fut.listen(new UpdateGetTimeStatClosure<EntryGetResult>(metrics0(), start));
-
         return fr;
     }
 
@@ -1545,18 +1517,11 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Override public final Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         Map<K, V> map = getAll0(keys, !ctx.keepBinary(), false);
 
         if (ctx.config().getInterceptor() != null)
             map = interceptGet(keys, map);
 
-        if (statsEnabled)
-            metrics0().addGetTimeNanos(System.nanoTime() - start);
-
         return map;
     }
 
@@ -1565,22 +1530,16 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
         throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         Map<K, EntryGetResult> map = (Map<K, EntryGetResult>)getAll0(keys, !ctx.keepBinary(), true);
 
         Collection<CacheEntry<K, V>> res = new HashSet<>();
 
         if (ctx.config().getInterceptor() != null)
             res = interceptGetEntries(keys, map);
-        else
+        else {
             for (Map.Entry<K, EntryGetResult> e : map.entrySet())
                 res.add(new CacheEntryImplEx<>(e.getKey(), (V)e.getValue().value(), e.getValue().version()));
-
-        if (statsEnabled)
-            metrics0().addGetTimeNanos(System.nanoTime() - start);
+        }
 
         return res;
     }
@@ -1589,10 +1548,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Override public IgniteInternalFuture<Map<K, V>> getAllAsync(@Nullable final Collection<? extends K> keys) {
         A.notNull(keys, "keys");
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         IgniteInternalFuture<Map<K, V>> fut = getAllAsync(keys, !ctx.keepBinary(), false);
 
         if (ctx.config().getInterceptor() != null)
@@ -1602,9 +1557,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
                 }
             });
 
-        if (statsEnabled)
-            fut.listen(new UpdateGetTimeStatClosure<Map<K, V>>(metrics0(), start));
-
         return fut;
     }
 
@@ -1613,10 +1565,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
         @Nullable final Collection<? extends K> keys) {
         A.notNull(keys, "keys");
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         IgniteInternalFuture<Map<K, EntryGetResult>> fut =
             (IgniteInternalFuture<Map<K, EntryGetResult>>)
                 ((IgniteInternalFuture)getAllAsync(keys, !ctx.keepBinary(), true));
@@ -1641,9 +1589,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
                 }
             });
 
-        if (statsEnabled)
-            fut.listen(new UpdateGetTimeStatClosure<Map<K, EntryGetResult>>(metrics0(), start));
-
         return rf;
     }
 
@@ -2212,21 +2157,12 @@ private void clearReservationsIfNeeded(
      */
     @Nullable public V getAndPut(final K key, final V val, @Nullable final CacheEntryPredicate filter)
         throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key", val, "val");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        V prevVal = getAndPut0(key, val, filter);
-
-        if (statsEnabled)
-            metrics0().addPutAndGetTimeNanos(System.nanoTime() - start);
-
-        return prevVal;
+        return getAndPut0(key, val, filter);
     }
 
     /**
@@ -2261,21 +2197,12 @@ protected V getAndPut0(final K key, final V val, @Nullable final CacheEntryPredi
      * @return Put operation future.
      */
     protected final IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key", val, "val");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<V> fut = getAndPutAsync0(key, val, filter);
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutAndGetTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
+        return getAndPutAsync0(key, val, filter);
     }
 
     /**
@@ -2288,7 +2215,11 @@ public IgniteInternalFuture<V> getAndPutAsync0(final K key,
         final V val,
         @Nullable final CacheEntryPredicate filter)
     {
-        return asyncOp(new AsyncOp<V>() {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
             @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAsync(ctx, readyTopVer, key, val, true, filter)
                     .chain((IgniteClosure<IgniteInternalFuture<GridCacheReturn>, V>)RET2VAL);
@@ -2298,6 +2229,11 @@ public IgniteInternalFuture<V> getAndPutAsync0(final K key,
                 return "putAsync [key=" + key + ", val=" + val + ", filter=" + filter + ']';
             }
         });
+
+        if (statsEnabled)
+            fut.listen(new UpdatePutAndGetTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -2316,21 +2252,12 @@ public IgniteInternalFuture<V> getAndPutAsync0(final K key,
      */
     public boolean put(final K key, final V val, final CacheEntryPredicate filter)
         throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key", val, "val");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        boolean stored = put0(key, val, filter);
-
-        if (statsEnabled && stored)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
-
-        return stored;
+        return put0(key, val, filter);
     }
 
     /**
@@ -2344,6 +2271,10 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
      */
     protected boolean put0(final K key, final V val, final CacheEntryPredicate filter)
         throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         Boolean res = syncOp(new SyncOp<Boolean>(true) {
             @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return tx.putAsync(ctx, null, key, val, false, filter).get().success();
@@ -2356,6 +2287,9 @@ protected boolean put0(final K key, final V val, final CacheEntryPredicate filte
 
         assert res != null;
 
+        if (statsEnabled && Boolean.TRUE.equals(res))
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
+
         return res;
     }
 
@@ -2647,16 +2581,7 @@ public final IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable Cach
         if (keyCheck)
             validateCacheKey(key);
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        IgniteInternalFuture<Boolean> fut = putAsync0(key, val, filter);
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+        return putAsync0(key, val, filter);
     }
 
     /**
@@ -2665,9 +2590,12 @@ public final IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable Cach
      * @param filter Optional filter.
      * @return Putx operation future.
      */
-    public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
-        @Nullable final CacheEntryPredicate filter) {
-        return asyncOp(new AsyncOp<Boolean>() {
+    public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val, @Nullable final CacheEntryPredicate filter) {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
             @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAsync(ctx,
                     readyTopVer,
@@ -2685,6 +2613,11 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                     "filter", filter, false);
             }
         });
+
+        if (statsEnabled)
+            fut.listen(new UpdatePutTimeStatClosure<Boolean>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -2754,17 +2687,10 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (F.isEmpty(m))
             return;
 
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         if (keyCheck)
             validateCacheKeys(m.keySet());
 
         putAll0(m);
-
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
     }
 
     /**
@@ -2772,6 +2698,10 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
      * @throws IgniteCheckedException If failed.
      */
     protected void putAll0(final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
         syncOp(new SyncInOp(m.size() == 1) {
             @Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 tx.putAllAsync(ctx, null, m, false).get();
@@ -2781,6 +2711,9 @@ protected void putAll0(final Map<? extends K, ? extends V> m) throws IgniteCheck
                 return "putAll [map=" + m + ']';
             }
         });
+
+        if (statsEnabled)
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
@@ -2815,21 +2748,12 @@ protected IgniteInternalFuture<?> putAllAsync0(final Map<? extends K, ? extends
 
     /** {@inheritDoc} */
     @Nullable @Override public V getAndRemove(final K key) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        V prevVal = getAndRemove0(key);
-
-        if (statsEnabled)
-            metrics0().addRemoveAndGetTimeNanos(System.nanoTime() - start);
-
-        return prevVal;
+        return getAndRemove0(key);
     }
 
     /**
@@ -2868,21 +2792,12 @@ protected V getAndRemove0(final K key) throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<V> getAndRemoveAsync(final K key) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<V> fut = getAndRemoveAsync0(key);
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
+        return getAndRemoveAsync0(key);
     }
 
     /**
@@ -2890,7 +2805,11 @@ protected V getAndRemove0(final K key) throws IgniteCheckedException {
      * @return Future.
      */
     protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
-        return asyncOp(new AsyncOp<V>() {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
             @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 // TODO should we invoke interceptor here?
                 return tx.removeAllAsync(ctx,
@@ -2905,6 +2824,11 @@ protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
                 return "removeAsync [key=" + key + ']';
             }
         });
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -2925,10 +2849,6 @@ protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
 
     /** {@inheritDoc} */
     @Override public void removeAll(final Collection<? extends K> keys) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(keys, "keys");
 
         if (F.isEmpty(keys))
@@ -2938,9 +2858,6 @@ protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
             validateCacheKeys(keys);
 
         removeAll0(keys);
-
-        if (statsEnabled)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
     }
 
     /**
@@ -2948,6 +2865,10 @@ protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
      * @throws IgniteCheckedException If failed.
      */
     protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
         syncOp(new SyncInOp(keys.size() == 1) {
             @Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 tx.removeAllAsync(ctx,
@@ -2962,6 +2883,9 @@ protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheck
                 return "removeAll [keys=" + keys + ']';
             }
         });
+
+        if (statsEnabled)
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
@@ -2969,19 +2893,10 @@ protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheck
         if (F.isEmpty(keys))
             return new GridFinishedFuture<Object>();
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         if (keyCheck)
             validateCacheKeys(keys);
 
-        IgniteInternalFuture<Object> fut = removeAllAsync0(keys);
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
-
-        return fut;
+        return removeAllAsync0(keys);
     }
 
     /**
@@ -2989,7 +2904,11 @@ protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheck
      * @return Future.
      */
     protected IgniteInternalFuture<Object> removeAllAsync0(final Collection<? extends K> keys) {
-        return asyncOp(new AsyncOp(keys) {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<Object> fut = asyncOp(new AsyncOp(keys) {
             @Override public IgniteInternalFuture<?> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.removeAllAsync(ctx,
                     readyTopVer,
@@ -3003,6 +2922,11 @@ protected IgniteInternalFuture<Object> removeAllAsync0(final Collection<? extend
                 return "removeAllAsync [keys=" + keys + ']';
             }
         });
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -3017,21 +2941,12 @@ protected IgniteInternalFuture<Object> removeAllAsync0(final Collection<? extend
      * @throws IgniteCheckedException If failed.
      */
     public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        boolean rmv = remove0(key, filter);
-
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
-
-        return rmv;
+        return remove0(key, filter);
     }
 
     /**
@@ -3041,6 +2956,10 @@ public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws
      * @throws IgniteCheckedException If failed.
      */
     protected boolean remove0(final K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
         Boolean res = syncOp(new SyncOp<Boolean>(true) {
             @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return tx.removeAllAsync(ctx,
@@ -3058,6 +2977,9 @@ protected boolean remove0(final K key, final CacheEntryPredicate filter) throws
 
         assert res != null;
 
+        if (statsEnabled && Boolean.TRUE.equals(res))
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+
         return res;
     }
 
@@ -3074,21 +2996,12 @@ protected boolean remove0(final K key, final CacheEntryPredicate filter) throws
      * @return Putx operation future.
      */
     public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final CacheEntryPredicate filter) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<Boolean> fut = removeAsync0(key, filter);
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+        return removeAsync0(key, filter);
     }
 
     /**
@@ -3097,7 +3010,11 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
      * @return Future.
      */
     protected IgniteInternalFuture<Boolean> removeAsync0(final K key, @Nullable final CacheEntryPredicate filter) {
-        return asyncOp(new AsyncOp<Boolean>() {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
             @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.removeAllAsync(ctx,
                     readyTopVer,
@@ -3112,6 +3029,11 @@ protected IgniteInternalFuture<Boolean> removeAsync0(final K key, @Nullable fina
                 return "removeAsync [key=" + key + ", filter=" + filter + ']';
             }
         });
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 417eb3536536f..ac73907bdf175 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -761,6 +761,10 @@ private IgniteInternalFuture<GridCacheEntryInfo> getDhtSingleAsync(
     protected void processNearSingleGetRequest(final UUID nodeId, final GridNearSingleGetRequest req) {
         assert ctx.affinityNode();
 
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        final long start = statsEnabled ? System.nanoTime() : 0L;
+
         final CacheExpiryPolicy expiryPlc = CacheExpiryPolicy.fromRemote(req.createTtl(), req.accessTtl());
 
         IgniteInternalFuture<GridCacheEntryInfo> fut =
@@ -851,6 +855,15 @@ else if (req.needVersion())
                 sendTtlUpdateRequest(expiryPlc);
             }
         });
+
+        if (statsEnabled) {
+            fut.listen(new CI1<IgniteInternalFuture<GridCacheEntryInfo>>() {
+                @Override public void apply(IgniteInternalFuture<GridCacheEntryInfo> f) {
+                    if (!req.skipValues() && f.error() == null)
+                        metrics0().addGetTimeNanos(System.nanoTime() - start);
+                }
+            });
+        }
     }
 
     /**
@@ -861,6 +874,10 @@ protected void processNearGetRequest(final UUID nodeId, final GridNearGetRequest
         assert ctx.affinityNode();
         assert !req.reload() : req;
 
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        final long start = statsEnabled ? System.nanoTime() : 0L;
+
         final CacheExpiryPolicy expiryPlc = CacheExpiryPolicy.fromRemote(req.createTtl(), req.accessTtl());
 
         IgniteInternalFuture<Collection<GridCacheEntryInfo>> fut =
@@ -915,6 +932,15 @@ protected void processNearGetRequest(final UUID nodeId, final GridNearGetRequest
                 sendTtlUpdateRequest(expiryPlc);
             }
         });
+
+        if (statsEnabled) {
+            fut.listen(new CI1<IgniteInternalFuture<Collection<GridCacheEntryInfo>>>() {
+                @Override public void apply(IgniteInternalFuture<Collection<GridCacheEntryInfo>> f) {
+                    if (!req.skipValues() && f.error() == null)
+                        metrics0().addGetTimeNanos(System.nanoTime() - start);
+                }
+            });
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 519239aecb532..9b7c8dc8de284 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -53,7 +53,6 @@
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -281,6 +280,10 @@ private void map(
 
             // If this is the primary or backup node for the keys.
             if (n.isLocal()) {
+                final boolean statsEnabled = cctx.config().isStatisticsEnabled();
+
+                final long start = (statsEnabled)? System.nanoTime() : 0L;
+
                 final GridDhtFuture<Collection<GridCacheEntryInfo>> fut =
                     cache().getDhtAsync(n.id(),
                         -1,
@@ -316,7 +319,12 @@ private void map(
                 add(fut.chain(new C1<IgniteInternalFuture<Collection<GridCacheEntryInfo>>, Map<K, V>>() {
                     @Override public Map<K, V> apply(IgniteInternalFuture<Collection<GridCacheEntryInfo>> fut) {
                         try {
-                            return createResultMap(fut.get());
+                            Map<K, V> result = createResultMap(fut.get());
+
+                            if (!skipVals && statsEnabled && fut.error() == null)
+                                cctx.cache().metrics0().addGetTimeNanos(System.nanoTime() - start);
+
+                            return result;
                         }
                         catch (Exception e) {
                             U.error(log, "Failed to get values from dht cache [fut=" + fut + "]", e);
@@ -435,6 +443,10 @@ private boolean map(
     private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
         assert cctx.affinityNode() : this;
 
+        boolean statsEnabled = cctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
         GridDhtCacheAdapter<K, V> cache = cache();
 
         while (true) {
@@ -516,9 +528,12 @@ private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
 
                 // Entry not found, do not continue search if topology did not change and there is no store.
                 if (!cctx.readThroughConfigured() && (topStable || partitionOwned(part))) {
-                    if (!skipVals && cctx.config().isStatisticsEnabled())
+                    if (!skipVals && statsEnabled) {
                         cache.metrics0().onRead(false);
 
+                        cache.metrics0().addGetTimeNanos(System.nanoTime() - start);
+                    }
+
                     return true;
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index a3f6b72376e8f..25e9f0042672a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -122,6 +122,9 @@ public class GridPartitionedSingleGetFuture extends GridFutureAdapter<Object> im
     /** */
     private final boolean keepCacheObjects;
 
+    /** Future start time in nanoseconds. */
+    private final long startTimeNanos;
+
     /** */
     @GridToStringInclude
     private ClusterNode node;
@@ -182,6 +185,8 @@ public GridPartitionedSingleGetFuture(
 
         futId = IgniteUuid.randomUuid();
 
+        startTimeNanos = cctx.config().isStatisticsEnabled() ? System.nanoTime() : 0L;
+
         if (log == null)
             log = U.logger(cctx.kernalContext(), logRef, GridPartitionedSingleGetFuture.class);
     }
@@ -735,6 +740,9 @@ private void remap(final AffinityTopologyVersion topVer) {
 
             cctx.dht().sendTtlUpdateRequest(expiryPlc);
 
+            if (cctx.config().isStatisticsEnabled() && !trackable && !skipVals && err == null)
+                cctx.cache().metrics0().addGetTimeNanos(System.nanoTime() - startTimeNanos);
+
             return true;
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index ffe68ed2cfee7..527b4a7d80926 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1573,6 +1573,10 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
 
         // Optimisation: try to resolve value locally and escape 'get future' creation.
         if (!forcePrimary && ctx.affinityNode()) {
+            boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+            long start = (statsEnabled)? System.nanoTime() : 0L;
+
             Map<K, V> locVals = U.newHashMap(keys.size());
 
             boolean success = true;
@@ -1599,7 +1603,7 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
                                     null,
                                     /*swap*/true,
                                     /*unmarshal*/true,
-                                    /**update-metrics*/false,
+                                    /*update-metrics*/false,
                                     /*event*/!skipVals,
                                     subjId,
                                     null,
@@ -1618,7 +1622,7 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
                                     null,
                                     /*swap*/true,
                                     /*read-through*/false,
-                                    /**update-metrics*/false,
+                                    /*update-metrics*/false,
                                     /*event*/!skipVals,
                                     /*temporary*/false,
                                     subjId,
@@ -1672,6 +1676,9 @@ else if (!skipVals && ctx.config().isStatisticsEnabled())
             if (success) {
                 sendTtlUpdateRequest(expiry);
 
+                if (!skipVals && statsEnabled)
+                    metrics0().addGetTimeNanos(System.nanoTime() - start);
+
                 return new GridFinishedFuture<>(locVals);
             }
         }
@@ -3203,9 +3210,31 @@ private void processNearAtomicUpdateRequest(UUID nodeId, GridNearAtomicAbstractU
                 ", node=" + nodeId + ']');
         }
 
+        long start = (ctx.config().isStatisticsEnabled())? System.nanoTime() : 0L;
+
         req.nodeId(ctx.localNodeId());
 
         updateAllAsyncInternal(nodeId, req, updateReplyClos);
+
+        if (ctx.config().isStatisticsEnabled()) {
+            boolean retVal = req.returnValue();
+
+            long duration = System.nanoTime() - start;
+
+            if (req.operation() == UPDATE) {
+                if (retVal)
+                    metrics0().addPutAndGetTimeNanos(duration);
+                else {
+                    metrics0().addPutTimeNanos(duration);
+                }
+            }
+            else if (req.operation() == DELETE) {
+                if (retVal)
+                    metrics0().addRemoveAndGetTimeNanos(duration);
+                else
+                    metrics0().addRemoveTimeNanos(duration);
+            }
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
index 936c8a3bd3c87..536d0abafb792 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
@@ -42,6 +42,8 @@
 
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
 
 /**
  * Base for near atomic update futures.
@@ -260,11 +262,34 @@ protected boolean storeFuture() {
      * @param req Request.
      */
     protected void mapSingle(UUID nodeId, GridNearAtomicAbstractUpdateRequest req) {
+        final boolean statsEnabled = cctx.config().isStatisticsEnabled();
+
+        final long start = (statsEnabled)? System.nanoTime() : 0L;
+
         if (cctx.localNodeId().equals(nodeId)) {
             cache.updateAllAsyncInternal(nodeId, req,
                 new CI2<GridNearAtomicAbstractUpdateRequest, GridNearAtomicUpdateResponse>() {
                     @Override public void apply(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicUpdateResponse res) {
                         onResult(res.nodeId(), res, false);
+
+                        if (statsEnabled && res.error() == null) {
+                            boolean retVal = req.returnValue();
+
+                            long duration = System.nanoTime() - start;
+
+                            if (req.operation() == UPDATE) {
+                                if (retVal)
+                                    cache.metrics0().addPutAndGetTimeNanos(duration);
+                                else
+                                    cache.metrics0().addPutTimeNanos(duration);
+                            }
+                            else if (req.operation() == DELETE) {
+                                if (retVal)
+                                    cache.metrics0().addRemoveAndGetTimeNanos(duration);
+                                else
+                                    cache.metrics0().addRemoveTimeNanos(duration);
+                            }
+                        }
                     }
                 });
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index 9bdd1becb7600..798b3889a8d6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -61,7 +61,9 @@
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
 
 /**
  * DHT atomic cache near update future.
@@ -588,10 +590,33 @@ private void doUpdate(Map<UUID, GridNearAtomicFullUpdateRequest> mappings) {
         }
 
         if (locUpdate != null) {
+            final boolean statsEnabled = cctx.config().isStatisticsEnabled();
+
+            final long start = (statsEnabled)? System.nanoTime() : 0L;
+
             cache.updateAllAsyncInternal(cctx.localNodeId(), locUpdate,
                 new CI2<GridNearAtomicFullUpdateRequest, GridNearAtomicUpdateResponse>() {
                     @Override public void apply(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdateResponse res) {
                         onResult(res.nodeId(), res, false);
+
+                        if (statsEnabled && res.error() == null) {
+                            boolean retVal = req.returnValue();
+
+                            long duration = System.nanoTime() - start;
+
+                            if (req.operation() == UPDATE) {
+                                if (retVal)
+                                    cache.metrics0().addPutAndGetTimeNanos(duration);
+                                else
+                                    cache.metrics0().addPutTimeNanos(duration);
+                            }
+                            else if (req.operation() == DELETE) {
+                                if (retVal)
+                                    cache.metrics0().addRemoveAndGetTimeNanos(duration);
+                                else
+                                    cache.metrics0().addRemoveTimeNanos(duration);
+                            }
+                        }
                     }
                 });
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index e1e0ec2bf087e..fc36f6102316c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -72,7 +72,6 @@
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.CX1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -289,12 +288,16 @@ public GridDistributedCacheEntry entryExx(
         if (keyCheck)
             validateCacheKeys(keys);
 
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         IgniteTxLocalAdapter tx = ctx.tm().threadLocalTx(ctx);
 
         final CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         if (tx != null && !tx.implicit() && !skipTx) {
-            return asyncOp(tx, new AsyncOp<Map<K, V>>(keys) {
+            IgniteInternalFuture<Map<K, V>> fut = asyncOp(tx, new AsyncOp<Map<K, V>>(keys) {
                 @Override public IgniteInternalFuture<Map<K, V>> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                     return tx.getAllAsync(ctx,
                         readyTopVer,
@@ -306,6 +309,11 @@ public GridDistributedCacheEntry entryExx(
                         needVer);
                 }
             }, opCtx);
+
+            if (statsEnabled)
+                fut.listen(new UpdateGetTimeStatClosure<Map<K, V>>(metrics0(), start));
+
+            return fut;
         }
 
         AffinityTopologyVersion topVer = tx == null ?
@@ -314,7 +322,7 @@ public GridDistributedCacheEntry entryExx(
 
         subjId = ctx.subjectIdPerCall(subjId, opCtx);
 
-        return loadAsync(
+        IgniteInternalFuture<Map<K, V>> fut = loadAsync(
             ctx.cacheKeysView(keys),
             opCtx == null || !opCtx.skipStore(),
             forcePrimary,
@@ -326,6 +334,11 @@ public GridDistributedCacheEntry entryExx(
             skipVals,
             canRemap,
             needVer);
+
+        if (statsEnabled)
+            fut.listen(new UpdateGetTimeStatClosure<Map<K, V>>(metrics0(), start));
+
+        return fut;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index fb2843c02eec9..8e1c8ef79964f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -81,6 +81,9 @@ public final class GridNearGetFuture<K, V> extends CacheDistributedGetFutureAdap
     /** */
     private GridCacheVersion ver;
 
+    /** Future start time in nanoseconds. */
+    private final long startTimeNanos;
+
     /**
      * @param cctx Context.
      * @param keys Keys.
@@ -133,6 +136,8 @@ public GridNearGetFuture(
 
         ver = tx == null ? cctx.versions().next() : tx.xidVersion();
 
+        startTimeNanos = cctx.config().isStatisticsEnabled() ? System.nanoTime() : 0L;
+
         if (log == null)
             log = U.logger(cctx.kernalContext(), logRef, GridNearGetFuture.class);
     }
@@ -224,6 +229,9 @@ public void onResult(UUID nodeId, GridNearGetResponse res) {
 
             cache().dht().sendTtlUpdateRequest(expiryPlc);
 
+            if (cctx.config().isStatisticsEnabled() && !trackable && !skipVals && err == null)
+                cctx.cache().metrics0().addGetTimeNanos(System.nanoTime() - startTimeNanos);
+
             return true;
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
index 5b44d759f9080..a7ada9db19228 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
@@ -19,12 +19,15 @@
 
 import java.io.Externalizable;
 import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
 import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheOperationContext;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
@@ -234,4 +237,43 @@ void onFutureDone(GridLocalLockFuture fut) {
                 log().debug("Explicitly removed future from map of futures: " + fut);
         }
     }
+
+    /** {@inheritDoc} */
+    protected IgniteInternalFuture<Map<K, V>> getAllAsync(
+        @Nullable Collection<? extends K> keys,
+        boolean forcePrimary,
+        boolean skipTx,
+        @Nullable UUID subjId,
+        String taskName,
+        boolean deserializeBinary,
+        boolean skipVals,
+        boolean canRemap,
+        final boolean needVer
+    ) {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
+        CacheOperationContext opCtx = ctx.operationContextPerCall();
+
+        subjId = ctx.subjectIdPerCall(subjId, opCtx);
+
+        IgniteInternalFuture<Map<K, V>> fut = getAllAsync(keys,
+            null,
+            opCtx == null || !opCtx.skipStore(),
+            !skipTx,
+            subjId,
+            taskName,
+            deserializeBinary,
+            forcePrimary,
+            skipVals ? null : expiryPolicy(opCtx != null ? opCtx.expiry() : null),
+            skipVals,
+            canRemap,
+            needVer);
+
+        if (statsEnabled)
+            fut.listen(new UpdateGetTimeStatClosure<Map<K, V>>(metrics0(), start));
+
+        return fut;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index f86df2f29d684..dedd84ecef957 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -142,6 +142,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @Override protected boolean put0(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         Boolean res = (Boolean)updateAllInternal(UPDATE,
@@ -158,29 +162,50 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
         assert res != null;
 
+        if (statsEnabled && Boolean.TRUE.equals(res))
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
+
         return res;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<V> getAndPutAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        return updateAllAsync0(F0.asMap(key, val),
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<V> fut = updateAllAsync0(F0.asMap(key, val),
             null,
             null,
             true,
             false,
             filter);
+
+        if (statsEnabled)
+            fut.listen(new UpdatePutAndGetTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        return updateAllAsync0(F0.asMap(key, val),
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<Boolean> fut = updateAllAsync0(F0.asMap(key, val),
             null,
             null,
             false,
             false,
             filter);
+
+        if (statsEnabled)
+            fut.listen(new UpdatePutTimeStatClosure<Boolean>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -230,7 +255,16 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<V> getAndRemoveAsync0(K key) {
-        return removeAllAsync0(Collections.singletonList(key), true, false, null);
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<V> fut = removeAllAsync0(Collections.singletonList(key), true, false, null);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -259,6 +293,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public boolean remove0(K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         Boolean rmv = (Boolean)updateAllInternal(DELETE,
@@ -275,6 +313,9 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
         assert rmv != null;
 
+        if (statsEnabled && Boolean.TRUE.equals(rmv))
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+
         return rmv;
     }
 
@@ -303,6 +344,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
         boolean deserializeBinary,
         boolean needVer) throws IgniteCheckedException
     {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         Map<K, V> m = getAllInternal(Collections.singleton(key),
             ctx.isSwapOrOffheapEnabled(),
             ctx.readThrough(),
@@ -313,6 +358,9 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
         assert m.isEmpty() || m.size() == 1 : m.size();
 
+        if (statsEnabled)
+            metrics0().addGetTimeNanos(System.nanoTime() - start);
+
         return F.firstValue(m);
     }
 
@@ -322,15 +370,24 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
         throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         String taskName = ctx.kernalContext().job().currentTaskName();
 
-        return getAllInternal(keys,
+        Map<K, V> result = getAllInternal(keys,
             ctx.isSwapOrOffheapEnabled(),
             ctx.readThrough(),
             taskName,
             deserializeBinary,
             false,
             needVer);
+
+        if (statsEnabled)
+            metrics0().addGetTimeNanos(System.nanoTime() - start);
+
+        return result;
     }
 
     /** {@inheritDoc} */
@@ -348,15 +405,24 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     ) {
         A.notNull(keys, "keys");
 
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         final boolean swapOrOffheap = ctx.isSwapOrOffheapEnabled();
         final boolean storeEnabled = ctx.readThrough();
 
-        return asyncOp(new Callable<Map<K, V>>() {
+        IgniteInternalFuture<Map<K, V>> fut = asyncOp(new Callable<Map<K, V>>() {
             @Override public Map<K, V> call() throws Exception {
                 return getAllInternal(keys, swapOrOffheap, storeEnabled, taskName, deserializeBinary, skipVals, needVer);
             }
         });
-    }
+
+        if (statsEnabled)
+            fut.listen(new UpdateGetTimeStatClosure<Map<K, V>>(metrics0(), start));
+
+        return fut;
+   }
 
     /**
      * Entry point to all public API get methods.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicClientServerMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicClientServerMetricsSelfTest.java
new file mode 100644
index 0000000000000..42179e194a399
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicClientServerMetricsSelfTest.java
@@ -0,0 +1,452 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.atomic;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
+
+public class GridCacheAtomicClientServerMetricsSelfTest extends GridCommonAbstractTest {
+    /** Number of nodes. */
+    private final static int GRID_CNT = 3;
+
+    /** Server node index. */
+    private static final int SERVER_NODE = 0;
+
+    /** Client node index. */
+    private static final int CLIENT_NODE = 2;
+
+    /** */
+    private static TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    private int gridCount() {
+        return GRID_CNT;
+    }
+
+    /** {@inheritDoc} */
+    protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrids(GRID_CNT - 1);
+
+        Ignition.setClientMode(true);
+
+        startGrid(CLIENT_NODE);
+
+        Ignition.setClientMode(false);
+    }
+
+    /** {@inheritDoc} */
+    protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * Creates a new Cache Configuration for the given cache mode.
+     *
+     * @param mode Cache mode.
+     * @return Cache configuration.
+     */
+    private CacheConfiguration<Integer, Integer> getCacheConfiguration(CacheMode mode) {
+        CacheConfiguration<Integer, Integer> cacheCfg = new CacheConfiguration<>();
+
+        cacheCfg.setCacheMode(mode);
+        cacheCfg.setAtomicityMode(ATOMIC);
+        cacheCfg.setStatisticsEnabled(true);
+        cacheCfg.setName("metrics");
+
+        return cacheCfg;
+    }
+
+    /**
+     * Wait for {@link EventType#EVT_NODE_METRICS_UPDATED} event will be receieved.
+     */
+    private void awaitMetricsUpdate() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(GRID_CNT * 2);
+
+        IgnitePredicate<Event> lsnr = new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event event) {
+                latch.countDown();
+                return true;
+            }
+        };
+
+        for (int i = 0; i < gridCount(); ++i)
+            grid(i).events().localListen(lsnr, EVT_NODE_METRICS_UPDATED);
+
+        latch.await();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedGetAvgTime() throws Exception {
+        testGetAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedGetAvgTime() throws Exception {
+        testGetAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testGetAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i)
+                cache.put(i, rand.nextInt(12_000_000));
+
+            for (int i = 0; i < numOfKeys; ++i)
+                cache.get(i);
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+            assertEquals(clientMetrics.getAverageGetTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+            assertTrue(serverMetrics.getAverageGetTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedGetAndPutAvgTime() throws Exception {
+        testGetAndPutAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedGetAndPutAvgTime() throws Exception {
+        testGetAndPutAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testGetAndPutAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i)
+                cache.getAndPut(i, rand.nextInt(12_000_000));
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+            assertEquals(clientMetrics.getAverageGetTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+            assertTrue(serverMetrics.getAverageGetTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedRemoveAvgTime() throws Exception {
+        testRemoveAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedRemoveAvgTime() throws Exception {
+        testRemoveAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testRemoveAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i)
+                cache.put(i, rand.nextInt(12_000_000));
+
+            for (int i = 0; i < numOfKeys; ++i)
+                cache.remove(i);
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+            assertEquals(clientMetrics.getAverageRemoveTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+            assertTrue(serverMetrics.getAverageRemoveTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedGetAllAvgTime() throws Exception {
+        testGetAllAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedGetAllAvgTime() throws Exception {
+        testGetAllAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testGetAllAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            Set<Integer> keys = new TreeSet<>();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i) {
+                cache.put(i, rand.nextInt(12_000_000));
+
+                keys.add(i);
+            }
+
+            cache.getAll(keys);
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+            assertEquals(clientMetrics.getAverageGetTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+            assertTrue(serverMetrics.getAverageGetTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedPutAllAvgTime() throws Exception {
+        testPutAllAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedPutAllAvgTime() throws Exception {
+        testPutAllAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testPutAllAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            Map<Integer, Integer> values = new TreeMap<>();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i)
+                values .put(i, rand.nextInt(12_000_000));
+
+            cache.putAll(values);
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedRemoveAllAvgTime() throws Exception {
+        testRemoveAllAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedRemoveAllAvgTime() throws Exception {
+        testRemoveAllAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testRemoveAllAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            Set<Integer> keys = new TreeSet<>();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i) {
+                cache.put(i, rand.nextInt(12_000_000));
+
+                keys.add(i);
+            }
+
+            cache.removeAll(keys);
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+            assertEquals(clientMetrics.getAverageRemoveTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+            assertTrue(serverMetrics.getAverageRemoveTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
index f3dd1c83d5924..5785237a103a2 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.CacheMetricsForClusterGroupSelfTest;
 import org.apache.ignite.internal.processors.cache.OffheapCacheMetricsForClusterGroupSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicClientServerMetricsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedMetricsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedTckMetricsSelfTestImpl;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearMetricsSelfTest;
@@ -58,6 +59,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheAtomicPartitionedMetricsSelfTest.class);
         suite.addTestSuite(GridCacheAtomicPartitionedTckMetricsSelfTestImpl.class);
         suite.addTestSuite(GridCacheAtomicLocalTckMetricsSelfTestImpl.class);
+        suite.addTestSuite(GridCacheAtomicClientServerMetricsSelfTest.class);
 
         // Cluster wide metrics.
         suite.addTestSuite(CacheMetricsForClusterGroupSelfTest.class);
@@ -65,4 +67,4 @@ public static TestSuite suite() throws Exception {
 
         return suite;
     }
-}
\ No newline at end of file
+}

From 0a65037ee01e6f1a552c3771fd3940b6b23827bc Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 26 Sep 2017 17:38:21 +0300
Subject: [PATCH 426/446] Backport for IGNITE-2779: BinaryMarshaller caches
 must be cleaned during client reconnect.

---
 .../ignite/internal/binary/BinaryContext.java |  15 ++
 .../CacheObjectBinaryProcessorImpl.java       |   4 +-
 ...IgniteClientReconnectBinaryContexTest.java | 185 ++++++++++++++++++
 .../IgniteClientReconnectTestSuite.java       |   2 +
 4 files changed, 205 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index b0fa68c30f0ff..bbf15bbcc4ecb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.binary;
 
+import java.util.Iterator;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
@@ -1303,6 +1304,20 @@ public void unregisterBinarySchemas() {
         schemas = null;
     }
 
+    /**
+     * Unregisters the user types descriptors.
+     **/
+    public void unregisterUserTypeDescriptors() {
+        Iterator<Map.Entry<Class<?>, BinaryClassDescriptor>> it = descByCls.entrySet().iterator();
+
+        while (it.hasNext()) {
+            Map.Entry<Class<?>, BinaryClassDescriptor> e = it.next();
+
+            if (e.getValue().userType())
+                it.remove();
+        }
+    }
+
     /**
      * Returns instance of {@link OptimizedMarshaller}.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 6b691c2236e52..5939d643aac48 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -155,9 +155,11 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
     @GridToStringExclude
     private IgniteBinary binaries;
 
-    /** Listener removes all registred binary schemas after the local client reconnected. */
+    /** Listener removes all registred binary schemas and user type descriptors after the local client reconnected. */
     private final GridLocalEventListener clientDisconLsnr = new GridLocalEventListener() {
         @Override public void onEvent(Event evt) {
+            binaryContext().unregisterUserTypeDescriptors();
+
             binaryContext().unregisterBinarySchemas();
         }
     };
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java
new file mode 100644
index 0000000000000..7401799629fac
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.Callable;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+
+/**
+ *
+ */
+public class IgniteClientReconnectBinaryContexTest extends IgniteClientReconnectAbstractTest {
+    /** */
+    public static final String DEFAULT_CACHE_NAME = "myCache";
+
+    /** {@inheritDoc} */
+    @Override protected int serverCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected int clientCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setPeerClassLoadingEnabled(false);
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectCleaningUsersMetadata() throws Exception {
+        Ignite client = grid(serverCount());
+
+        assertTrue(client.cluster().localNode().isClient());
+
+        Ignite srv = grid(0);
+
+        srv.destroyCache(DEFAULT_CACHE_NAME);
+
+        CacheConfiguration<Integer, UserClass> cacheCfg = new CacheConfiguration<Integer, UserClass>()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.REPLICATED);
+
+        IgniteCache<Integer, UserClass> cache = client.createCache(cacheCfg);
+
+        Integer key = 1;
+        UserClass val = new UserClass(1);
+
+        cache.put(key, val); // For registering user types binary metadata
+
+        reconnectServersRestart(log, client, Collections.singleton(srv), new Callable<Collection<Ignite>>() {
+            /** {@inheritDoc} */
+            @Override public Collection<Ignite> call() throws Exception {
+                return Collections.singleton((Ignite)startGrid(0));
+            }
+        });
+
+        cache = client.createCache(cacheCfg);
+
+        cache.put(key, val);
+
+        assertEquals(val, cache.get(key));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectCleaningUsersMetadata2() throws Exception {
+        Ignite client = grid(serverCount());
+
+        assertTrue(client.cluster().localNode().isClient());
+
+        Ignite srv = grid(0);
+
+        srv.destroyCache(DEFAULT_CACHE_NAME);
+
+        CacheConfiguration<Integer, UserClass> cacheCfg = new CacheConfiguration<Integer, UserClass>()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.REPLICATED);
+
+        IgniteCache<Integer, UserClass> cache = client.createCache(cacheCfg);
+
+        Integer key = 1;
+        final UserClass val = new UserClass(1);
+
+        EntryProcessor<Integer, UserClass, Object> ep = new DummyEntryProccessor(val);
+
+        cache.invoke(key, ep); // For registering user types binary metadata
+
+        reconnectServersRestart(log, client, Collections.singleton(srv), new Callable<Collection<Ignite>>() {
+            /** {@inheritDoc} */
+            @Override public Collection<Ignite> call() throws Exception {
+                return Collections.singleton((Ignite)startGrid(0));
+            }
+        });
+
+        cache = client.createCache(cacheCfg);
+
+        cache.invoke(key, ep);
+
+        assertEquals(val, cache.get(key));
+    }
+
+    /** */
+    private static class UserClass {
+        /** */
+        private final int field;
+
+        /**
+         * @param field Value.
+         */
+        private UserClass(int field) {
+            this.field = field;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            UserClass val = (UserClass)o;
+
+            return field == val.field;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return field;
+        }
+    }
+
+    /** */
+    private static class DummyEntryProccessor implements EntryProcessor<Integer, UserClass, Object> {
+        /** */
+        private final UserClass val;
+
+        /** Constructor. */
+        public DummyEntryProccessor(UserClass val) {
+            this.val = val;
+        }
+
+        /** {@inheritDoc} */
+        @Override public UserClass process(MutableEntry<Integer, UserClass> entry, Object... arguments)
+            throws EntryProcessorException {
+            entry.setValue(val);
+
+            return null;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index 67d88e1e29eac..03d3fe230c296 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.IgniteClientReconnectApiExceptionTest;
 import org.apache.ignite.internal.IgniteClientReconnectAtomicsTest;
+import org.apache.ignite.internal.IgniteClientReconnectBinaryContexTest;
 import org.apache.ignite.internal.IgniteClientReconnectCacheTest;
 import org.apache.ignite.internal.IgniteClientReconnectCollectionsTest;
 import org.apache.ignite.internal.IgniteClientReconnectComputeTest;
@@ -46,6 +47,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteClientReconnectApiExceptionTest.class);
         suite.addTestSuite(IgniteClientReconnectDiscoveryStateTest.class);
         suite.addTestSuite(IgniteClientReconnectCacheTest.class);
+        suite.addTestSuite(IgniteClientReconnectBinaryContexTest.class);
         suite.addTestSuite(IgniteClientReconnectContinuousProcessorTest.class);
         suite.addTestSuite(IgniteClientReconnectComputeTest.class);
         suite.addTestSuite(IgniteClientReconnectAtomicsTest.class);

From 544e73a11feb29a499c6eebde97a1b90aa6e8ece Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Thu, 28 Sep 2017 14:14:43 +0300
Subject: [PATCH 427/446] ignite-gg-12855: Fix
 IgniteCacheTopologySafeGetSelfTest#testGetTopologySafeNodeJoinPrimaryLeave

---
 .../cache/IgniteCacheTopologySafeGetSelfTest.java        | 4 ++--
 .../ignite/testframework/junits/GridAbstractTest.java    | 9 +++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTopologySafeGetSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTopologySafeGetSelfTest.java
index 674f2b0adf503..6da99bbf7c5a3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTopologySafeGetSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTopologySafeGetSelfTest.java
@@ -147,7 +147,7 @@ public void checkGetTopologySafeNodeJoin(boolean failPrimary) throws Exception {
             IgniteInternalFuture<?> nodeFut = startNodeAsync();
 
             if (failPrimary)
-                stopGrid(1);
+                stopGrid(1, false, false);
 
             assertEquals(key, ((IgniteKernal)ignite(0)).internalCache("tx").getTopologySafe(key));
             assertEquals(key, ((IgniteKernal)ignite(0)).internalCache("atomic").getTopologySafe(key));
@@ -219,4 +219,4 @@ private void releaseTx() {
 
         releaseLatch.countDown();
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index e853b5046c094..22c796f6cd3e1 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -1182,6 +1182,15 @@ protected void stopGrid(int idx, boolean cancel) {
         }
     }
 
+    /**
+     * @param idx Index of the grid to stop.
+     * @param cancel Cancel flag.
+     * @param awaitTop Await topology change flag.
+     */
+    protected void stopGrid(int idx, boolean cancel, boolean awaitTop) {
+        stopGrid(getTestGridName(idx), false, awaitTop);
+    }
+
     /**
      * @param idx Index of the grid to stop.
      */

From 3b62f1c5a76eb203f76789e7c7084a5ceebb0c3a Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 21 Mar 2017 14:15:52 +0300
Subject: [PATCH 428/446] More info in toString for lock futures.

(cherry picked from commit 4322e00)
---
 .../distributed/dht/colocated/GridDhtColocatedLockFuture.java    | 1 +
 .../processors/cache/distributed/near/GridNearLockFuture.java    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index 79ca1087a3b88..4c37c538d4950 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -105,6 +105,7 @@ public final class GridDhtColocatedLockFuture extends GridCompoundIdentityFuture
     private final long threadId;
 
     /** Keys to lock. */
+    @GridToStringInclude
     private Collection<KeyCacheObject> keys;
 
     /** Future ID. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index d3e3a15831bb2..d1f3fac1b5416 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -100,6 +100,7 @@ public final class GridNearLockFuture extends GridCompoundIdentityFuture<Boolean
     private long threadId;
 
     /** Keys to lock. */
+    @GridToStringInclude
     private final Collection<KeyCacheObject> keys;
 
     /** Future ID. */

From f681640719720e6bdec6559f6ce6e47de34f15ee Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 5 Oct 2017 15:18:21 +0300
Subject: [PATCH 429/446] Backport of IGNITE-6545: Failure during Ignite
 Service.cancel() can break normal shutdown process.

---
 .../processors/service/GridServiceProcessor.java      | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 49009c2277f4d..963189c5241fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -337,7 +337,16 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
             Service svc = ctx.service();
 
             if (svc != null)
-                svc.cancel(ctx);
+                try {
+                    svc.cancel(ctx);
+                }
+                catch (Throwable e) {
+                    log.error("Failed to cancel service (ignoring) [name=" + ctx.name() +
+                        ", execId=" + ctx.executionId() + ']', e);
+
+                    if (e instanceof Error)
+                        throw e;
+                }
 
             ctx.executor().shutdownNow();
         }

From 916d0f17ebd8b52b7211a54c1552873820337eaa Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 6 Oct 2017 19:09:50 +0300
Subject: [PATCH 430/446] Backport of IGNITE-6562: Dynamic service deployment
 should use projection if NodeFilter is not set.

---
 .../ignite/internal/IgniteServicesImpl.java   |  2 +-
 .../service/GridServiceProcessor.java         | 60 +++++++++-------
 .../GridServiceProcessorAbstractSelfTest.java |  5 +-
 ...GridServiceProcessorMultiNodeSelfTest.java | 71 ++++++++++++++++++-
 4 files changed, 106 insertions(+), 32 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
index 58b3a2a96ff1c..0141aea02c918 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
@@ -161,7 +161,7 @@ public IgniteServicesImpl(GridKernalContext ctx, ClusterGroupAdapter prj, boolea
         guard();
 
         try {
-            saveOrGet(ctx.service().deployAll(cfgs));
+            saveOrGet(ctx.service().deployAll(prj, cfgs));
         }
         catch (IgniteCheckedException e) {
             throw U.convertException(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 963189c5241fb..079f00fe09e9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -90,6 +90,7 @@
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
@@ -298,15 +299,9 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
 
         ServiceConfiguration[] cfgs = ctx.config().getServiceConfiguration();
 
-        if (cfgs != null) {
-            for (ServiceConfiguration c : cfgs) {
-                // Deploy only on server nodes by default.
-                if (c.getNodeFilter() == null)
-                    c.setNodeFilter(ctx.cluster().get().forServers().predicate());
-            }
-
-            deployAll(Arrays.asList(cfgs)).get();
-        }
+        if (cfgs != null)
+            // Deploy only on server nodes by default.
+            deployAll(Arrays.asList(cfgs), ctx.cluster().get().forServers().predicate()).get();
 
         if (log.isDebugEnabled())
             log.debug("Started service processor.");
@@ -467,9 +462,8 @@ public IgniteInternalFuture<?> deployMultiple(ClusterGroup prj, String name, Ser
         cfg.setService(svc);
         cfg.setTotalCount(totalCnt);
         cfg.setMaxPerNodeCount(maxPerNodeCnt);
-        cfg.setNodeFilter(F.<ClusterNode>alwaysTrue() == prj.predicate() ? null : prj.predicate());
 
-        return deploy(cfg);
+        return deployAll(prj, Collections.singleton(cfg));
     }
 
     /**
@@ -492,14 +486,17 @@ public IgniteInternalFuture<?> deployKeyAffinitySingleton(String name, Service s
         cfg.setTotalCount(1);
         cfg.setMaxPerNodeCount(1);
 
-        return deploy(cfg);
+        // Ignore projection here.
+        return deployAll(Collections.singleton(cfg), null);
     }
 
     /**
      * @param cfgs Service configurations.
+     * @param dfltNodeFilter Default NodeFilter.
      * @return Configurations to deploy.
      */
-    private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceConfiguration> cfgs) {
+    private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceConfiguration> cfgs,
+        IgnitePredicate<ClusterNode> dfltNodeFilter) {
         List<ServiceConfiguration> cfgsCp = new ArrayList<>(cfgs.size());
 
         ServicesCompatibilityState state = markCompatibilityStateAsUsed();
@@ -511,6 +508,11 @@ private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceCo
         for (ServiceConfiguration cfg : cfgs) {
             Exception err = null;
 
+            // Deploy to projection node by default
+            // or only on server nodes if no projection .
+            if (cfg.getNodeFilter() == null && dfltNodeFilter != null)
+                cfg.setNodeFilter(dfltNodeFilter);
+
             try {
                 validate(cfg);
             }
@@ -566,11 +568,28 @@ private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceCo
         return new PreparedConfigurations(cfgsCp, failedFuts);
     }
 
+    /**
+     * @param prj Grid projection.
+     * @param cfgs Service configurations.
+     * @return Future for deployment.
+     */
+    public IgniteInternalFuture<?> deployAll(ClusterGroup prj, Collection<ServiceConfiguration> cfgs) {
+        if (prj == null)
+            // Deploy to servers by default if no projection specified.
+            return deployAll(cfgs,  ctx.cluster().get().forServers().predicate());
+        else if (prj.predicate() == F.<ClusterNode>alwaysTrue())
+            return deployAll(cfgs,  null);
+        else
+            // Deploy to predicate nodes by default.
+            return deployAll(cfgs,  prj.predicate());
+    }
+
     /**
      * @param cfgs Service configurations.
+     * @param dfltNodeFilter Default NodeFilter.
      * @return Future for deployment.
      */
-    public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs) {
+    private IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs, IgnitePredicate<ClusterNode> dfltNodeFilter) {
         assert cfgs != null;
 
         if (!busyLock.enterBusy()) {
@@ -581,8 +600,7 @@ public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs)
         }
 
         try {
-
-            PreparedConfigurations srvCfg = prepareServiceConfigurations(cfgs);
+            PreparedConfigurations srvCfg = prepareServiceConfigurations(cfgs, dfltNodeFilter);
 
             List<ServiceConfiguration> cfgsCp = srvCfg.cfgs;
 
@@ -745,16 +763,6 @@ private void writeServiceToCache(GridServiceDeploymentCompoundFuture res, Servic
         }
     }
 
-    /**
-     * @param cfg Service configuration.
-     * @return Future for deployment.
-     */
-    public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
-        A.notNull(cfg, "cfg");
-
-        return deployAll(Collections.singleton(cfg));
-    }
-
     /**
      * @return Compatibility state.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
index 6d91f36aab00b..debe4859d1c19 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
@@ -107,10 +107,7 @@ protected ServiceConfiguration[] services() {
     /** {@inheritDoc} */
     @SuppressWarnings("ConstantConditions")
     @Override protected void beforeTestsStarted() throws Exception {
-        assert nodeCount() >= 1;
-
-        for (int i = 0; i < nodeCount(); i++)
-            startGrid(i);
+        startGridsMultiThreaded(nodeCount());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index 9e7b9955babaa..4c1d979941036 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -175,7 +175,7 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
                 // Since we start extra nodes, there may be extra start and cancel events,
                 // so we check only the difference between start and cancel and
                 // not start and cancel events individually.
-                assertEquals(name, nodeCount() + servers,  DummyService.started(name) - DummyService.cancelled(name));
+                assertEquals(name, nodeCount() + servers, DummyService.started(name) - DummyService.cancelled(name));
 
                 checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
             }
@@ -188,6 +188,75 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachProjectionNodeUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
+
+        try {
+            final String name = "serviceOnEachProjectionNodeUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            int prestartedSrvcs = g.cluster().forClients().nodes().size();
+
+            CountDownLatch latch = new CountDownLatch(prestartedSrvcs);
+
+            DummyService.exeLatch(name, latch);
+
+            IgniteServices svcs = g.services(g.cluster().forClients()).withAsync();
+
+            svcs.deployNodeSingleton(name, new DummyService());
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
+            latch.await();
+
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+            assertEquals(name, prestartedSrvcs, DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
+
+            int servers = 2;
+
+            int clients = 2;
+
+            latch = new CountDownLatch(clients);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                waitForDeployment(name, clients);
+
+                // Since we start extra nodes, there may be extra start and cancel events,
+                // so we check only the difference between start and cancel and
+                // not start and cancel events individually.
+                assertEquals(name, clients + prestartedSrvcs, DummyService.started(name) - DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), clients + prestartedSrvcs);
+            }
+            finally {
+                stopExtraNodes(servers + clients);
+            }
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From 32931da8bcb64ff09edeb64daed9a885d631b6eb Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Mon, 9 Oct 2017 19:03:11 +0300
Subject: [PATCH 431/446] GG-12870 Remove incorrect assertion from
 GridDhtAtomicDeferredUpdateResponse.

---
 .../dht/atomic/GridDhtAtomicDeferredUpdateResponse.java         | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java
index 923b220dd1df5..78e90876b622a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java
@@ -65,8 +65,6 @@ public GridDhtAtomicDeferredUpdateResponse() {
      * @param addDepInfo Deployment info.
      */
     public GridDhtAtomicDeferredUpdateResponse(int cacheId, Collection<GridCacheVersion> futVers, boolean addDepInfo) {
-        assert !F.isEmpty(futVers);
-
         this.cacheId = cacheId;
         this.futVers = futVers;
         this.addDepInfo = addDepInfo;

From 1182666b4c810edb5b5500de862840681041ae52 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Wed, 11 Oct 2017 19:57:33 +0300
Subject: [PATCH 432/446] IGNITE-4450 Need release locks for failing nodes
 during exchange processing.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../GridDhtPartitionsExchangeFuture.java      |   6 +-
 .../CacheLockReleaseNodeLeaveTest.java        | 134 ++++++++++++++++++
 2 files changed, 138 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index ae2b1b6285506..6ccc07873335f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -881,8 +881,8 @@ private void waitPartitionRelease() throws IgniteCheckedException {
             }
             catch (IgniteFutureTimeoutCheckedException ignored) {
                 if (dumpedObjects < DUMP_PENDING_OBJECTS_THRESHOLD) {
-                    U.warn(log, "Failed to wait for locks release future. " +
-                        "Dumping pending objects that might be the cause: " + cctx.localNodeId());
+                    U.warn(log, "Failed to wait for locks release future. Dumping pending objects that might be the " +
+                        "cause [topVer=" + topologyVersion() + ", nodeId=" + cctx.localNodeId() + "]: ");
 
                     U.warn(log, "Locked keys:");
 
@@ -1800,6 +1800,8 @@ public void onNodeLeft(final ClusterNode node) {
         if (isDone() || !enterBusy())
             return;
 
+        cctx.mvcc().removeExplicitNodeLocks(node.id(), topologyVersion());
+
         try {
             onDiscoveryEvent(new IgniteRunnable() {
                 @Override public void run() {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java
index 687fe0bb6fa1a..7bed1544534d5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java
@@ -27,6 +27,9 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -221,4 +224,135 @@ public void testTxLockRelease() throws Exception {
 
         fut2.get(5, SECONDS);
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLockRelease2() throws Exception {
+        final Ignite ignite0 = startGrid(0);
+
+        Ignite ignite1 = startGrid(1);
+
+        Lock lock = ignite1.cache(null).lock("key");
+        lock.lock();
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                startGrid(2);
+
+                return null;
+            }
+        });
+
+        final AffinityTopologyVersion topVer = new AffinityTopologyVersion(2, 0);
+
+        // Wait when affinity change exchange start.
+        boolean wait = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                AffinityTopologyVersion topVer0 =
+                    ((IgniteKernal)ignite0).context().cache().context().exchange().topologyVersion();
+
+                return topVer.compareTo(topVer0) < 0;
+            }
+        }, 10_000);
+
+        assertTrue(wait);
+
+        assertFalse(fut.isDone());
+
+        ignite1.close();
+
+        fut.get(10_000);
+
+        Ignite ignite2 = ignite(2);
+
+        lock = ignite2.cache(null).lock("key");
+        lock.lock();
+        lock.unlock();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLockRelease3() throws Exception {
+        startGrid(0);
+
+        Ignite ignite1 = startGrid(1);
+
+        awaitPartitionMapExchange();
+
+        Lock lock = ignite1.cache(null).lock("key");
+        lock.lock();
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                startGrid(2);
+
+                return null;
+            }
+        });
+
+        assertFalse(fut.isDone());
+
+        ignite1.close();
+
+        fut.get(10_000);
+
+        Ignite ignite2 = ignite(2);
+
+        lock = ignite2.cache(null).lock("key");
+        lock.lock();
+        lock.unlock();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxLockRelease2() throws Exception {
+        final Ignite ignite0 = startGrid(0);
+
+        Ignite ignite1 = startGrid(1);
+
+        IgniteCache cache = ignite1.cache(null);
+        ignite1.transactions().txStart(PESSIMISTIC, REPEATABLE_READ);
+        cache.get(1);
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                startGrid(2);
+
+                return null;
+            }
+        });
+
+        final AffinityTopologyVersion topVer = new AffinityTopologyVersion(2, 0);
+
+        // Wait when affinity change exchange start.
+        boolean wait = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                AffinityTopologyVersion topVer0 =
+                    ((IgniteKernal)ignite0).context().cache().context().exchange().topologyVersion();
+
+                return topVer.compareTo(topVer0) < 0;
+            }
+        }, 10_000);
+
+        assertTrue(wait);
+
+        assertFalse(fut.isDone());
+
+        ignite1.close();
+
+        fut.get(10_000);
+
+        Ignite ignite2 = ignite(2);
+
+        cache = ignite2.cache(null);
+
+        try (Transaction tx = ignite2.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+            cache.get(1);
+
+            tx.commit();
+        }
+    }
 }

From fd88aef18826206fff78acd0c423871c7cd0e14d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 6 Oct 2017 15:26:21 +0300
Subject: [PATCH 433/446] GG-12398: Backport test fix
 GridCacheAtomicInvalidPartitionHandlingSelfTest.testClockFullAsync.

(cherry picked from commit 0519fb7)
---
 .../GridCacheAtomicInvalidPartitionHandlingSelfTest.java       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java
index 644e31012f454..dd3f5476f42be 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java
@@ -354,7 +354,8 @@ private void checkRestarts(CacheAtomicWriteOrderMode writeOrder,
             for (int r = 0; r < 20; r++) {
                 int idx0 = rnd.nextInt(gridCnt - 1) + 1;
 
-                stopGrid(idx0);
+                // No wait for PartitionMapExchange
+                stopGrid(idx0, false, false);
 
                 U.sleep(200);
 

From 5061c468f33bd94a441972e684e3b35ec0402887 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 13 Oct 2017 12:53:13 +0300
Subject: [PATCH 434/446] GG-12942: Fixed Fix Cache Failover 2 suite tests.

Fixed tests:
GridCachePartitionedTxSalvageSelfTest.testOptimisticTxSalvageBeforeTimeout
GridCachePartitionedTxSalvageSelfTest.testPessimisticcTxSalvageBeforeTimeout

(cherry picked from commit 3cc9435)
---
 .../distributed/near/GridCachePartitionedTxSalvageSelfTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java
index c2de8c576c597..5a093440ceeb3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java
@@ -235,7 +235,7 @@ private void startTxAndPutKeys(final TransactionConcurrency mode, final boolean
      * @throws Exception If failed.
      */
     private void stopNodeAndSleep(long timeout) throws Exception {
-        stopGrid(0);
+        stopGrid(0, false, false);
 
         info("Stopped grid.");
 

From 77eb963953a04fd87c5638932d53d0f473603682 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 13 Oct 2017 13:36:06 +0300
Subject: [PATCH 435/446] GG-12941: Fixed Ignite Cache 5 suite hanging.

Fixed GridCachePartitionEvictionDuringReadThroughSelfTest.

(cherry picked from commit 3cc9435)

(cherry picked from commit e46a9d1)
---
 ...tionEvictionDuringReadThroughSelfTest.java | 68 +++++++++++++------
 1 file changed, 47 insertions(+), 21 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java
index d5351f7e37776..5fef40024e0e4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java
@@ -18,8 +18,8 @@
 package org.apache.ignite.internal.processors.cache.distributed;
 
 import java.util.LinkedHashSet;
+import java.util.Set;
 import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.cache.Cache;
@@ -36,6 +36,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -43,6 +44,12 @@
  *
  */
 public class GridCachePartitionEvictionDuringReadThroughSelfTest extends GridCommonAbstractTest {
+    /** Failing key. */
+    private static final int FAILING_KEY = 3;
+
+    /** Data read grid index. */
+    private static final int DATA_READ_GRID_IDX = 0;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -75,28 +82,37 @@ public class GridCachePartitionEvictionDuringReadThroughSelfTest extends GridCom
      * @throws Exception if failed.
      */
     public void testPartitionRent() throws Exception {
-        startGrid(0);
+//        fail("https://issues.apache.org/jira/browse/IGNITE-5759");
+
+        startGrid(DATA_READ_GRID_IDX);
 
         final AtomicBoolean done = new AtomicBoolean();
 
-        IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Integer>() {
+        IgniteInternalFuture<Long> gridAndCacheAccessFut = GridTestUtils.runMultiThreadedAsync(new Callable<Integer>() {
             @Override
             public Integer call() throws Exception {
-                LinkedHashSet<Integer> set = new LinkedHashSet<>();
+                final Set<Integer> keysSet = new LinkedHashSet<>();
 
-                set.add(1);
-                set.add(2);
-                set.add(3);
-                set.add(4);
-                set.add(5);
+                keysSet.add(1);
+                keysSet.add(2);
+                keysSet.add(FAILING_KEY);
+                keysSet.add(4);
+                keysSet.add(5);
 
                 while (!done.get()) {
                     try {
-                        grid(0).<Integer, Integer>cache("config").getAll(set);
+                        grid(DATA_READ_GRID_IDX).<Integer, Integer>cache("config").getAll(keysSet);
                     }
-                    catch (Throwable ignore) {
-                        // No-op.
+                    catch (Throwable ex) {
+                        if (ex instanceof Error)
+                            throw ex;
+
+                        if (ex instanceof InterruptedException)
+                            Thread.currentThread().interrupt();
                     }
+
+                    if (Thread.currentThread().isInterrupted())
+                        throw new IgniteInterruptedCheckedException("Execution of [" + Thread.currentThread().getName() + "] Interrupted. Test is probably timed out");
                 }
 
                 return null;
@@ -115,11 +131,14 @@ public Integer call() throws Exception {
             }
         });
 
-        startFut.get();
-
-        done.set(true);
+        try {
+            startFut.get();
+        }
+        finally {
+            done.set(true);
+        }
 
-        fut.get();
+        gridAndCacheAccessFut.get();
     }
 
     /**
@@ -136,13 +155,10 @@ private static class CacheStoreFactory implements Factory<CacheStore<Integer, In
      *
      */
     private static class HangingCacheStore extends CacheStoreAdapter<Integer, Integer> {
-        /** */
-        private CountDownLatch releaseLatch = new CountDownLatch(1);
-
         /** {@inheritDoc} */
         @Override public Integer load(Integer key) throws CacheLoaderException {
-            if (key == 3)
-                throw new CacheLoaderException();
+            if (key == FAILING_KEY)
+                throw new TestCacheLoaderExpectedException();
 
             return key;
         }
@@ -157,4 +173,14 @@ private static class HangingCacheStore extends CacheStoreAdapter<Integer, Intege
 
         }
     }
+
+    /**
+     *
+     */
+    private static class TestCacheLoaderExpectedException extends CacheLoaderException {
+        /** {@inheritDoc} to reduce amount of logging, trace is not filled */
+        @Override public synchronized Throwable fillInStackTrace() {
+            return this;
+        }
+    }
 }

From 92acf74c2537c5fd9f39a2b8c5b901294f588f59 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 13 Oct 2017 18:25:50 +0300
Subject: [PATCH 436/446] GG-12941: Fixed hanged test
 CacheLateAffinityAssignmentTest.testJoinExchangeBecomeCoordinator.

(cherry picked from commit 5c35a58)
---
 .../cache/distributed/CacheLateAffinityAssignmentTest.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 771ab34116030..541514c763fd6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -1103,7 +1103,7 @@ public void testJoinExchangeBecomeCoordinator() throws Exception {
                 U.sleep(5000);
 
                 for (int i = 0; i < NODES; i++)
-                    stopGrid(i);
+                    stopGrid(i, false, false);
 
                 return null;
             }

From 37aae675618d32a114cb79b0a68640c1d71dc882 Mon Sep 17 00:00:00 2001
From: Alexander Paschenko <alexander.a.paschenko@gmail.com>
Date: Thu, 12 Jan 2017 10:50:14 +0300
Subject: [PATCH 437/446] IGNITE-4531: SQL: Use correct property name in
 BinaryProperty. This closes 1419.

(cherry picked from commit abc8b90)
---
 .../processors/query/GridQueryProcessor.java   |  2 +-
 ...iteCacheAbstractInsertSqlQuerySelfTest.java | 14 +++++++-------
 ...IgniteCacheAbstractSqlDmlQuerySelfTest.java |  6 +++---
 .../IgniteCacheInsertSqlQuerySelfTest.java     | 18 ++++++++++--------
 .../IgniteCacheMergeSqlQuerySelfTest.java      | 14 +++++++-------
 .../IgniteCacheUpdateSqlQuerySelfTest.java     |  4 ++--
 6 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 7511a991472c1..fb040fcdfa216 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -2124,7 +2124,7 @@ else if (val instanceof BinaryObject && ((BinaryObject)val).hasField(propName))
             if (!(obj instanceof BinaryObjectBuilder))
                 throw new UnsupportedOperationException("Individual properties can be set for binary builders only");
 
-            setValue0((BinaryObjectBuilder) obj, name(), propVal, type());
+            setValue0((BinaryObjectBuilder) obj, propName, propVal, type());
         }
 
         /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
index df4259e94c24b..86d01c788ee5b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
@@ -151,7 +151,7 @@ final void createBinaryCaches() {
             LinkedHashMap<String, String> flds = new LinkedHashMap<>();
 
             flds.put("id", Integer.class.getName());
-            flds.put("name", String.class.getName());
+            flds.put("firstName", String.class.getName());
 
             s2p.setFields(flds);
 
@@ -172,7 +172,7 @@ final void createBinaryCaches() {
             LinkedHashMap<String, String> flds = new LinkedHashMap<>();
 
             flds.put("id", Integer.class.getName());
-            flds.put("name", String.class.getName());
+            flds.put("firstName", String.class.getName());
 
             i2p.setFields(flds);
 
@@ -194,7 +194,7 @@ final void createBinaryCaches() {
 
             flds.put("key", Integer.class.getName());
             flds.put("id", Integer.class.getName());
-            flds.put("name", String.class.getName());
+            flds.put("firstName", String.class.getName());
 
             k2p.setFields(flds);
 
@@ -216,7 +216,7 @@ final void createBinaryCaches() {
 
             flds.put("Id", Integer.class.getName());
             flds.put("id", Integer.class.getName());
-            flds.put("name", String.class.getName());
+            flds.put("firstName", String.class.getName());
             flds.put("IntVal", Integer.class.getName());
 
             k22p.setFields(flds);
@@ -240,7 +240,7 @@ final void createBinaryCaches() {
             flds.put("key", Integer.class.getName());
             flds.put("strKey", String.class.getName());
             flds.put("id", Integer.class.getName());
-            flds.put("name", String.class.getName());
+            flds.put("firstName", String.class.getName());
 
             k32p.setFields(flds);
 
@@ -263,7 +263,7 @@ final void createBinaryCaches() {
             flds.put("key", Integer.class.getName());
             flds.put("strKey", String.class.getName());
             flds.put("id", Integer.class.getName());
-            flds.put("name", String.class.getName());
+            flds.put("firstName", String.class.getName());
 
             k42p.setFields(flds);
 
@@ -515,7 +515,7 @@ public Person(int id) {
         protected int id;
 
         /** */
-        @QuerySqlField
+        @QuerySqlField(name = "firstName")
         protected String name;
 
         /** {@inheritDoc} */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractSqlDmlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractSqlDmlQuerySelfTest.java
index 7f79ec4623206..649012fd545c6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractSqlDmlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractSqlDmlQuerySelfTest.java
@@ -139,7 +139,7 @@ private Object createBinPerson(int id, String name, String secondName) {
         BinaryObjectBuilder bldr = ignite(0).binary().builder("Person");
 
         bldr.setField("id", id);
-        bldr.setField("name", name);
+        bldr.setField("firstName", name);
         bldr.setField("secondName", secondName);
 
         return bldr.build();
@@ -186,7 +186,7 @@ private static CacheConfiguration createBinCacheConfig() {
         LinkedHashMap<String, String> flds = new LinkedHashMap<>();
 
         flds.put("id", Integer.class.getName());
-        flds.put("name", String.class.getName());
+        flds.put("firstName", String.class.getName());
         flds.put("secondName", String.class.getName());
 
         e.setFields(flds);
@@ -214,7 +214,7 @@ public Person(int id, String name, String secondName) {
         protected int id;
 
         /** */
-        @QuerySqlField
+        @QuerySqlField(name = "firstName")
         protected final String name;
 
         /** */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
index 04a352fd24940..e9c21dc2cea5d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
@@ -56,7 +56,7 @@ public class IgniteCacheInsertSqlQuerySelfTest extends IgniteCacheAbstractInsert
     public void testInsertWithExplicitKey() {
         IgniteCache<String, Person> p = ignite(0).cache("S2P").withKeepBinary();
 
-        p.query(new SqlFieldsQuery("insert into Person (_key, id, name) values ('s', ?, ?), " +
+        p.query(new SqlFieldsQuery("insert into Person (_key, id, firstName) values ('s', ?, ?), " +
             "('a', 2, 'Alex')").setArgs(1, "Sergi"));
 
         assertEquals(createPerson(1, "Sergi"), p.get("s"));
@@ -76,7 +76,7 @@ public void testInsertFromSubquery() {
         assertEquals("Sergi", p.get("s"));
         assertEquals("Alex", p.get("a"));
 
-        p.query(new SqlFieldsQuery("insert into Person(_key, id, name) " +
+        p.query(new SqlFieldsQuery("insert into Person(_key, id, firstName) " +
             "(select substring(lower(_val), 0, 2), cast(length(_val) as int), _val from String)"));
 
         assertEquals(createPerson(5, "Sergi"), p.get("se"));
@@ -91,7 +91,7 @@ public void testInsertWithExplicitPrimitiveKey() {
         IgniteCache<Integer, Person> p = ignite(0).cache("I2P").withKeepBinary();
 
         p.query(new SqlFieldsQuery(
-            "insert into Person (_key, id, name) values (cast('1' as int), ?, ?), (2, (5 - 3), 'Alex')")
+            "insert into Person (_key, id, firstName) values (cast('1' as int), ?, ?), (2, (5 - 3), 'Alex')")
             .setArgs(1, "Sergi"));
 
         assertEquals(createPerson(1, "Sergi"), p.get(1));
@@ -106,7 +106,7 @@ public void testInsertWithDynamicKeyInstantiation() {
         IgniteCache<Key, Person> p = ignite(0).cache("K2P").withKeepBinary();
 
         p.query(new SqlFieldsQuery(
-            "insert into Person (key, id, name) values (1, ?, ?), (2, 2, 'Alex')").setArgs(1, "Sergi"));
+            "insert into Person (key, id, firstName) values (1, ?, ?), (2, 2, 'Alex')").setArgs(1, "Sergi"));
 
         assertEquals(createPerson(1, "Sergi"), p.get(new Key(1)));
 
@@ -119,8 +119,8 @@ public void testInsertWithDynamicKeyInstantiation() {
     public void testFieldsCaseSensitivity() {
         IgniteCache<Key2, Person> p = ignite(0).cache("K22P").withKeepBinary();
 
-        p.query(new SqlFieldsQuery("insert into \"Person2\" (\"Id\", \"id\", \"name\", \"IntVal\") values (1, ?, ?, 5), " +
-            "(2, 3, 'Alex', 6)").setArgs(4, "Sergi"));
+        p.query(new SqlFieldsQuery("insert into \"Person2\" (\"Id\", \"id\", \"firstName\", \"IntVal\") " +
+            "values (1, ?, ?, 5),  (2, 3, 'Alex', 6)").setArgs(4, "Sergi"));
 
         assertEquals(createPerson2(4, "Sergi", 5), p.get(new Key2(1)));
 
@@ -177,7 +177,8 @@ public void testFieldsListIdentity() {
         IgniteCache<Key3, Person> p = ignite(0).cache("K32P").withKeepBinary();
 
         p.query(new SqlFieldsQuery(
-            "insert into Person (key, strKey, id, name) values (1, 'aa', ?, ?), (2, 'bb', 2, 'Alex')").setArgs(1, "Sergi"));
+            "insert into Person (key, strKey, id, firstName) values (1, 'aa', ?, ?), (2, 'bb', 2, 'Alex')")
+            .setArgs(1, "Sergi"));
 
         assertEquals(createPerson(1, "Sergi"), p.get(new Key3(1)));
 
@@ -194,7 +195,8 @@ public void testCustomIdentity() {
         IgniteCache<Key4, Person> p = ignite(0).cache("K42P").withKeepBinary();
 
         p.query(new SqlFieldsQuery(
-            "insert into Person (key, strKey, id, name) values (1, 'aa', ?, ?), (2, 'bb', 2, 'Alex')").setArgs(1, "Sergi"));
+            "insert into Person (key, strKey, id, firstName) values (1, 'aa', ?, ?), (2, 'bb', 2, 'Alex')")
+            .setArgs(1, "Sergi"));
 
         assertEquals(createPerson(1, "Sergi"), p.get(new Key4(1)));
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMergeSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMergeSqlQuerySelfTest.java
index 0ff3fda0da87f..58d07af25ce3f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMergeSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMergeSqlQuerySelfTest.java
@@ -32,7 +32,7 @@ public class IgniteCacheMergeSqlQuerySelfTest extends IgniteCacheAbstractInsertS
     public void testMergeWithExplicitKey() {
         IgniteCache<String, Person> p = ignite(0).cache("S2P").withKeepBinary();
 
-        p.query(new SqlFieldsQuery("merge into Person (_key, id, name) values ('s', ?, ?), " +
+        p.query(new SqlFieldsQuery("merge into Person (_key, id, firstName) values ('s', ?, ?), " +
             "('a', 2, 'Alex')").setArgs(1, "Sergi"));
 
         assertEquals(createPerson(1, "Sergi"), p.get("s"));
@@ -52,7 +52,7 @@ public void testMergeFromSubquery() {
         assertEquals("Sergi", p.get("s"));
         assertEquals("Alex", p.get("a"));
 
-        p.query(new SqlFieldsQuery("merge into Person(_key, id, name) " +
+        p.query(new SqlFieldsQuery("merge into Person(_key, id, firstName) " +
             "(select substring(lower(_val), 0, 2), cast(length(_val) as int), _val from String)"));
 
         assertEquals(createPerson(5, "Sergi"), p.get("se"));
@@ -67,7 +67,7 @@ public void testMergeWithExplicitPrimitiveKey() {
         IgniteCache<Integer, Person> p = ignite(0).cache("I2P").withKeepBinary();
 
         p.query(new SqlFieldsQuery(
-            "merge into Person (_key, id, name) values (cast(? as int), ?, ?), (2, (5 - 3), 'Alex')")
+            "merge into Person (_key, id, firstName) values (cast(? as int), ?, ?), (2, (5 - 3), 'Alex')")
             .setArgs("1", 1, "Sergi"));
 
         assertEquals(createPerson(1, "Sergi"), p.get(1));
@@ -82,7 +82,7 @@ public void testMergeWithDynamicKeyInstantiation() {
         IgniteCache<Key, Person> p = ignite(0).cache("K2P").withKeepBinary();
 
         p.query(new SqlFieldsQuery(
-            "merge into Person (key, id, name) values (1, ?, ?), (2, 2, 'Alex')").setArgs(1, "Sergi"));
+            "merge into Person (key, id, firstName) values (1, ?, ?), (2, 2, 'Alex')").setArgs(1, "Sergi"));
 
         assertEquals(createPerson(1, "Sergi"), p.get(new Key(1)));
 
@@ -95,7 +95,7 @@ public void testMergeWithDynamicKeyInstantiation() {
     public void testFieldsCaseSensitivity() {
         IgniteCache<Key2, Person> p = ignite(0).cache("K22P").withKeepBinary();
 
-        p.query(new SqlFieldsQuery("merge into \"Person2\" (\"Id\", \"id\", \"name\", \"IntVal\") values (1, ?, ?, 5), " +
+        p.query(new SqlFieldsQuery("merge into \"Person2\" (\"Id\", \"id\", \"firstName\", \"IntVal\") values (1, ?, ?, 5), " +
             "(2, 3, 'Alex', 6)").setArgs(4, "Sergi"));
 
         assertEquals(createPerson2(4, "Sergi", 5), p.get(new Key2(1)));
@@ -127,7 +127,7 @@ public void testFieldsListIdentity() {
         IgniteCache<Key3, Person> p = ignite(0).cache("K32P").withKeepBinary();
 
         p.query(new SqlFieldsQuery(
-            "merge into Person (key, strKey, id, name) values (1, 'aa', ?, ?), (2, 'bb', 2, 'Alex')").setArgs(1, "Sergi"));
+            "merge into Person (key, strKey, id, firstName) values (1, 'aa', ?, ?), (2, 'bb', 2, 'Alex')").setArgs(1, "Sergi"));
 
         assertEquals(createPerson(1, "Sergi"), p.get(new Key3(1)));
 
@@ -144,7 +144,7 @@ public void testCustomIdentity() {
         IgniteCache<Key4, Person> p = ignite(0).cache("K42P").withKeepBinary();
 
         p.query(new SqlFieldsQuery(
-            "merge into Person (key, strKey, id, name) values (1, 'aa', ?, ?), (2, 'bb', 2, 'Alex')").setArgs(1, "Sergi"));
+            "merge into Person (key, strKey, id, firstName) values (1, 'aa', ?, ?), (2, 'bb', 2, 'Alex')").setArgs(1, "Sergi"));
 
         assertEquals(createPerson(1, "Sergi"), p.get(new Key4(1)));
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheUpdateSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheUpdateSqlQuerySelfTest.java
index 332a082201512..58bcaacf2f741 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheUpdateSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheUpdateSqlQuerySelfTest.java
@@ -61,8 +61,8 @@ private static CacheConfiguration createAllTypesCacheConfig() {
     public void testUpdateSimple() {
         IgniteCache p = cache();
 
-        QueryCursor<List<?>> c = p.query(new SqlFieldsQuery("update Person p set p.id = p.id * 2, p.name = " +
-            "substring(p.name, 0, 2) where length(p._key) = ? or p.secondName like ?").setArgs(2, "%ite"));
+        QueryCursor<List<?>> c = p.query(new SqlFieldsQuery("update Person p set p.id = p.id * 2, p.firstName = " +
+            "substring(p.firstName, 0, 2) where length(p._key) = ? or p.secondName like ?").setArgs(2, "%ite"));
 
         c.iterator();
 

From c4d29c1d6de2228bfd6c8cca501fe4b5f08e6a26 Mon Sep 17 00:00:00 2001
From: Konstantin Dudkov <kdudkov@ya.ru>
Date: Tue, 17 Oct 2017 11:43:30 +0300
Subject: [PATCH 438/446] IGNITE-6581 fix client deadlock

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |   3 +-
 .../TcpDiscoveryClientConnectionFailTest.java | 161 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   3 +
 3 files changed, 166 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientConnectionFailTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 71550447fa04d..a470224da5664 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -1680,7 +1680,8 @@ else if (msg == SPI_RECONNECT_FAILED) {
 
                                 onDisconnected();
 
-                                notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
+                                if (joinLatch.getCount() == 0)
+                                    notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
                             }
 
                             UUID newId = UUID.randomUUID();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientConnectionFailTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientConnectionFailTest.java
new file mode 100644
index 0000000000000..c77f1b7f62b45
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientConnectionFailTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.Socket;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryClientHeartbeatMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddedMessage;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class TcpDiscoveryClientConnectionFailTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+    /** */
+    private static volatile boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.endsWith("0"))
+            cfg.setFailureDetectionTimeout(100);
+
+        TestDiscoverySpi spi = new TestDiscoverySpi(gridName);
+        spi.setClientReconnectDisabled(false);
+        spi.setIpFinder(ipFinder);
+
+        spi.setJoinTimeout(20000);
+        spi.setMaxMissedClientHeartbeats(2);
+        spi.failureDetectionTimeoutEnabled(true);
+
+        cfg.setDiscoverySpi(spi);
+
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /** */
+    public void testClientFailsIfCantConnect() throws Exception {
+        client = false;
+
+        startGrid(0);
+
+        client = true;
+
+        IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    startGrid(1);
+                }
+                catch (Exception ignored) {
+                }
+
+            }
+        }, 1, "client start thread");
+
+        try {
+            fut.get(10000);
+        }
+        catch (IgniteFutureTimeoutCheckedException e) {
+            fail("client node start hangs (possible deadlock detected)");
+        }
+
+    }
+
+    /**
+     *
+     */
+    private static class TestDiscoverySpi extends TcpDiscoverySpi {
+        /** */
+        private final String name;
+        /** */
+        private boolean block;
+
+        /** */
+        TestDiscoverySpi(String name) {
+            this.name = name;
+        }
+
+        /**
+         * @param msg Message.
+         * @return {@code False} if should not further process message.
+         * @throws IOException If failed.
+         */
+        private boolean onMessage(TcpDiscoveryAbstractMessage msg) throws IOException {
+            if (msg == null)
+                return true;
+
+            boolean blocked = false;
+
+            // we block first connection attempt only
+            if (msg instanceof TcpDiscoveryNodeAddedMessage)
+                block = !block;
+
+            if (msg instanceof TcpDiscoveryClientHeartbeatMessage && block)
+                blocked = true;
+
+            if (msg instanceof TcpDiscoveryNodeAddFinishedMessage && block)
+                blocked = true;
+
+            return !blocked;
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, byte[] data,
+            long timeout) throws IOException {
+
+            if (onMessage(msg))
+                super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(TcpDiscoveryAbstractMessage msg, Socket sock, int res,
+            long timeout) throws IOException {
+            if (onMessage(msg))
+                super.writeToSocket(msg, sock, res, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (onMessage(msg))
+                super.writeToSocket(sock, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (onMessage(msg))
+                super.writeToSocket(sock, out, msg, timeout);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index c2913ba6c6a5b..5f7d1236ecc67 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryClientConnectionFailTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMultiThreadedTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeAttributesUpdateOnReconnectTest;
@@ -94,6 +95,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(IgniteClientConnectTest.class));
         suite.addTest(new TestSuite(IgniteClientReconnectMassiveShutdownTest.class));
 
+        suite.addTest(new TestSuite(TcpDiscoveryClientConnectionFailTest.class));
+
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));

From 47ac655b9543569bc05a3b7e1265272e0d5069e5 Mon Sep 17 00:00:00 2001
From: voipp <aleksaero@gmail.com>
Date: Tue, 14 Feb 2017 15:08:59 +0300
Subject: [PATCH 439/446] IGNITE-4492 Add MBean for StripedExecutor. This
 closes #1491.

(cherry picked from commit 8e12513)
---
 .../apache/ignite/internal/IgniteKernal.java  |  48 ++++-
 .../StripedExecutorMXBeanAdapter.java         |  90 ++++++++++
 .../ignite/internal/util/StripedExecutor.java |  55 +++++-
 .../ignite/mxbean/StripedExecutorMXBean.java  |  90 ++++++++++
 .../internal/util/StripedExecutorTest.java    | 168 ++++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |   2 +
 6 files changed, 447 insertions(+), 6 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/StripedExecutorMXBeanAdapter.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/mxbean/StripedExecutorMXBean.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 0ac83d1a6eb40..03603978de0cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -165,6 +165,7 @@
 import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.mxbean.ClusterLocalNodeMetricsMXBean;
 import org.apache.ignite.mxbean.IgniteMXBean;
+import org.apache.ignite.mxbean.StripedExecutorMXBean;
 import org.apache.ignite.mxbean.ThreadPoolMXBean;
 import org.apache.ignite.plugin.IgnitePlugin;
 import org.apache.ignite.plugin.PluginNotFoundException;
@@ -305,6 +306,10 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     @GridToStringExclude
     private ObjectName restExecSvcMBean;
 
+    /** */
+    @GridToStringExclude
+    private ObjectName stripedExecSvcMBean;
+
     /** Kernal start timestamp. */
     private long startTime = U.currentTimeMillis();
 
@@ -1002,6 +1007,7 @@ public void start(
                 restExecSvc,
                 utilityCachePool,
                 marshCachePool);
+            registerStripedExecutorMBean(stripedExecSvc);
 
             // Lifecycle bean notifications.
             notifyLifecycleBeans(AFTER_NODE_START);
@@ -1592,7 +1598,14 @@ private void registerLocalNodeMBean() throws IgniteCheckedException {
         }
     }
 
-    /** @throws IgniteCheckedException If registration failed. */
+    /**
+     * @param execSvc
+     * @param sysExecSvc
+     * @param p2pExecSvc
+     * @param mgmtExecSvc
+     * @param restExecSvc
+     * @throws IgniteCheckedException If failed.
+     */
     private void registerExecutorMBeans(
         ExecutorService execSvc,
         ExecutorService sysExecSvc,
@@ -1642,8 +1655,34 @@ private ObjectName registerExecutorMBean(ExecutorService exec, String name) thro
             return res;
         }
         catch (JMException e) {
-            throw new IgniteCheckedException("Failed to register executor service MBean [name=" + name + ", exec=" + exec + ']',
-                e);
+            throw new IgniteCheckedException("Failed to register executor service MBean [name=" + name +
+                ", exec=" + exec + ']', e);
+        }
+    }
+
+    /**
+     * @param stripedExecSvc Executor service.
+     * @throws IgniteCheckedException If registration failed.
+     */
+    private void registerStripedExecutorMBean(StripedExecutor stripedExecSvc) throws IgniteCheckedException {
+        if (stripedExecSvc != null) {
+            String name = "StripedExecutor";
+
+            try {
+                stripedExecSvcMBean = U.registerMBean(
+                    cfg.getMBeanServer(),
+                    cfg.getGridName(),
+                    "Thread Pools",
+                    name,
+                    new StripedExecutorMXBeanAdapter(stripedExecSvc),
+                    StripedExecutorMXBean.class);
+
+                if (log.isDebugEnabled())
+                    log.debug("Registered executor service MBean: " + stripedExecSvcMBean);
+            } catch (JMException e) {
+                throw new IgniteCheckedException("Failed to register executor service MBean [name="
+                    + name + ", exec=" + stripedExecSvc + ']', e);
+            }
         }
     }
 
@@ -2110,7 +2149,8 @@ else if (state == STARTING)
                     unregisterMBean(locNodeMBean) &
                     unregisterMBean(utilityExecSvcMBean) &
                     unregisterMBean(marshallerExecSvcMBean) &
-                    unregisterMBean(restExecSvcMBean)
+                    unregisterMBean(restExecSvcMBean) &
+                    unregisterMBean(stripedExecSvcMBean)
             ))
                 errOnStop = false;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/StripedExecutorMXBeanAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/StripedExecutorMXBeanAdapter.java
new file mode 100644
index 0000000000000..e6811b7aee762
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/StripedExecutorMXBeanAdapter.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.concurrent.ExecutorService;
+import org.apache.ignite.internal.util.StripedExecutor;
+import org.apache.ignite.mxbean.StripedExecutorMXBean;
+
+/**
+ * Adapter for {@link StripedExecutorMXBean} which delegates all method calls to the underlying
+ * {@link ExecutorService} instance.
+ */
+public class StripedExecutorMXBeanAdapter implements StripedExecutorMXBean {
+    /** */
+    private final StripedExecutor exec;
+
+    /**
+     * @param exec Executor service
+     */
+    StripedExecutorMXBeanAdapter(StripedExecutor exec) {
+        assert exec != null;
+
+        this.exec = exec;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void checkStarvation() {
+        exec.checkStarvation();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getStripesCount() {
+        return exec.stripes();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isShutdown() {
+        return exec.isShutdown();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isTerminated() {
+        return exec.isTerminated();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getTotalQueueSize() {
+        return exec.queueSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTotalCompletedTasksCount() {
+        return exec.completedTasks();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long[] getStripesCompletedTasksCounts() {
+        return exec.stripesCompletedTasks();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getActiveCount() {
+        return exec.activeStripesCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean[] getStripesActiveStatuses() {
+        return exec.stripesActiveStatuses();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int[] getStripesQueueSizes() {
+        return exec.stripesQueueSizes();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
index e9ec74b181578..970beb8db6c5b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
@@ -55,9 +55,10 @@ public class StripedExecutor implements ExecutorService {
     private final IgniteLogger log;
 
     /**
-     * Constructor.
-     *
      * @param cnt Count.
+     * @param gridName Node name.
+     * @param poolName Pool name.
+     * @param log Logger.
      */
     public StripedExecutor(int cnt, String gridName, String poolName, final IgniteLogger log) {
         A.ensure(cnt > 0, "cnt > 0");
@@ -267,6 +268,56 @@ public long completedTasks() {
         return cnt;
     }
 
+    /**
+     * @return Completed tasks per stripe count.
+     */
+    public long[] stripesCompletedTasks() {
+        long[] res = new long[stripes()];
+
+        for (int i = 0; i < res.length; i++)
+            res[i] = stripes[i].completedCnt;
+
+        return res;
+    }
+
+    /**
+     * @return Number of active tasks per stripe.
+     */
+    public boolean[] stripesActiveStatuses() {
+        boolean[] res = new boolean[stripes()];
+
+        for (int i = 0; i < res.length; i++)
+            res[i] = stripes[i].active;
+
+        return res;
+    }
+
+    /**
+     * @return Number of active tasks.
+     */
+    public int activeStripesCount() {
+        int res = 0;
+
+        for (boolean status : stripesActiveStatuses()) {
+            if (status)
+                res++;
+        }
+
+        return res;
+    }
+
+    /**
+     * @return Size of queue per stripe.
+     */
+    public int[] stripesQueueSizes() {
+        int[] res = new int[stripes()];
+
+        for (int i = 0; i < res.length; i++)
+            res[i] = stripes[i].queueSize();
+
+        return res;
+    }
+
     /**
      * Operation not supported.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/StripedExecutorMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/StripedExecutorMXBean.java
new file mode 100644
index 0000000000000..7428b19600d20
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/StripedExecutorMXBean.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mxbean;
+
+/**
+ * MBean that provides access to information about striped executor service.
+ */
+@MXBeanDescription("MBean that provides access to information about striped executor service.")
+public interface StripedExecutorMXBean {
+    /**
+     * Checks for starvation in striped pool, dumps in log information if potential starvation
+     * was found.
+     */
+    @MXBeanDescription("Checks for starvation in striped pool.")
+    public void checkStarvation();
+
+    /**
+     * @return Stripes count.
+     */
+    @MXBeanDescription("Stripes count.")
+    public int getStripesCount();
+
+    /**
+     *
+     * @return {@code True} if this executor has been shut down.
+     */
+    @MXBeanDescription("True if this executor has been shut down.")
+    public boolean isShutdown();
+
+    /**
+     * Note that
+     * {@code isTerminated()} is never {@code true} unless either {@code shutdown()} or
+     * {@code shutdownNow()} was called first.
+     *
+     * @return {@code True} if all tasks have completed following shut down.
+     */
+    @MXBeanDescription("True if all tasks have completed following shut down.")
+    public boolean isTerminated();
+
+    /**
+     * @return Return total queue size of all stripes.
+     */
+    @MXBeanDescription("Total queue size of all stripes.")
+    public int getTotalQueueSize();
+
+    /**
+     * @return Completed tasks count.
+     */
+    @MXBeanDescription("Completed tasks count of all stripes.")
+    public long getTotalCompletedTasksCount();
+
+    /**
+     * @return Number of completed tasks per stripe.
+     */
+    @MXBeanDescription("Number of completed tasks per stripe.")
+    public long[] getStripesCompletedTasksCounts();
+
+    /**
+     * @return Number of active tasks.
+     */
+    @MXBeanDescription("Number of active tasks of all stripes.")
+    public int getActiveCount();
+
+    /**
+     * @return Number of active tasks per stripe.
+     */
+    @MXBeanDescription("Number of active tasks per stripe.")
+    public boolean[] getStripesActiveStatuses();
+
+    /**
+     * @return Size of queue per stripe.
+     */
+    @MXBeanDescription("Size of queue per stripe.")
+    public int[] getStripesQueueSizes();
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
new file mode 100644
index 0000000000000..543907fcde60a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import org.apache.ignite.logger.java.JavaLogger;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class StripedExecutorTest extends GridCommonAbstractTest {
+    /** */
+    private StripedExecutor stripedExecSvc;
+
+    /** {@inheritDoc} */
+    @Override public void beforeTest() {
+        stripedExecSvc = new StripedExecutor(3, "foo name", "pool name", new JavaLogger());
+    }
+
+    /** {@inheritDoc} */
+    @Override public void afterTest() {
+        stripedExecSvc.shutdown();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCompletedTasks() throws Exception {
+        stripedExecSvc.execute(0, new TestRunnable());
+        stripedExecSvc.execute(1, new TestRunnable());
+
+        sleepASec();
+
+        assertEquals(2, stripedExecSvc.completedTasks());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStripesCompletedTasks() throws Exception {
+        stripedExecSvc.execute(0, new TestRunnable());
+        stripedExecSvc.execute(1, new TestRunnable());
+
+        sleepASec();
+
+        long[] completedTaks = stripedExecSvc.stripesCompletedTasks();
+
+        assertEquals(1, completedTaks[0]);
+        assertEquals(1, completedTaks[1]);
+        assertEquals(0, completedTaks[2]);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStripesActiveStatuses() throws Exception {
+        stripedExecSvc.execute(0, new TestRunnable());
+        stripedExecSvc.execute(1, new TestRunnable(true));
+
+        sleepASec();
+
+        boolean[] statuses = stripedExecSvc.stripesActiveStatuses();
+
+        assertFalse(statuses[0]);
+        assertTrue(statuses[1]);
+        assertFalse(statuses[0]);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testActiveStripesCount() throws Exception {
+        stripedExecSvc.execute(0, new TestRunnable());
+        stripedExecSvc.execute(1, new TestRunnable(true));
+
+        sleepASec();
+
+        assertEquals(1, stripedExecSvc.activeStripesCount());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStripesQueueSizes() throws Exception {
+        stripedExecSvc.execute(0, new TestRunnable());
+        stripedExecSvc.execute(0, new TestRunnable(true));
+        stripedExecSvc.execute(0, new TestRunnable(true));
+        stripedExecSvc.execute(1, new TestRunnable(true));
+        stripedExecSvc.execute(1, new TestRunnable(true));
+        stripedExecSvc.execute(1, new TestRunnable(true));
+
+        sleepASec();
+
+        int[] queueSizes = stripedExecSvc.stripesQueueSizes();
+
+        assertEquals(1, queueSizes[0]);
+        assertEquals(2, queueSizes[1]);
+        assertEquals(0, queueSizes[2]);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testQueueSize() throws Exception {
+        stripedExecSvc.execute(1, new TestRunnable());
+        stripedExecSvc.execute(1, new TestRunnable(true));
+        stripedExecSvc.execute(1, new TestRunnable(true));
+
+        sleepASec();
+
+        assertEquals(1, stripedExecSvc.queueSize());
+    }
+
+    /**
+     *
+     */
+    private final class TestRunnable implements Runnable {
+        /** */
+        private final boolean infinitely;
+
+        /**
+         *
+         */
+        public TestRunnable() {
+            this(false);
+        }
+
+        /**
+         * @param infinitely {@code True} if should sleep infinitely.
+         */
+        public TestRunnable(boolean infinitely) {
+            this.infinitely = infinitely;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            try {
+                while (infinitely)
+                    sleepASec();
+            }
+            catch (InterruptedException e) {
+                info("Got interrupted exception while sleeping: " + e);
+            }
+        }
+    }
+
+    /**
+     * @throws InterruptedException If interrupted.
+     */
+    private void sleepASec() throws InterruptedException {
+        Thread.sleep(1000);
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 9e6c72e0e6f9c..7e299ca9a8cff 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -74,6 +74,7 @@
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointTaskSelfTest;
 import org.apache.ignite.internal.managers.communication.GridCommunicationManagerListenersSelfTest;
 import org.apache.ignite.internal.processors.compute.PublicThreadpoolStarvationTest;
+import org.apache.ignite.internal.util.StripedExecutorTest;
 import org.apache.ignite.p2p.GridMultinodeRedeployContinuousModeSelfTest;
 import org.apache.ignite.p2p.GridMultinodeRedeployIsolatedModeSelfTest;
 import org.apache.ignite.p2p.GridMultinodeRedeployPrivateModeSelfTest;
@@ -156,6 +157,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteRoundRobinErrorAfterClientReconnectTest.class);
         suite.addTestSuite(PublicThreadpoolStarvationTest.class);
         suite.addTestSuite(IgniteComputeJobOneThreadTest.class);
+        suite.addTestSuite(StripedExecutorTest.class);
 
         return suite;
     }

From 8e87a11b8bbc23578deddba05a3715e79adef554 Mon Sep 17 00:00:00 2001
From: apopov <tank2.alex@gmail.com>
Date: Fri, 20 Oct 2017 16:06:09 +0300
Subject: [PATCH 440/446] GG-12974 Don't use org.apache.spark.Logging in
 Ignite-spark

---
 examples/config/spark/example-shared-rdd.xml  |  83 ++++++++++++
 examples/pom-standalone-lgpl.xml              |  12 ++
 examples/pom-standalone.xml                   |  12 ++
 examples/pom.xml                              |  41 +++++-
 .../spark/ScalarSharedRDDExample.scala        |  89 ++++++++++++
 .../examples/spark/SharedRDDExample.java      | 127 ++++++++++++++++++
 .../ignite/examples/spark/package-info.java   |  22 +++
 .../IgniteExamplesJ8SelfTestSuite.java        |   1 +
 .../examples/ScalarExamplesSelfTest.scala     |   6 +
 .../examples/SharedRDDExampleSelfTest.java    |  37 +++++
 .../IgniteExamplesSparkSelfTestSuite.java     |  46 +++++++
 .../apache/ignite/spark/IgniteContext.scala   |  22 ++-
 .../ignite/spark/JavaIgniteContext.scala      |   6 +
 13 files changed, 497 insertions(+), 7 deletions(-)
 create mode 100644 examples/config/spark/example-shared-rdd.xml
 create mode 100644 examples/src/main/scala/org/apache/ignite/scalar/examples/spark/ScalarSharedRDDExample.scala
 create mode 100644 examples/src/main/spark/org/apache/ignite/examples/spark/SharedRDDExample.java
 create mode 100644 examples/src/main/spark/org/apache/ignite/examples/spark/package-info.java
 create mode 100644 examples/src/test/spark/org/apache/ignite/spark/examples/SharedRDDExampleSelfTest.java
 create mode 100644 examples/src/test/spark/org/apache/ignite/spark/testsuites/IgniteExamplesSparkSelfTestSuite.java

diff --git a/examples/config/spark/example-shared-rdd.xml b/examples/config/spark/example-shared-rdd.xml
new file mode 100644
index 0000000000000..83de6a3ecb114
--- /dev/null
+++ b/examples/config/spark/example-shared-rdd.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    Ignite Spring configuration file to startup Ignite cache.
+
+    This file demonstrates how to configure cache using Spring. Provided cache
+    will be created on node startup.
+
+    When starting a standalone node, you need to execute the following command:
+    {IGNITE_HOME}/bin/ignite.{bat|sh} examples/config/example-shared-rdd.xml
+
+    When starting Ignite from Java IDE, pass path to this file to Ignition:
+    Ignition.start("examples/config/example-shared-rdd.xml");
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+    <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+        <property name="cacheConfiguration">
+            <!-- SharedRDD cache example configuration (Atomic mode). -->
+            <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                <!-- Set a cache name. -->
+                <property name="name" value="sharedRDD"/>
+                <!-- Set a cache mode. -->
+                <property name="cacheMode" value="PARTITIONED"/>
+                <!-- Index Integer pairs used in the example. -->
+                <property name="indexedTypes">
+                    <list>
+                        <value>java.lang.Integer</value>
+                        <value>java.lang.Integer</value>
+                    </list>
+                </property>
+                <!-- Set atomicity mode. -->
+                <property name="atomicityMode" value="ATOMIC"/>
+                <!-- Configure a number of backups. -->
+                <property name="backups" value="1"/>
+            </bean>
+        </property>
+
+        <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. -->
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <!--
+                        Ignite provides several options for automatic discovery that can be used
+                        instead os static IP based discovery. For information on all options refer
+                        to our documentation: http://apacheignite.readme.io/docs/cluster-config
+                    -->
+                    <!-- Uncomment static IP finder to enable static-based discovery of initial nodes. -->
+                    <!--<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">-->
+                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
+                        <property name="addresses">
+                            <list>
+                                <!-- In distributed environment, replace with actual host IP address. -->
+                                <value>127.0.0.1:47500..47509</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+    </bean>
+</beans>
diff --git a/examples/pom-standalone-lgpl.xml b/examples/pom-standalone-lgpl.xml
index 38146725a641b..f664514d5c83f 100644
--- a/examples/pom-standalone-lgpl.xml
+++ b/examples/pom-standalone-lgpl.xml
@@ -32,6 +32,7 @@
         <!-- will be changed by profile activation. allows to combine profiles. -->
         <lgpl.folder>src/main/java</lgpl.folder>
         <java8.folder>src/main/java</java8.folder>
+        <spark.folder>src/main/java</spark.folder>
         <java.ver>1.7</java.ver>
     </properties>
 
@@ -100,12 +101,22 @@
         <profile>
             <id>scala</id>
 
+            <properties>
+                <spark.folder>src/main/spark</spark.folder>
+            </properties>
+
             <dependencies>
                 <dependency>
                     <groupId>org.apache.ignite</groupId>
                     <artifactId>ignite-scalar</artifactId>
                     <version>to_be_replaced_by_ignite_version</version>
                 </dependency>
+
+                <dependency>
+                    <groupId>org.apache.ignite</groupId>
+                    <artifactId>ignite-spark</artifactId>
+                    <version>to_be_replaced_by_ignite_version</version>
+                </dependency>
             </dependencies>
 
             <build>
@@ -213,6 +224,7 @@
                                 <source>schema-import/src/main/java</source>
                                 <source>${lgpl.folder}</source>
                                 <source>${java8.folder}</source>
+                                <source>${spark.folder}</source>
                             </sources>
                         </configuration>
                     </execution>
diff --git a/examples/pom-standalone.xml b/examples/pom-standalone.xml
index 08c27fe675d53..029ca777ae242 100644
--- a/examples/pom-standalone.xml
+++ b/examples/pom-standalone.xml
@@ -32,6 +32,7 @@
         <!-- will be changed by profile activation. allows to combine profiles. -->
         <lgpl.folder>src/main/java</lgpl.folder>
         <java8.folder>src/main/java</java8.folder>
+        <spark.folder>src/main/java</spark.folder>
         <java.ver>1.7</java.ver>
     </properties>
 
@@ -100,12 +101,22 @@
         <profile>
             <id>scala</id>
 
+            <properties>
+                <spark.folder>src/main/spark</spark.folder>
+            </properties>
+
             <dependencies>
                 <dependency>
                     <groupId>org.apache.ignite</groupId>
                     <artifactId>ignite-scalar</artifactId>
                     <version>to_be_replaced_by_ignite_version</version>
                 </dependency>
+
+                <dependency>
+                    <groupId>org.apache.ignite</groupId>
+                    <artifactId>ignite-spark</artifactId>
+                    <version>to_be_replaced_by_ignite_version</version>
+                </dependency>
             </dependencies>
 
             <build>
@@ -214,6 +225,7 @@
                                 <source>schema-import/src/main/java</source>
                                 <source>${lgpl.folder}</source>
                                 <source>${java8.folder}</source>
+                                <source>${spark.folder}</source>
                             </sources>
                         </configuration>
                     </execution>
diff --git a/examples/pom.xml b/examples/pom.xml
index 255ce019fd147..85562f04ee08f 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -17,7 +17,8 @@
   limitations under the License.
 -->
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
@@ -111,6 +112,8 @@
         <!-- will be changed by profile activation. allows to combine profiles. -->
         <lgpl.folder>src/main/java</lgpl.folder>
         <java8.folder>src/main/java</java8.folder>
+        <spark.folder>src/main/java</spark.folder>
+        <spark.test.folder>src/test/java</spark.test.folder>
         <lgpl.test.folder>src/test/java</lgpl.test.folder>
         <java8.test.folder>src/test/java</java8.test.folder>
     </properties>
@@ -119,6 +122,11 @@
         <profile>
             <id>scala</id>
 
+            <properties>
+                <spark.folder>src/main/spark</spark.folder>
+                <spark.test.folder>src/test/spark</spark.test.folder>
+            </properties>
+
             <dependencies>
                 <dependency>
                     <groupId>org.apache.ignite</groupId>
@@ -138,6 +146,18 @@
                         </exclusion>
                     </exclusions>
                 </dependency>
+
+                <dependency>
+                    <groupId>org.apache.ignite</groupId>
+                    <artifactId>ignite-spark</artifactId>
+                    <version>${project.version}</version>
+                </dependency>
+
+                <dependency>
+                    <groupId>org.jboss.netty</groupId>
+                    <artifactId>netty</artifactId>
+                    <version>3.2.9.Final</version>
+                </dependency>
             </dependencies>
 
             <build>
@@ -153,6 +173,11 @@
         <profile>
             <id>scala-2.10</id>
 
+            <properties>
+                <spark.folder>src/main/spark</spark.folder>
+                <spark.test.folder>src/test/spark</spark.test.folder>
+            </properties>
+
             <dependencies>
                 <dependency>
                     <groupId>org.apache.ignite</groupId>
@@ -172,6 +197,18 @@
                         </exclusion>
                     </exclusions>
                 </dependency>
+
+                <dependency>
+                    <groupId>org.apache.ignite</groupId>
+                    <artifactId>ignite-spark_2.10</artifactId>
+                    <version>${project.version}</version>
+                </dependency>
+
+                <dependency>
+                    <groupId>org.jboss.netty</groupId>
+                    <artifactId>netty</artifactId>
+                    <version>3.2.9.Final</version>
+                </dependency>
             </dependencies>
 
             <build>
@@ -248,6 +285,7 @@
                                 <source>schema-import/src/main/java</source>
                                 <source>${lgpl.folder}</source>
                                 <source>${java8.folder}</source>
+                                <source>${spark.folder}</source>
                             </sources>
                         </configuration>
                     </execution>
@@ -261,6 +299,7 @@
                         <configuration>
                             <sources>
                                 <source>${lgpl.test.folder}</source>
+                                <source>${spark.test.folder}</source>
                                 <source>${java8.test.folder}</source>
                             </sources>
                         </configuration>
diff --git a/examples/src/main/scala/org/apache/ignite/scalar/examples/spark/ScalarSharedRDDExample.scala b/examples/src/main/scala/org/apache/ignite/scalar/examples/spark/ScalarSharedRDDExample.scala
new file mode 100644
index 0000000000000..18662e846000d
--- /dev/null
+++ b/examples/src/main/scala/org/apache/ignite/scalar/examples/spark/ScalarSharedRDDExample.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.scalar.examples.spark
+
+import org.apache.ignite.spark.{IgniteContext, IgniteRDD}
+import org.apache.log4j.{Level, Logger}
+import org.apache.spark.{SparkConf, SparkContext}
+
+/**
+  * This example demonstrates how to create an IgnitedRDD and share it with multiple spark workers.
+  * The goal of this particular example is to provide the simplest code example of this logic.
+  * <p>
+  * This example will start Ignite in the embedded mode and will start an IgniteContext on each Spark worker node.
+  * <p>
+  * The example can work in the standalone mode as well that can be enabled by setting IgniteContext's {@code isClient}
+  * property to {@code true} and running an Ignite node separately with `examples/config/spark/
+  * example-shared-rdd.xml` config.
+  * <p>
+  */
+object ScalarSharedRDDExample extends App {
+    // Spark Configuration.
+    private val conf = new SparkConf()
+        .setAppName("IgniteRDDExample")
+        .setMaster("local")
+        .set("spark.executor.instances", "2")
+
+    // Spark context.
+    val sparkContext = new SparkContext(conf)
+
+    // Adjust the logger to exclude the logs of no interest.
+    Logger.getRootLogger.setLevel(Level.ERROR)
+    Logger.getLogger("org.apache.ignite").setLevel(Level.INFO)
+
+    // Defines spring cache Configuration path.
+    private val CONFIG = "examples/config/spark/example-shared-rdd.xml"
+
+    // Creates Ignite context with above configuration.
+    val igniteContext = new IgniteContext(sparkContext, CONFIG, false)
+
+    // Creates an Ignite Shared RDD of Type (Int,Int) Integer Pair.
+    val sharedRDD: IgniteRDD[Int, Int] = igniteContext.fromCache[Int, Int]("sharedRDD")
+
+    // Fill the Ignite Shared RDD in with Int pairs.
+    sharedRDD.savePairs(sparkContext.parallelize(1 to 100000, 10).map(i => (i, i)))
+
+    // Transforming Pairs to contain their Squared value.
+    sharedRDD.mapValues(x => (x * x))
+
+    // Retrieve sharedRDD back from the Cache.
+    val transformedValues: IgniteRDD[Int, Int] = igniteContext.fromCache("sharedRDD")
+
+    // Perform some transformations on IgniteRDD and print.
+    val squareAndRootPair = transformedValues.map { case (x, y) => (x, Math.sqrt(y.toDouble)) }
+
+    println(">>> Transforming values stored in Ignite Shared RDD...")
+
+    // Filter out pairs which square roots are less than 100 and
+    // take the first five elements from the transformed IgniteRDD and print them.
+    squareAndRootPair.filter(_._2 < 100.0).take(5).foreach(println)
+
+    println(">>> Executing SQL query over Ignite Shared RDD...")
+
+    // Execute a SQL query over the Ignite Shared RDD.
+    val df = transformedValues.sql("select _val from Integer where _val < 100 and _val > 9 ")
+
+    // Show ten rows from the result set.
+    df.show(10)
+
+    // Close IgniteContext on all workers.
+    igniteContext.close(true)
+
+    // Stop SparkContext.
+    sparkContext.stop()
+}
diff --git a/examples/src/main/spark/org/apache/ignite/examples/spark/SharedRDDExample.java b/examples/src/main/spark/org/apache/ignite/examples/spark/SharedRDDExample.java
new file mode 100644
index 0000000000000..1dfc130e0f770
--- /dev/null
+++ b/examples/src/main/spark/org/apache/ignite/examples/spark/SharedRDDExample.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.spark;
+
+import org.apache.ignite.spark.JavaIgniteContext;
+import org.apache.ignite.spark.JavaIgniteRDD;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.function.PairFunction;
+import org.apache.spark.api.java.function.VoidFunction;
+import org.apache.spark.sql.DataFrame;
+import scala.Tuple2;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This example demonstrates how to create an JavaIgnitedRDD and share it with multiple spark workers. The goal of this
+ * particular example is to provide the simplest code example of this logic.
+ * <p>
+ * This example will start Ignite in the embedded mode and will start an JavaIgniteContext on each Spark worker node.
+ * <p>
+ * The example can work in the standalone mode as well that can be enabled by setting JavaIgniteContext's
+ * {@code standalone} property to {@code true} and running an Ignite node separately with
+ * `examples/config/spark/example-shared-rdd.xml` config.
+ */
+public class SharedRDDExample {
+    /**
+     * Executes the example.
+     * @param args Command line arguments, none required.
+     */
+    public static void main(String args[]) {
+        // Spark Configuration.
+        SparkConf sparkConf = new SparkConf()
+            .setAppName("JavaIgniteRDDExample")
+            .setMaster("local")
+            .set("spark.executor.instances", "2");
+
+        // Spark context.
+        JavaSparkContext sparkContext = new JavaSparkContext(sparkConf);
+
+        // Adjust the logger to exclude the logs of no interest.
+        Logger.getRootLogger().setLevel(Level.ERROR);
+        Logger.getLogger("org.apache.ignite").setLevel(Level.INFO);
+
+        // Creates Ignite context with specific configuration and runs Ignite in the embedded mode.
+        JavaIgniteContext<Integer, Integer> igniteContext = new JavaIgniteContext<Integer, Integer>(
+            sparkContext,"examples/config/spark/example-shared-rdd.xml", false);
+
+        // Create a Java Ignite RDD of Type (Int,Int) Integer Pair.
+        JavaIgniteRDD<Integer, Integer> sharedRDD = igniteContext.<Integer, Integer>fromCache("sharedRDD");
+
+        // Define data to be stored in the Ignite RDD (cache).
+        List<Integer> data = new ArrayList<>(20);
+
+        for (int i = 0; i<20; i++) {
+            data.add(i);
+        }
+
+        // Preparing a Java RDD.
+        JavaRDD<Integer> javaRDD = sparkContext.<Integer>parallelize(data);
+
+        // Fill the Ignite RDD in with Int pairs. Here Pairs are represented as Scala Tuple2.
+        sharedRDD.savePairs(javaRDD.<Integer, Integer>mapToPair(new PairFunction<Integer, Integer, Integer>() {
+            @Override public Tuple2<Integer, Integer> call(Integer val) throws Exception {
+                return new Tuple2<Integer, Integer>(val, val);
+            }
+        }));
+
+        System.out.println(">>> Iterating over Ignite Shared RDD...");
+
+        // Iterate over the Ignite RDD.
+        sharedRDD.foreach(new VoidFunction<Tuple2<Integer, Integer>>() {
+            @Override public void call(Tuple2<Integer, Integer> tuple) throws Exception {
+                System.out.println("(" + tuple._1 + "," + tuple._2 + ")");
+            }
+        });
+
+        System.out.println(">>> Transforming values stored in Ignite Shared RDD...");
+
+        // Filter out even values as a transformed RDD.
+        JavaPairRDD<Integer, Integer> transformedValues =
+            sharedRDD.filter(new Function<Tuple2<Integer, Integer>, Boolean>() {
+                @Override public Boolean call(Tuple2<Integer, Integer> tuple) throws Exception {
+                    return tuple._2() % 2 == 0;
+                }
+            });
+
+        // Print out the transformed values.
+        transformedValues.foreach(new VoidFunction<Tuple2<Integer, Integer>>() {
+            @Override public void call(Tuple2<Integer, Integer> tuple) throws Exception {
+                System.out.println("(" + tuple._1 + "," + tuple._2 + ")");
+            }
+        });
+
+        System.out.println(">>> Executing SQL query over Ignite Shared RDD...");
+
+        // Execute SQL query over the Ignite RDD.
+        DataFrame df = sharedRDD.sql("select _val from Integer where _key < 9");
+
+        // Show the result of the execution.
+        df.show();
+
+        // Close IgniteContext on all the workers.
+        igniteContext.close(true);
+    }
+}
\ No newline at end of file
diff --git a/examples/src/main/spark/org/apache/ignite/examples/spark/package-info.java b/examples/src/main/spark/org/apache/ignite/examples/spark/package-info.java
new file mode 100644
index 0000000000000..f97a7b7f26841
--- /dev/null
+++ b/examples/src/main/spark/org/apache/ignite/examples/spark/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Basic examples for ignite functionality with spark.
+ */
+package org.apache.ignite.examples.spark;
\ No newline at end of file
diff --git a/examples/src/test/java8/org/apache/ignite/java8/testsuites/IgniteExamplesJ8SelfTestSuite.java b/examples/src/test/java8/org/apache/ignite/java8/testsuites/IgniteExamplesJ8SelfTestSuite.java
index 949324c367204..7b62ea8ffbd54 100644
--- a/examples/src/test/java8/org/apache/ignite/java8/testsuites/IgniteExamplesJ8SelfTestSuite.java
+++ b/examples/src/test/java8/org/apache/ignite/java8/testsuites/IgniteExamplesJ8SelfTestSuite.java
@@ -49,6 +49,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(IndexingBridgeMethodTest.class));
         suite.addTest(new TestSuite(CacheExamplesSelfTest.class));
         suite.addTest(new TestSuite(BasicExamplesSelfTest.class));
+
 //        suite.addTest(new TestSuite(ContinuationExamplesSelfTest.class));
 //        suite.addTest(new TestSuite(ContinuousMapperExamplesSelfTest.class));
 //        suite.addTest(new TestSuite(DeploymentExamplesSelfTest.class));
diff --git a/examples/src/test/scala/org/apache/ignite/scalar/tests/examples/ScalarExamplesSelfTest.scala b/examples/src/test/scala/org/apache/ignite/scalar/tests/examples/ScalarExamplesSelfTest.scala
index 94c41ad5a6482..28e509ed1023c 100644
--- a/examples/src/test/scala/org/apache/ignite/scalar/tests/examples/ScalarExamplesSelfTest.scala
+++ b/examples/src/test/scala/org/apache/ignite/scalar/tests/examples/ScalarExamplesSelfTest.scala
@@ -18,6 +18,7 @@
 package org.apache.ignite.scalar.tests.examples
 
 import org.apache.ignite.scalar.examples._
+import org.apache.ignite.scalar.examples.spark._
 import org.apache.ignite.scalar.scalar
 import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest
 import org.scalatest.junit.JUnitSuiteLike
@@ -95,4 +96,9 @@ class ScalarExamplesSelfTest extends GridAbstractExamplesTest with JUnitSuiteLik
     def testScalarSnowflakeSchemaExample() {
         ScalarSnowflakeSchemaExample.main(EMPTY_ARGS)
     }
+
+    /** */
+    def testScalarSharedRDDExample() {
+        ScalarSharedRDDExample.main(EMPTY_ARGS)
+    }
 }
diff --git a/examples/src/test/spark/org/apache/ignite/spark/examples/SharedRDDExampleSelfTest.java b/examples/src/test/spark/org/apache/ignite/spark/examples/SharedRDDExampleSelfTest.java
new file mode 100644
index 0000000000000..55149ff795f3e
--- /dev/null
+++ b/examples/src/test/spark/org/apache/ignite/spark/examples/SharedRDDExampleSelfTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.examples;
+
+import junit.framework.TestCase;
+import org.apache.ignite.examples.spark.SharedRDDExample;
+import org.junit.Test;
+
+/**
+ * SharedRDD  examples self test.
+ */
+public class SharedRDDExampleSelfTest extends TestCase {
+    static final String[] EMPTY_ARGS = new String[0];
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testSharedRDDExample() throws Exception {
+        SharedRDDExample.main(EMPTY_ARGS);
+    }
+
+}
diff --git a/examples/src/test/spark/org/apache/ignite/spark/testsuites/IgniteExamplesSparkSelfTestSuite.java b/examples/src/test/spark/org/apache/ignite/spark/testsuites/IgniteExamplesSparkSelfTestSuite.java
new file mode 100644
index 0000000000000..bff914cb8d763
--- /dev/null
+++ b/examples/src/test/spark/org/apache/ignite/spark/testsuites/IgniteExamplesSparkSelfTestSuite.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.spark.examples.SharedRDDExampleSelfTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_OVERRIDE_MCAST_GRP;
+
+/**
+ * Examples test suite.
+ * <p>
+ * Contains only Spring ignite examples tests.
+ */
+public class IgniteExamplesSparkSelfTestSuite extends TestSuite {
+    /**
+     * @return Suite.
+     * @throws Exception If failed.
+     */
+    public static TestSuite suite() throws Exception {
+        System.setProperty(IGNITE_OVERRIDE_MCAST_GRP,
+            GridTestUtils.getNextMulticastGroup(IgniteExamplesSparkSelfTestSuite.class));
+
+        TestSuite suite = new TestSuite("Ignite Spark Examples Test Suite");
+
+        suite.addTest(new TestSuite(SharedRDDExampleSelfTest.class));
+
+        return suite;
+    }
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
index 04139d1e91268..4d2c3b53f2e97 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
@@ -22,7 +22,8 @@ import org.apache.ignite.configuration.{CacheConfiguration, IgniteConfiguration}
 import org.apache.ignite.internal.IgnitionEx
 import org.apache.ignite.internal.util.IgniteUtils
 import org.apache.spark.sql.SQLContext
-import org.apache.spark.{Logging, SparkContext}
+import org.apache.spark.SparkContext
+import org.apache.log4j.Logger
 
 /**
  * Ignite context.
@@ -34,7 +35,7 @@ class IgniteContext(
     @transient val sparkContext: SparkContext,
     cfgF: () ⇒ IgniteConfiguration,
     standalone: Boolean = true
-    ) extends Serializable with Logging {
+    ) extends Serializable  {
     private val cfgClo = new Once(cfgF)
 
     private val igniteHome = IgniteUtils.getIgniteHome
@@ -47,7 +48,7 @@ class IgniteContext(
         if (workers <= 0)
             throw new IllegalStateException("No Spark executors found to start Ignite nodes.")
 
-        logInfo("Will start Ignite nodes on " + workers + " workers")
+        Logging.log.info("Will start Ignite nodes on " + workers + " workers")
 
         // Start ignite server node on each worker in server mode.
         sparkContext.parallelize(1 to workers, workers).foreachPartition(it ⇒ ignite())
@@ -126,7 +127,7 @@ class IgniteContext(
         val home = IgniteUtils.getIgniteHome
 
         if (home == null && igniteHome != null) {
-            logInfo("Setting IGNITE_HOME from driver not as it is not available on this worker: " + igniteHome)
+            Logging.log.info("Setting IGNITE_HOME from driver not as it is not available on this worker: " + igniteHome)
 
             IgniteUtils.nullifyHomeDirectory()
 
@@ -143,7 +144,7 @@ class IgniteContext(
         }
         catch {
             case e: IgniteException ⇒
-                logError("Failed to start Ignite.", e)
+                Logging.log.error("Failed to start Ignite.", e)
 
                 throw e
         }
@@ -161,7 +162,7 @@ class IgniteContext(
                 sparkContext.getExecutorStorageStatus.length)
 
             if (workers > 0) {
-                logInfo("Will stop Ignite nodes on " + workers + " workers")
+                Logging.log.info("Will stop Ignite nodes on " + workers + " workers")
 
                 // Start ignite server node on each worker in server mode.
                 sparkContext.parallelize(1 to workers, workers).foreachPartition(it ⇒ doClose())
@@ -200,3 +201,12 @@ private class Once(clo: () ⇒ IgniteConfiguration) extends Serializable {
         res
     }
 }
+
+/**
+  * Spark uses log4j by default. Using this logger in IgniteContext as well.
+  *
+  * This object is used to avoid problems with log4j serialization.
+  */
+object Logging extends Serializable {
+    @transient lazy val log = Logger.getLogger(classOf[IgniteContext])
+}
\ No newline at end of file
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/JavaIgniteContext.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/JavaIgniteContext.scala
index 689a22d12710f..d8a521b563102 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/JavaIgniteContext.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/JavaIgniteContext.scala
@@ -51,6 +51,12 @@ class JavaIgniteContext[K, V](
         })
     }
 
+    def this(sc: JavaSparkContext, springUrl: String, standalone: Boolean) {
+        this(sc, new IgniteOutClosure[IgniteConfiguration] {
+            override def apply() = IgnitionEx.loadConfiguration(springUrl).get1()
+        }, standalone)
+    }
+
     def fromCache(cacheName: String): JavaIgniteRDD[K, V] =
         JavaIgniteRDD.fromIgniteRDD(new IgniteRDD[K, V](ic, cacheName, null, false))
 

From d8fdd49794b8b672da9c57f985c1d87461ebeae4 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Mon, 19 Jun 2017 14:34:58 +0300
Subject: [PATCH 441/446] Backported IGNITE-5100 Do not remove active exchange
 futures from ExchangeFutureSet.

(cherry picked from commit ecb9e4d)
---
 .../apache/ignite/IgniteSystemProperties.java |  3 +
 .../GridCachePartitionExchangeManager.java    | 15 +++-
 ...ePartitionExchangeManagerHistSizeTest.java | 76 +++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |  4 +-
 4 files changed, 93 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 5f13d951c07db..5f9e88b2c6ca1 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -156,6 +156,9 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_CONSOLE_APPENDER = "IGNITE_CONSOLE_APPENDER";
 
+    /** Maximum size for exchange history. Default value is {@code 1000}.*/
+    public static final String IGNITE_EXCHANGE_HISTORY_SIZE = "IGNITE_EXCHANGE_HISTORY_SIZE";
+
     /**
      * Name of the system property defining name of command line program.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 4809fd2c0df73..b45f36fb9b5d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -117,7 +117,8 @@
  */
 public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedManagerAdapter<K, V> {
     /** Exchange history size. */
-    private static final int EXCHANGE_HISTORY_SIZE = 1000;
+    private static final int EXCHANGE_HISTORY_SIZE =
+        IgniteSystemProperties.getInteger(IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE, 1_000);
 
     /** Atomic reference for pending timeout object. */
     private AtomicReference<ResendTimeoutObject> pendingResend = new AtomicReference<>();
@@ -1986,8 +1987,14 @@ private ExchangeFutureSet() {
             GridDhtPartitionsExchangeFuture fut) {
             GridDhtPartitionsExchangeFuture cur = super.addx(fut);
 
-            while (size() > EXCHANGE_HISTORY_SIZE)
+            while (size() > EXCHANGE_HISTORY_SIZE) {
+                GridDhtPartitionsExchangeFuture last = last();
+
+                if (!last.isDone())
+                    break;
+
                 removeLast();
+            }
 
             // Return the value in the set.
             return cur == null ? fut : cur;
@@ -1995,8 +2002,8 @@ private ExchangeFutureSet() {
 
         /** {@inheritDoc} */
         @Nullable @Override public synchronized GridDhtPartitionsExchangeFuture removex(
-            GridDhtPartitionsExchangeFuture val
-        ) {
+            GridDhtPartitionsExchangeFuture val) {
+
             return super.removex(val);
         }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java
new file mode 100644
index 0000000000000..5f1c3a885a946
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE;
+
+/**
+ * Test exchange history size parameter effect.
+ */
+public class GridCachePartitionExchangeManagerHistSizeTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private String oldHistVal;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        oldHistVal = System.getProperty(IGNITE_EXCHANGE_HISTORY_SIZE);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        if (oldHistVal != null)
+            System.setProperty(IGNITE_EXCHANGE_HISTORY_SIZE, oldHistVal);
+        else
+            System.clearProperty(IGNITE_EXCHANGE_HISTORY_SIZE);
+    }
+
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSingleExchangeHistSize() throws Exception {
+        System.setProperty(IGNITE_EXCHANGE_HISTORY_SIZE, "1");
+
+        startGridsMultiThreaded(10);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 35bcc42f9fe5d..b368129ffe8f0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.GridCachePartitionExchangeManagerHistSizeTest;
 import org.apache.ignite.internal.processors.cache.CacheKeepBinaryTransactionTest;
 import org.apache.ignite.internal.processors.cache.CacheNearReaderUpdateTest;
 import org.apache.ignite.internal.processors.cache.CacheRebalancingSelfTest;
@@ -35,7 +36,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest;
 import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest;
 
 /**
@@ -73,6 +73,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(GridCachePartitionEvictionDuringReadThroughSelfTest.class);
 
+        suite.addTestSuite(GridCachePartitionExchangeManagerHistSizeTest.class);
+
         return suite;
     }
 }

From b7ba1d4eaf54f42639fdc3c2defb11d93f225c64 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 23 Oct 2017 19:39:26 +0300
Subject: [PATCH 442/446] Backported IGNITE-5116: Fixed stale DML plan caching
 in DmlStatementsProcessor.

---
 .../query/h2/DmlStatementsProcessor.java      |  9 ++++++++
 .../processors/query/h2/IgniteH2Indexing.java |  5 ++++-
 ...teCacheAbstractInsertSqlQuerySelfTest.java |  3 ++-
 .../IgniteCacheInsertSqlQuerySelfTest.java    | 22 +++++++++++++++++++
 4 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
index 40307581b93e1..d1cef3292ea68 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
@@ -114,6 +114,15 @@ public class DmlStatementsProcessor {
         this.indexing = indexing;
     }
 
+    /**
+     * Handle cache stop.
+     *
+     * @param spaceName Cache name.
+     */
+    public void onCacheStop(String spaceName) {
+        planCache.remove(spaceName);
+    }
+
     /**
      * Execute DML statement, possibly with few re-attempts in case of concurrent data modifications.
      *
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index e4b0c1feb7a21..5d51e13412ee5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -1367,7 +1367,9 @@ public static Session session(Connection c) {
             cancel = new GridQueryCancel();
 
         QueryCursorImpl<List<?>> cursor = new QueryCursorImpl<>(
-            runQueryTwoStep(cctx, twoStepQry, cctx.keepBinary(), enforceJoinOrder, qry.getTimeout(), cancel), cancel);
+            runQueryTwoStep(cctx, twoStepQry, cctx.keepBinary(), enforceJoinOrder, qry.getTimeout(), cancel),
+
+            cancel);
 
         cursor.fieldsMeta(meta);
 
@@ -2038,6 +2040,7 @@ private void createSqlFunctions(String schema, Class<?>[] clss) throws IgniteChe
         if (rmv != null) {
             space2schema.remove(emptyIfNull(rmv.spaceName));
             mapQryExec.onCacheStop(ccfg.getName());
+            dmlProc.onCacheStop(rmv.spaceName);
 
             rmv.onDrop();
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
index 86d01c788ee5b..922ccb5324246 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
@@ -301,6 +301,7 @@ final void createBinaryCaches() {
         ignite(0).cache("K2P").clear();
         ignite(0).cache("K22P").clear();
         ignite(0).cache("I2I").clear();
+        ignite(0).cache("I2AT").clear();
 
         if (isBinaryMarshaller()) {
             ignite(0).cache("K32P").clear();
@@ -357,7 +358,7 @@ Object createPerson2(int id, String name, int valFld) {
      * @param idxTypes Indexed types.
      * @return Cache configuration.
      */
-    private static CacheConfiguration cacheConfig(String name, boolean partitioned, boolean escapeSql, Class<?>... idxTypes) {
+    static CacheConfiguration cacheConfig(String name, boolean partitioned, boolean escapeSql, Class<?>... idxTypes) {
         return new CacheConfiguration()
             .setName(name)
             .setCacheMode(partitioned ? CacheMode.PARTITIONED : CacheMode.REPLICATED)
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
index e9c21dc2cea5d..28953f33db805 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
@@ -202,4 +202,26 @@ public void testCustomIdentity() {
 
         assertEquals(createPerson(2, "Alex"), p.get(new Key4(2)));
     }
+
+    /** */
+    public void testCacheRestartHandling() {
+        IgniteCache<Integer, IgniteCacheUpdateSqlQuerySelfTest.AllTypes> p = ignite(0).cache("I2AT");
+
+        p.query(new SqlFieldsQuery("insert into AllTypes(_key, _val) values (1, ?)")
+            .setArgs(new IgniteCacheUpdateSqlQuerySelfTest.AllTypes(1L)));
+
+        p.destroy();
+
+        p = ignite(0).getOrCreateCache(cacheConfig("I2AT", true, false, Integer.class,
+            IgniteCacheUpdateSqlQuerySelfTest.AllTypes.class));
+
+        p.query(new SqlFieldsQuery("insert into AllTypes(_key, _val, dateCol) values (1, ?, null)")
+            .setArgs(new IgniteCacheUpdateSqlQuerySelfTest.AllTypes(1L)));
+
+        IgniteCacheUpdateSqlQuerySelfTest.AllTypes exp = new IgniteCacheUpdateSqlQuerySelfTest.AllTypes(1L);
+
+        exp.dateCol = null;
+
+        assertEquals(exp, p.get(1));
+    }
 }

From 13e5520e961bf8ec5e08969af0b5dfae5e64906b Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 24 Oct 2017 12:52:21 +0300
Subject: [PATCH 443/446] Fixed compilation.

---
 .../internal/GridCachePartitionExchangeManagerHistSizeTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java
index 5f1c3a885a946..934f40d94376f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java
@@ -42,7 +42,7 @@ public class GridCachePartitionExchangeManagerHistSizeTest extends GridCommonAbs
 
         ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
 
-        cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME));
+        cfg.setCacheConfiguration(new CacheConfiguration());
 
         return cfg;
     }

From 3c344c4e935c43c2e536afe4a7c6a3f76abf6ab9 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Mon, 23 Oct 2017 12:50:15 +0300
Subject: [PATCH 444/446] IGNITE-6071 White list of exceptions to suppress in
 createTcpClient.

---
 .../tcp/TcpCommunicationSpi.java              |   3 +-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   2 +
 ...tConnectAfterCommunicationFailureTest.java | 156 ++++++++++++++++++
 .../IgniteClientReconnectTestSuite.java       |   4 +-
 4 files changed, 163 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientConnectAfterCommunicationFailureTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 0743ea47c3e67..066692d959d5b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3144,7 +3144,8 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                 }
             }
 
-            if (X.hasCause(errs, ConnectException.class, HandshakeException.class))
+            if (!X.hasCause(errs, SocketTimeoutException.class, HandshakeTimeoutException.class,
+                IgniteSpiOperationTimeoutException.class))
                 throw errs;
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index b1fd7e23f308f..aa33e2ea809c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -721,6 +721,8 @@ else if (!spi.failureDetectionTimeoutEnabled() && reconCnt == spi.getReconnectCo
                     finally {
                         U.closeQuiet(sock);
                     }
+
+                    U.sleep(200);
                 }
             }
             catch (Throwable t) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientConnectAfterCommunicationFailureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientConnectAfterCommunicationFailureTest.java
new file mode 100644
index 0000000000000..301d5f24e4922
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientConnectAfterCommunicationFailureTest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Arrays;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests client to be able restore connection to cluster on subsequent attempts after communication problems.
+ */
+public class IgniteClientConnectAfterCommunicationFailureTest extends GridCommonAbstractTest {
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setNetworkTimeout(500);
+        cfg.setCommunicationSpi(new TcpCommunicationSpi(gridName.contains("block")));
+
+        if (gridName.contains("client")) {
+            cfg.setClientMode(true);
+        }
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientReconnects() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+        startGrid("client-block");
+
+        assertEquals(1, srv2.cluster().forClients().nodes().size());
+        assertEquals(1, srv1.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientThreadsSuspended() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+        Ignite client = startGrid("client");
+
+        boolean blockedAnything = false;
+
+        for (Thread thread : Thread.getAllStackTraces().keySet()) {
+            if (thread.getName().contains("%client%")) {
+                thread.suspend();
+                blockedAnything = true;
+            }
+        }
+
+        Thread.sleep(10000);
+
+        for (Thread thread : Thread.getAllStackTraces().keySet()) {
+            if (thread.getName().contains("%client%"))
+                thread.resume();
+        }
+
+        for (int j = 0; j < 10; j++) {
+            boolean topOk = true;
+
+            for (Ignite node : Arrays.asList(srv1, srv2, client)) {
+                if (node.cluster().nodes().size() != 3) {
+                    U.warn(log, "Grid size is incorrect (will re-run check in 1000 ms) " +
+                        "[name=" + node.name() + ", size=" + node.cluster().nodes().size() + ']');
+
+                    topOk = false;
+
+                    break;
+                }
+            }
+
+            if (topOk)
+                return;
+            else
+                Thread.sleep(1000);
+        }
+
+        assertTrue(blockedAnything);
+        assertEquals(1, srv2.cluster().forClients().nodes().size());
+        assertEquals(1, srv1.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * Will never connect with the first node id, normal operation after.
+     */
+    private class TcpCommunicationSpi extends org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi {
+        /**
+         * Whether this instance should actually block.
+         */
+        private final boolean isBlocking;
+
+        /**
+         * Local node ID that is prevented from creating connections.
+         */
+        private volatile UUID blockedNodeId = null;
+
+        /**
+         *
+         * @param isBlocking Whether this instance should actually block.
+         */
+        public TcpCommunicationSpi(boolean isBlocking) {
+            this.isBlocking = isBlocking;
+        }
+
+        /** {@inheritDoc} */
+        @Override protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
+            throws IgniteCheckedException {
+            if (blockHandshakeOnce(getLocalNode().id())) {
+                throw new IgniteCheckedException("Node is blocked");
+            }
+
+            return super.createTcpClient(node, connIdx);
+        }
+
+        /** Check if this connection is blocked. */
+        private boolean blockHandshakeOnce(UUID nodeId) {
+            if (isBlocking && (blockedNodeId == null || blockedNodeId.equals(nodeId))) {
+                blockedNodeId = nodeId;
+                return true;
+            }
+            return false;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index 03d3fe230c296..d0e907cde01f8 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.IgniteClientConnectAfterCommunicationFailureTest;
 import org.apache.ignite.internal.IgniteClientReconnectApiExceptionTest;
 import org.apache.ignite.internal.IgniteClientReconnectAtomicsTest;
 import org.apache.ignite.internal.IgniteClientReconnectBinaryContexTest;
@@ -43,6 +44,7 @@ public class IgniteClientReconnectTestSuite extends TestSuite {
     public static TestSuite suite() throws Exception {
         TestSuite suite = new TestSuite("Ignite Client Reconnect Test Suite");
 
+        suite.addTestSuite(IgniteClientConnectAfterCommunicationFailureTest.class);
         suite.addTestSuite(IgniteClientReconnectStopTest.class);
         suite.addTestSuite(IgniteClientReconnectApiExceptionTest.class);
         suite.addTestSuite(IgniteClientReconnectDiscoveryStateTest.class);
@@ -59,4 +61,4 @@ public static TestSuite suite() throws Exception {
 
         return suite;
     }
-}
\ No newline at end of file
+}

From 3cfd8149350a0b3ea754f48c73664ffdfde67c1e Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgian.com>
Date: Thu, 26 Oct 2017 16:48:21 +0700
Subject: [PATCH 445/446] WC-322 Annotations for REST serialization.

---
 .../apache/ignite/internal/visor/file/VisorFileBlockTask.java  | 3 ++-
 .../org/apache/ignite/internal/visor/log/VisorLogFile.java     | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlockTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlockTask.java
index 58a25eed8b673..8b73d20cec954 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlockTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlockTask.java
@@ -23,6 +23,7 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.NoSuchFileException;
+import org.apache.ignite.internal.LessNamingBean;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -50,7 +51,7 @@ public class VisorFileBlockTask extends VisorOneNodeTask<VisorFileBlockTask.Viso
      * Arguments for {@link VisorFileBlockTask}
      */
     @SuppressWarnings("PublicInnerClass")
-    public static class VisorFileBlockArg implements Serializable {
+    public static class VisorFileBlockArg implements Serializable, LessNamingBean {
         /** */
         private static final long serialVersionUID = 0L;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogFile.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogFile.java
index 211a171a5fbe8..59bd3d2c0222d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogFile.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogFile.java
@@ -19,12 +19,13 @@
 
 import java.io.File;
 import java.io.Serializable;
+import org.apache.ignite.internal.LessNamingBean;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
  * Visor log file.
  */
-public class VisorLogFile implements Serializable {
+public class VisorLogFile implements Serializable, LessNamingBean {
     /** */
     private static final long serialVersionUID = 0L;
 

From 926553479d52465241862a87c3d5fa750a77004b Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 7 Feb 2017 14:19:16 +0300
Subject: [PATCH 446/446] ignite-4552 Use for rmvQueue ConcurrentLinkedDeque
 instead of GridCircularBuffer to reduce memory usage. This closes #1465.

(cherry picked from commit e6ea938)

(cherry picked from commit 5544978)
---
 .../apache/ignite/IgniteSystemProperties.java |   3 +
 .../processors/cache/GridCacheProcessor.java  |  86 ++++++++++-
 .../distributed/dht/GridDhtCacheAdapter.java  |  10 +-
 .../dht/GridDhtLocalPartition.java            | 120 ++++++++++++----
 .../cache/CacheDeferredDeleteQueueTest.java   | 134 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite.java      |   2 +
 6 files changed, 322 insertions(+), 33 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 5f9e88b2c6ca1..3c46e2d457163 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -293,6 +293,9 @@ public final class IgniteSystemProperties {
     /** Maximum size for atomic cache queue delete history (default is 200 000 entries per partition). */
     public static final String IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE = "IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE";
 
+    /** Ttl of removed cache entries (ms). */
+    public static final String IGNITE_CACHE_REMOVED_ENTRIES_TTL = "IGNITE_CACHE_REMOVED_ENTRIES_TTL";
+
     /**
      * Comma separated list of addresses in format "10.100.22.100:45000,10.100.22.101:45000".
      * Makes sense only for {@link org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder}.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 673294184a365..b4f0163ed2f0a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -39,7 +39,6 @@
 import javax.cache.configuration.Factory;
 import javax.cache.integration.CacheLoader;
 import javax.cache.integration.CacheWriter;
-import javax.management.JMException;
 import javax.management.MBeanServer;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -79,6 +78,8 @@
 import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearAtomicCache;
@@ -99,6 +100,7 @@
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.plugin.CachePluginManager;
 import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -120,6 +122,7 @@
 import org.apache.ignite.spi.IgniteNodeValidationResult;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_REMOVED_ENTRIES_TTL;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
 import static org.apache.ignite.IgniteSystemProperties.getBoolean;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -847,6 +850,17 @@ private Set<String> internalCachesNames() {
 
         assert caches.containsKey(CU.MARSH_CACHE_NAME) : "Marshaller cache should be started";
         assert ctx.config().isDaemon() || caches.containsKey(CU.UTILITY_CACHE_NAME) : "Utility cache should be started";
+
+        if (!ctx.clientNode() && !ctx.isDaemon())
+            addRemovedItemsCleanupTask(Long.getLong(IGNITE_CACHE_REMOVED_ENTRIES_TTL, 10_000));
+
+    }
+
+    /**
+     * @param timeout Cleanup timeout.
+     */
+    private void addRemovedItemsCleanupTask(long timeout) {
+        ctx.timeout().addTimeoutObject(new RemovedItemsCleanupTask(timeout));
     }
 
     /** {@inheritDoc} */
@@ -3975,4 +3989,74 @@ private static class LocalAffinityFunction implements AffinityFunction {
             // No-op.
         }
     }
+
+    /**
+     *
+     */
+    private class RemovedItemsCleanupTask implements GridTimeoutObject {
+        /** */
+        private final IgniteUuid id = IgniteUuid.randomUuid();
+
+        /** */
+        private final long endTime;
+
+        /** */
+        private final long timeout;
+
+        /**
+         * @param timeout Timeout.
+         */
+        RemovedItemsCleanupTask(long timeout) {
+            this.timeout = timeout;
+            this.endTime = U.currentTimeMillis() + timeout;
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteUuid timeoutId() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long endTime() {
+            return endTime;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onTimeout() {
+            ctx.closure().runLocalSafe(new Runnable() {
+                @Override public void run() {
+                    try {
+                        for (GridCacheContext cacheCtx : sharedCtx.cacheContexts()) {
+                            if (!cacheCtx.isLocal() && cacheCtx.affinityNode()) {
+                                GridDhtPartitionTopology top = null;
+
+                                try {
+                                    top = cacheCtx.topology();
+                                }
+                                catch (IllegalStateException ignore) {
+                                    // Cache stopped.
+                                }
+
+                                if (top != null) {
+                                    for (GridDhtLocalPartition part : top.currentLocalPartitions())
+                                        part.cleanupRemoveQueue();
+                                }
+
+                                if (ctx.isStopping())
+                                    return;
+                            }
+                        }
+                    }
+                    catch (Exception e) {
+                        U.error(log, "Failed to cleanup removed cache items: " + e, e);
+                    }
+
+                    if (ctx.isStopping())
+                        return;
+
+                    addRemovedItemsCleanupTask(timeout);
+                }
+            }, true);
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index ac73907bdf175..0f71aa6000fa7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -1212,14 +1212,8 @@ private PartitionEntrySet(int partId) {
         GridDhtLocalPartition part = topology().localPartition(entry.partition(), AffinityTopologyVersion.NONE,
             false);
 
-        if (part != null) {
-            try {
-                part.onDeferredDelete(entry.key(), ver);
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to enqueue deleted entry [key=" + entry.key() + ", ver=" + ver + ']', e);
-            }
-        }
+        if (part != null)
+            part.onDeferredDelete(entry.key(), ver);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 44d3f92160770..480a537585a37 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -29,7 +29,6 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
 import org.apache.ignite.internal.processors.cache.CacheObject;
@@ -46,21 +45,20 @@
 import org.apache.ignite.internal.processors.cache.extras.GridCacheObsoleteEntryExtras;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.query.GridQueryProcessor;
-import org.apache.ignite.internal.util.GridCircularBuffer;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridCloseableIterator;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
-import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.jsr166.ConcurrentLinkedDeque8;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_REMOVED_ENTRIES_TTL;
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_OBJECT_UNLOADED;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
@@ -72,8 +70,13 @@
  */
 public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>, GridReservable, GridCacheConcurrentMap {
     /** Maximum size for delete queue. */
-    public static final int MAX_DELETE_QUEUE_SIZE = Integer.getInteger(IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE,
-        200_000);
+    public static final int MAX_DELETE_QUEUE_SIZE = Integer.getInteger(IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE, 200_000);
+
+    /** Maximum size for {@link #rmvQueue}. */
+    private final int rmvQueueMaxSize;
+
+    /** Removed items TTL. */
+    private final long rmvdEntryTtl;
 
     /** Static logger to avoid re-creation. */
     private static final AtomicReference<IgniteLogger> logRef = new AtomicReference<>();
@@ -109,7 +112,7 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
     private final ReentrantLock lock = new ReentrantLock();
 
     /** Remove queue. */
-    private final GridCircularBuffer<T2<KeyCacheObject, GridCacheVersion>> rmvQueue;
+    private final ConcurrentLinkedDeque8<RemovedEntryHolder> rmvQueue = new ConcurrentLinkedDeque8<>();
 
     /** Group reservations. */
     private final CopyOnWriteArrayList<GridDhtPartitionsReservation> reservations = new CopyOnWriteArrayList<>();
@@ -146,7 +149,9 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
         int delQueueSize = CU.isSystemCache(cctx.name()) ? 100 :
             Math.max(MAX_DELETE_QUEUE_SIZE / cctx.affinity().partitions(), 20);
 
-        rmvQueue = new GridCircularBuffer<>(U.ceilPow2(delQueueSize));
+        rmvQueueMaxSize = U.ceilPow2(delQueueSize);
+
+        rmvdEntryTtl = Long.getLong(IGNITE_CACHE_REMOVED_ENTRIES_TTL, 10_000);
     }
 
     /**
@@ -299,24 +304,42 @@ void onRemoved(GridDhtCacheEntry entry) {
     }
 
     /**
-     * @param key Removed key.
-     * @param ver Removed version.
-     * @throws IgniteCheckedException If failed.
+     *
      */
-    public void onDeferredDelete(KeyCacheObject key, GridCacheVersion ver) throws IgniteCheckedException {
-        try {
-            T2<KeyCacheObject, GridCacheVersion> evicted = rmvQueue.add(new T2<>(key, ver));
+    public void cleanupRemoveQueue() {
+        while (rmvQueue.sizex() >= rmvQueueMaxSize) {
+            RemovedEntryHolder item = rmvQueue.pollFirst();
 
-            if (evicted != null)
-                cctx.dht().removeVersionedEntry(evicted.get1(), evicted.get2());
+            if (item != null)
+                cctx.dht().removeVersionedEntry(item.key(), item.version());
         }
-        catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
 
-            throw new IgniteInterruptedCheckedException(e);
+        if (!cctx.isDrEnabled()) {
+            RemovedEntryHolder item = rmvQueue.peekFirst();
+
+            while (item != null && item.expireTime() < U.currentTimeMillis()) {
+                item = rmvQueue.pollFirst();
+
+                if (item == null)
+                    break;
+
+                cctx.dht().removeVersionedEntry(item.key(), item.version());
+
+                item = rmvQueue.peekFirst();
+            }
         }
     }
 
+    /**
+     * @param key Removed key.
+     * @param ver Removed version.
+     */
+    public void onDeferredDelete(KeyCacheObject key, GridCacheVersion ver) {
+        cleanupRemoveQueue();
+
+        rmvQueue.add(new RemovedEntryHolder(key, ver, rmvdEntryTtl));
+    }
+
     /**
      * Locks partition.
      */
@@ -807,11 +830,8 @@ private Iterator<GridDhtCacheEntry> unswapIterator(
      *
      */
     private void clearDeferredDeletes() {
-        rmvQueue.forEach(new CI1<T2<KeyCacheObject, GridCacheVersion>>() {
-            @Override public void apply(T2<KeyCacheObject, GridCacheVersion> t) {
-                cctx.dht().removeVersionedEntry(t.get1(), t.get2());
-            }
-        });
+        for (RemovedEntryHolder e : rmvQueue)
+            cctx.dht().removeVersionedEntry(e.key(), e.version());
     }
 
     /** {@inheritDoc} */
@@ -841,4 +861,56 @@ private void clearDeferredDeletes() {
             "empty", isEmpty(),
             "createTime", U.format(createTime));
     }
+
+    /**
+     * Removed entry holder.
+     */
+    private static class RemovedEntryHolder {
+        /** Cache key */
+        private final KeyCacheObject key;
+
+        /** Entry version */
+        private final GridCacheVersion ver;
+
+        /** Entry expire time. */
+        private final long expireTime;
+
+        /**
+         * @param key Key.
+         * @param ver Entry version.
+         * @param ttl TTL.
+         */
+        private RemovedEntryHolder(KeyCacheObject key, GridCacheVersion ver, long ttl) {
+            this.key = key;
+            this.ver = ver;
+
+            expireTime = U.currentTimeMillis() + ttl;
+        }
+
+        /**
+         * @return Key.
+         */
+        KeyCacheObject key() {
+            return key;
+        }
+
+        /**
+         * @return Version.
+         */
+        GridCacheVersion version() {
+            return ver;
+        }
+
+        /**
+         * @return item expired time
+         */
+        long expireTime() {
+            return expireTime;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(RemovedEntryHolder.class, this);
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java
new file mode 100644
index 0000000000000..b764d5b3d7852
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collection;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_REMOVED_ENTRIES_TTL;
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class CacheDeferredDeleteQueueTest extends GridCommonAbstractTest {
+    /** */
+    private static String ttlProp;
+
+    /** */
+    private static int NODES = 2;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        ttlProp = System.getProperty(IGNITE_CACHE_REMOVED_ENTRIES_TTL);
+
+        System.setProperty(IGNITE_CACHE_REMOVED_ENTRIES_TTL, "1000");
+
+        startGridsMultiThreaded(NODES);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        if (ttlProp != null)
+            System.setProperty(IGNITE_CACHE_REMOVED_ENTRIES_TTL, ttlProp);
+        else
+            System.clearProperty(IGNITE_CACHE_REMOVED_ENTRIES_TTL);
+
+        stopAllGrids();
+
+        super.afterTestsStopped();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeferredDeleteQueue() throws Exception {
+        testQueue(ATOMIC, false);
+
+        testQueue(TRANSACTIONAL, false);
+
+        testQueue(ATOMIC, true);
+
+        testQueue(TRANSACTIONAL, true);
+    }
+
+    /**
+     * @param atomicityMode Cache atomicity mode.
+     * @param nearCache {@code True} if need create near cache.
+     *
+     * @throws Exception If failed.
+     */
+    private void testQueue(CacheAtomicityMode atomicityMode, boolean nearCache) throws Exception {
+        CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>();
+
+        ccfg.setCacheMode(PARTITIONED);
+        ccfg.setAtomicityMode(atomicityMode);
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+        ccfg.setBackups(1);
+
+        if (nearCache)
+            ccfg.setNearConfiguration(new NearCacheConfiguration<Integer, Integer>());
+
+        IgniteCache<Integer, Integer> cache = ignite(0).createCache(ccfg);
+
+        try {
+            final int KEYS = cache.getConfiguration(CacheConfiguration.class).getAffinity().partitions() * 3;
+
+            for (int i = 0; i < KEYS; i++)
+                cache.put(i, i);
+
+            for (int i = 0; i < KEYS; i++)
+                cache.remove(i);
+
+            boolean wait = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    for (int i = 0; i < NODES; i++) {
+                        final GridDhtPartitionTopology top =
+                            ((IgniteKernal)ignite(i)).context().cache().cache(null).context().topology();
+
+                        for (GridDhtLocalPartition p : top.currentLocalPartitions()) {
+                            Collection<Object> rmvQueue = GridTestUtils.getFieldValue(p, "rmvQueue");
+
+                            if (!rmvQueue.isEmpty() || p.size() != 0)
+                                return false;
+                        }
+                    }
+
+                    return true;
+                }
+            }, 5000);
+
+            assertTrue("Failed to wait for rmvQueue cleanup.", wait);
+        }
+        finally {
+            ignite(0).destroyCache(ccfg.getName());
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index d461452ac3c44..159bc78c38ec3 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -50,6 +50,7 @@
 import org.apache.ignite.internal.managers.communication.IgniteIoTestMessagesTest;
 import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest;
 import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest;
+import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteQueueTest;
 import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest;
 import org.apache.ignite.internal.processors.cache.CacheEntryProcessorCopySelfTest;
 import org.apache.ignite.internal.processors.cache.CacheFutureExceptionSelfTest;
@@ -327,6 +328,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridCacheTxPartitionedLocalStoreSelfTest.class);
         suite.addTestSuite(IgniteCacheSystemTransactionsSelfTest.class);
         suite.addTestSuite(CacheDeferredDeleteSanitySelfTest.class);
+        suite.addTestSuite(CacheDeferredDeleteQueueTest.class);
 
         suite.addTest(IgniteCacheTcpClientDiscoveryTestSuite.suite());